diff --git a/MusicModelCreation/createMusicalFeaturesets.py b/MusicModelCreation/createMusicalFeaturesets.py new file mode 100644 index 0000000..3160a54 --- /dev/null +++ b/MusicModelCreation/createMusicalFeaturesets.py @@ -0,0 +1,110 @@ +''' +Thomas Matlak, Avi Vajpeyi, Avery Rapson +CS 310 Final Project + +Given textfiles with the musical notes in int format, this creates a pickle of +the attributes and classes for all the musical data stored in the text files +(each text file is for one class). + +The data is stored as frequencies of each note on a keyboard, and the class label +is stored in 'one hot' format. 10 pre cent of data present set aside as testing data. + + +Usage: +python createMusicalFeaturesets.py + +OUTPUT: notesData.pickle + A pickle with the attributes and classes for music data + pickle data continas: train_attribute ,train_class, test_attribute, test_class + +NOTE: Need to update the follwoing depending on usage of script +ROOT_DIR = root/directrory/where/text/reside +DataFile = ["emotion1.txt","emotion2.txt"...]) +''' + +from mido import MidiFile, MidiTrack, Message +import mido +import random +import pickle +from collections import Counter +import numpy as np +import os + + +''' +Assume we have the following as our 'LEXICON' + unique word list : [chair, table, spoon, television] + +Assume this is our current sample data: + String: I pulled my chair up to the table + +Create a training vector that holds the count of each lexicon word: + training vector : [1, 1, 0, 0] + (since chair table are in string, but spoon TV arnt) + + +Do this for all strings + +''' + +ROOT_DIR = "TrainingData/" +DataFile = ["NegExamples/sadSongs.txt","PosExamples/happySongs.txt"] + +pianoSize = 128 # notes 0 - 127 +# this also defines our lexicon + +# larger dataset, more memory gets used up MemoryError +def sample_handling(sample, classification): + featureset = [] + ''' + featureset = + [ + [[0 1 0 0 1 0 0 ...], [1, 0]] + [[0 1 0 0 1 1 1 ...], [0, 1]] + .... + ] + so the first list is the array of matches with the lexicon + the second is which classification the features falls into (yes or no) + ''' + with open(sample,'r') as f: + contents = f.readlines() + for l in contents: + notes = np.fromstring(l, dtype=int, sep=' ') + noteCount = np.zeros(pianoSize) + for note in notes: + noteCount[note] += 1 + noteCount = list(noteCount) + featureset.append([noteCount, classification]) + return featureset + + + +def create_feature_sets_and_labels(DataFile,test_size = 0.1): + features = [] + features += sample_handling(ROOT_DIR+DataFile[0],[0,1])# neg + features += sample_handling(ROOT_DIR+DataFile[1],[1,0]) # pos + random.shuffle(features) + ''' + does tf.argmax([output]) == tf.argmax([expectations]) will look like: + tf.argmax([55454, 342324]) == tf.argmax([1,0]) + ''' + + features = np.array(features) + testing_size = int(test_size*len(features)) + train_x = list(features[:,0][:-testing_size]) #[[5,8],[7,9]] --> [:,0] does [5,7] (all of the 0 elememts) ie the labels in this case + train_y = list(features[:,1][:-testing_size]) + + test_x = list(features[:,0][-testing_size:]) + test_y = list(features[:,1][-testing_size:]) + + + return train_x,train_y,test_x,test_y + + +if __name__ == '__main__': + train_x,train_y,test_x,test_y = create_feature_sets_and_labels(DataFile) + with open('notesData.pickle','wb') as f: + pickle.dump([train_x,train_y,test_x,test_y],f) # dump data as a list, into a file + # this saves the lexicon for pos and neg words + # every inputted value is converted to a lexicon saving this info + # a lot of memory! diff --git a/MusicModelCreation/midiNoteSegmenter.py b/MusicModelCreation/midiNoteSegmenter.py new file mode 100644 index 0000000..95af5cb --- /dev/null +++ b/MusicModelCreation/midiNoteSegmenter.py @@ -0,0 +1,81 @@ +''' +Thomas Matlak +CS 310 Final Project + +Takes directory containing midi files as input, produces a text file containing only the midi note values for the first 10 seconds of each musical piece. + +Usage: +python midiNoteSegments.py /path/to/midi/folder/ [/path/to/output/file.txt] +''' + +import sys, glob +from mido import MidiFile, MidiTrack, Message +from keras.layers import LSTM, Dense, Activation, Dropout +from keras.preprocessing import sequence +from keras.models import Sequential +from keras.optimizers import RMSprop +from sklearn.preprocessing import MinMaxScaler +import numpy as np +import mido +import csv + +indir = sys.argv[1] +outfile_name = indir + "/out.txt" + +if 2 < len(sys.argv): + outfile_name = sys.argv[2] + +midi_files = glob.glob(indir + "/*.mid") + +transposition_intervals = { + 'Cb': -11, + 'Gb': -6, + 'Db': -1, + 'Ab': -8, + 'Eb': -3, + 'Bb': -10, + 'F': -5, + 'C': 0, + 'G': -7, + 'D': -2, + 'A': -9, + 'E': -4, + 'B': -11, + 'F#': -6, + 'C#':-1 +} + +with open(outfile_name, 'wb') as outfile: + writer = csv.writer(outfile, delimiter=' ') + + for midi_file in midi_files: + mid = MidiFile(midi_file) + + notes = [] + + time = float(0) + prev = float(0) + + key = "C" + + for msg in mid: + if time >= 10: + break + ### this time is in seconds, not ticks + time += msg.time + + if msg.type == "key_signature": + key = msg.key + + if not msg.is_meta: + ### only interested in piano channel + if msg.channel == 0: + if msg.type == 'note_on': + # note in vector form to train on + note = msg.bytes() + # only interested in the note #and velocity. note message is in the form of [type, note, velocity] + note = note[1] #:3] + # note.append(time - prev) + prev = time + notes.append(note + transposition_intervals[key]) # this preserves the intervlas, but transposes a;; samples to C + writer.writerow(notes) diff --git a/MusicModelCreation/notesData.pickle b/MusicModelCreation/notesData.pickle new file mode 100644 index 0000000..63daa0e Binary files /dev/null and b/MusicModelCreation/notesData.pickle differ diff --git a/MusicModelCreation/trainMusicNN.py b/MusicModelCreation/trainMusicNN.py new file mode 100644 index 0000000..7198824 --- /dev/null +++ b/MusicModelCreation/trainMusicNN.py @@ -0,0 +1,158 @@ +import tensorflow as tf +import numpy as np +import pickle +import os +# from tensorflow.examples.tutorials.mnist import input_data +# mnist = input_data.read_data_sets("/tmp/data/", one_hot = True) +# from createMusicalFeaturesets import create_feature_sets_and_labels +train_x,train_y,test_x,test_y = pickle.load(open("notesData2.pickle", "rb")) + + +saveFile = "savedModels/musicModelpy27" + + +n_nodes_hl1 = 1000 +n_nodes_hl2 = 1000 +n_nodes_hl3 = 1000 + +n_classes = 2 +batch_size = 10 +hm_epochs = 9 + +input_data_size = len(train_x[0])# each train_x instance is one song, and so one lexicon of notes +print("DEBUG: input data size = "+str(input_data_size)) + +x = tf.placeholder('float') +y = tf.placeholder('float') + +hidden_1_layer = {'f_fum':n_nodes_hl1, + 'weight':tf.Variable(tf.random_normal([128, n_nodes_hl1])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))} + +hidden_2_layer = {'f_fum':n_nodes_hl2, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))} + +hidden_3_layer = {'f_fum':n_nodes_hl3, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))} + +output_layer = {'f_fum':None, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])), + 'bias':tf.Variable(tf.random_normal([n_classes])),} + + + +# Nothing changes +def neural_network_model(data): + ####INPUT LAYER (HIDDEN LAYER 1) + l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias']) + l1 = tf.nn.relu(l1) + ####HIDDEN LAYER 2 + l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias']) + l2 = tf.nn.relu(l2) + ####HIDDEN LAYER 3 + l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias']) + l3 = tf.nn.relu(l3) + ####OUTPUT LAYER + output = tf.matmul(l3,output_layer['weight']) + output_layer['bias'] + + return output + + +def train_neural_network(x): + prediction = neural_network_model(x) + cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y) ) + optimizer = tf.train.AdamOptimizer().minimize(cost) + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + # try: + # epoch = int(open(tf_log,'r').read().split('\n')[-2])+1 + # print('STARTING EPOCH:',epoch) + # except: + # epoch = 1 + batches_run = 0 + epoch = 1 + while epoch <= hm_epochs: + # if epoch != 1: + # #saver.restore(sess,'/'+saveFile) + # print("Should Restore Saved File") + epoch_loss = 1 + i=0 + while i < len(train_x): + start = i + end = i+batch_size + batch_x = np.array(train_x[start:end]) + batch_y = np.array(train_y[start:end]) + _, c = sess.run([optimizer, cost], feed_dict={x: batch_x, y: batch_y}) + epoch_loss += c + i+=batch_size + batches_run +=1 + print('Batch run:',batches_run,'/',batch_size,'| Epoch:', + epoch,'| Batch Loss:',c,) + saver.save(sess, saveFile) + print("Should Save session in "+ saveFile ) + print('Epoch', epoch+1, 'completed out of',hm_epochs,'loss:', epoch_loss) + # with open(tf_log,'a') as f: + # f.write(str(epoch)+'\n') + epoch +=1 + correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1)) + accuracy = tf.reduce_mean(tf.cast(correct, 'float')) + print('Trained',len(train_x),'samples.') + print('Tested',len(test_x),'samples.') + accPercent = accuracy.eval({x:test_x, y:test_y})*100 + print('Accuracy: '+ str(accPercent)+ '%') + + +saver = tf.train.Saver() +# tf_log = 'tf.log' ## SAVES EPOCH NUMBER +train_neural_network(x) + + +def test_neural_network(): + prediction = neural_network_model(x) + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + # for epoch in range(hm_epochs): + # try: + # y =2 + # # saver.restore(sess,'/'+saveFile) + # print("Restoring "+ saveFile ) + # except Exception as e: + # print(str(e)) + # epoch_loss = 0 + + correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1)) + accuracy = tf.reduce_mean(tf.cast(correct, 'float')) + + ## WHEN WE SAVE TESTING DATA SEPARATLY + # feature_sets = [] + # labels = [] + # counter = 0 + # with open('processed-test-set.csv', buffering=20000) as f: + # for line in f: + # try: + # features = list(eval(line.split('::')[0])) + # label = list(eval(line.split('::')[1])) + # feature_sets.append(features) + # labels.append(label) + # counter += 1 + # except: + # pass + + testx = np.array(test_x) + testy = np.array(test_y) + counter = len(test_x) + print(testx,testy) + print(test_x,test_y) + print('******RESULTS******') + print('Tested',counter,'samples.') + print('Accuracy:', accuracy.eval({x:testx, y:testy}) ) + + +#test_neural_network() + + +print ("\n\n\nFINISHED\n\n\n") +# x =os.remove("tf.log") +# print("removed :" + str(x)) diff --git a/MusicModelCreation/usingMusicNN.py b/MusicModelCreation/usingMusicNN.py new file mode 100644 index 0000000..1d86094 --- /dev/null +++ b/MusicModelCreation/usingMusicNN.py @@ -0,0 +1,134 @@ +''' +Thomas Matlak Avi Vajpeyi, Avery Rapson +CS 310 Final Project + +Loads the NN saved in the dir 'savedFile'. The function predictmood(input_midi_file) +takes a midi files in MIDO format and returns if it is happy or sad + +Usage: +python usingMusicNN.py +''' + +import tensorflow as tf +import json +from mido import MidiFile +import numpy as np +import tempfile + + +midiFile = "01.mid" +saveFile = "savedModels/musicModelpy27" + + +pianoSize = 128 + +print("Bad ass Neural Net being loaded...") + + +hm_data = 2000000 + + +n_nodes_hl1 = 1000 +n_nodes_hl2 = 1000 +n_nodes_hl3 = 1000 + +n_classes = 2 +batch_size = 10 +hm_epochs = 9 + + +x = tf.placeholder('float') +y = tf.placeholder('float') + + +current_epoch = tf.Variable(1) + +hidden_1_layer = {'f_fum':n_nodes_hl1, + 'weight':tf.Variable(tf.random_normal([pianoSize, n_nodes_hl1])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))} + +hidden_2_layer = {'f_fum':n_nodes_hl2, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))} + +hidden_3_layer = {'f_fum':n_nodes_hl3, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))} + +output_layer = {'f_fum':None, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])), + 'bias':tf.Variable(tf.random_normal([n_classes])),} + + + +def neural_network_model(data): + ####INPUT LAYER (HIDDEN LAYER 1) + l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias']) + l1 = tf.nn.relu(l1) + ####HIDDEN LAYER 2 + l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias']) + l2 = tf.nn.relu(l2) + ####HIDDEN LAYER 3 + l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias']) + l3 = tf.nn.relu(l3) + ####OUTPUT LAYER + output = tf.matmul(l3,output_layer['weight']) + output_layer['bias'] + return output + + + + + + +# +def predictmood(input_midi_file): + prediction = neural_network_model(x) + # with open('musicModel.pickle','rb') as f: + # lexicon = pickle.load(f) + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + saver = tf.train.import_meta_graph(saveFile+'.meta') + saver.restore(sess, saveFile) + #### CONVERT THE MIDI TO NOTES AND FEATURES (without [0,1]) + #### need it in the [0 112 1 1 0 0 0 ....] format + mid = input_midi_file + notes = [] + time = float(0) + prev = float(0) + for msg in mid: + if time >= 10: + break + ### this time is in seconds, not ticks + time += msg.time + if not msg.is_meta: + ### only interested in piano channel + if msg.channel == 0: + if msg.type == 'note_on': + # note in vector form to train on + note = msg.bytes() + # only interested in the note #and velocity. note message is in the form of [type, note, velocity] + note = note[1] #:3] + # note.append(time - prev) + prev = time + notes.append(note) + noteCount = np.zeros(pianoSize) + for note in notes: + noteCount[note] += 1 + noteCount = list(noteCount) + #features = np.array(list(features)) + # pos: [1,0] , argmax: 0 + # neg: [0,1] , argmax: 1 + result = (sess.run(tf.argmax(prediction.eval(feed_dict={x:[noteCount]}),1))) + if result[0] == 0: + return ("Sad") + elif result[0] == 1: + return ("Happy") + # with open('mood.txt', 'w') as outfile: + # mood_dict = dict() + # if result[0] == 0: + # mood_dict = {'Mood': "Happy"} + # elif result[0] == 1: + # mood_dict = {'Mood': "Sad"} + # json.dump(mood_dict, outfile) +# output.seek(0) #resets the pointer to the data of the file to the start +# return output diff --git a/README_CS310.md b/README_CS310.md new file mode 100644 index 0000000..e945c4a --- /dev/null +++ b/README_CS310.md @@ -0,0 +1,37 @@ +***Detecting Emotions in Music*** +**Thomas, Avi, and Avery** + +To extend the AI Duet program we added an extra step to the preexisting flow of music data from the client to the server, and back to the client. + +The files modified were: +-server/server.py +-server/savedModels/* +-server/usingMusicNN.py +-static/src/ai/AI.js +-static/src/interface/Glow.js +-static/src/style/common.scss +-static/src/style/glow.css + + + +**To create a music sentiment detection model:** + + ***-Store happy/sad songs in seperate folders.*** + + ***-Run XXX path/to/folder/with/midi/files*** + (This will generate a text file with the notes normailized to the C and c minor scales, and represents the notes as numbers.) + + ***-Run createMusicalFeatureSets.py*** + (This takes the text files and generates musical feature sets, which is the input data of frequencies of notes, and the labels for each set of note frequencies. This is then saved as a pickle, with the testing and training data sepearated) + + ***-Run trainMusicNN.py*** + (This takes the pickle created and passes the data through a NN. It saves the NN's weights as a model) + + + +**To use the sentiment detection model to classify a song according it its sentiment:** + + ***-Run usingMusicNN.py*** + (This takes the sentiment detection model and a midi file. It converts the MIDI file to a feature set of fequencies, which it passes through the trained NN. It prints the classification of the set.) + + diff --git a/server/predict.py b/server/predict.py index 27d0258..81f16cc 100644 --- a/server/predict.py +++ b/server/predict.py @@ -22,7 +22,7 @@ from magenta.models.melody_rnn import melody_rnn_model from magenta.models.melody_rnn import melody_rnn_sequence_generator from magenta.protobuf import generator_pb2 from magenta.protobuf import music_pb2 - +from usingMusicNN import predictmood import os import time @@ -68,8 +68,9 @@ def generate_midi(midi_data, total_seconds=10): # generate the output sequence generated_sequence = generator.generate(primer_sequence, generator_options) - + output = tempfile.NamedTemporaryFile() magenta.music.midi_io.sequence_proto_to_midi_file(generated_sequence, output.name) output.seek(0) return output + \ No newline at end of file diff --git a/server/savedModels/checkpoint b/server/savedModels/checkpoint new file mode 100755 index 0000000..5bce638 --- /dev/null +++ b/server/savedModels/checkpoint @@ -0,0 +1,2 @@ +model_checkpoint_path: "musicModel" +all_model_checkpoint_paths: "musicModel" diff --git a/server/savedModels/musicModel.data-00000-of-00001 b/server/savedModels/musicModel.data-00000-of-00001 new file mode 100755 index 0000000..c282a80 Binary files /dev/null and b/server/savedModels/musicModel.data-00000-of-00001 differ diff --git a/server/savedModels/musicModel.index b/server/savedModels/musicModel.index new file mode 100755 index 0000000..1376e25 Binary files /dev/null and b/server/savedModels/musicModel.index differ diff --git a/server/savedModels/musicModel.meta b/server/savedModels/musicModel.meta new file mode 100755 index 0000000..7a19ff7 Binary files /dev/null and b/server/savedModels/musicModel.meta differ diff --git a/server/savedModels/musicModelpy27.data-00000-of-00001 b/server/savedModels/musicModelpy27.data-00000-of-00001 new file mode 100644 index 0000000..1c6d03f Binary files /dev/null and b/server/savedModels/musicModelpy27.data-00000-of-00001 differ diff --git a/server/savedModels/musicModelpy27.index b/server/savedModels/musicModelpy27.index new file mode 100644 index 0000000..a7d60d8 Binary files /dev/null and b/server/savedModels/musicModelpy27.index differ diff --git a/server/savedModels/musicModelpy27.meta b/server/savedModels/musicModelpy27.meta new file mode 100644 index 0000000..03ee742 Binary files /dev/null and b/server/savedModels/musicModelpy27.meta differ diff --git a/server/server.py b/server/server.py index cb31fe8..6747e22 100644 --- a/server/server.py +++ b/server/server.py @@ -25,10 +25,26 @@ else: from io import StringIO import time import json +import mido +import tempfile +from usingMusicNN import predictmood +import numpy as np from flask import Flask app = Flask(__name__, static_url_path='', static_folder=os.path.abspath('../static')) +HappyNote = 0 +SadNote = 1 + +def HappyTrack(): + track = mido.MidiTrack() + track.append(mido.Message('note_on', note=HappyNote, velocity=3, time=6)) + return track + +def SadTrack(): + track = mido.MidiTrack() + track.append(mido.Message('note_on', note=SadNote, velocity=3, time=6)) + return track @app.route('/predict', methods=['POST']) def predict(): @@ -37,9 +53,27 @@ def predict(): midi_data = pretty_midi.PrettyMIDI(StringIO(''.join(chr(v) for v in values))) duration = float(request.args.get('duration')) ret_midi = generate_midi(midi_data, duration) - return send_file(ret_midi, attachment_filename='return.mid', - mimetype='audio/midi', as_attachment=True) + # Store the received midi file in a temporary file to be able to use it with mido + mfile = tempfile.NamedTemporaryFile() + midi_data.write(mfile) + mfile.seek(0) + + midofile = mido.MidiFile(mfile.name) + + mood = predictmood(midofile) + print mood + + # Add a new track with the first note indicating the mood + midi_to_mod = mido.MidiFile(ret_midi.name) + midi_to_mod.tracks.append(HappyTrack() if mood == 'happy' else SadTrack()) + + ret_file = tempfile.NamedTemporaryFile() + midi_to_mod.save(ret_file.name) + ret_file.seek(0) + + return send_file(ret_file, attachment_filename='return.mid', + mimetype='audio/midi', as_attachment=True) @app.route('/', methods=['GET', 'POST']) def index(): diff --git a/server/usingMusicNN.py b/server/usingMusicNN.py new file mode 100755 index 0000000..d009613 --- /dev/null +++ b/server/usingMusicNN.py @@ -0,0 +1,132 @@ +''' + Thomas Matlak Avi Vajpeyi, Avery Rapson + CS 310 Final Project + + Loads the NN saved in the dir 'savedFile'. The function predictmood(input_midi_file) + takes a midi files in MIDO format and returns if it is happy or sad + + Usage: + python usingMusicNN.py +''' + + +import tensorflow as tf +import json +from mido import MidiFile +import numpy as np +import tempfile + + +midiFile = "testMidi.mid" +saveFile = "savedModels/musicModel" + +pianoSize = 128 + + +n_nodes_hl1 = 1500 +n_nodes_hl2 = 1500 +n_nodes_hl3 = 1500 + +n_classes = 2 +hm_data = 2000000 + +batch_size = 32 +hm_epochs = 10 + +x = tf.placeholder('float') +y = tf.placeholder('float') + + +current_epoch = tf.Variable(1) + +hidden_1_layer = {'f_fum':n_nodes_hl1, + 'weight':tf.Variable(tf.random_normal([pianoSize, n_nodes_hl1])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl1]))} + +hidden_2_layer = {'f_fum':n_nodes_hl2, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl2]))} + +hidden_3_layer = {'f_fum':n_nodes_hl3, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])), + 'bias':tf.Variable(tf.random_normal([n_nodes_hl3]))} + +output_layer = {'f_fum':None, + 'weight':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])), + 'bias':tf.Variable(tf.random_normal([n_classes])),} + + + +def neural_network_model(data): + ####INPUT LAYER (HIDDEN LAYER 1) + l1 = tf.add(tf.matmul(data,hidden_1_layer['weight']), hidden_1_layer['bias']) + l1 = tf.nn.relu(l1) + ####HIDDEN LAYER 2 + l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias']) + l2 = tf.nn.relu(l2) + ####HIDDEN LAYER 3 + l3 = tf.add(tf.matmul(l2,hidden_3_layer['weight']), hidden_3_layer['bias']) + l3 = tf.nn.relu(l3) + ####OUTPUT LAYER + output = tf.matmul(l3,output_layer['weight']) + output_layer['bias'] + return output + + + + + + +# +def predictmood(input_midi_file): + output = tempfile.NamedTemporaryFile() + prediction = neural_network_model(x) + # with open('musicModel.pickle','rb') as f: + # lexicon = pickle.load(f) + with tf.Session() as sess: + sess.run(tf.global_variables_initializer()) + saver = tf.train.import_meta_graph('savedModels/musicModel.meta') + saver.restore(sess, 'savedModels/musicModel') + #### CONVERT THE MIDI TO NOTES AND FEATURES (without [0,1]) + #### need it in the [0 112 1 1 0 0 0 ....] format + mid = MidiFile(input_midi_file) + notes = [] + time = float(0) + prev = float(0) + for msg in mid: + if time >= 10: + break + ### this time is in seconds, not ticks + time += msg.time + if not msg.is_meta: + ### only interested in piano channel + if msg.channel == 0: + if msg.type == 'note_on': + # note in vector form to train on + note = msg.bytes() + # only interested in the note #and velocity. note message is in the form of [type, note, velocity] + note = note[1] #:3] + # note.append(time - prev) + prev = time + notes.append(note) + noteCount = np.zeros(pianoSize) + for note in notes: + noteCount[note] += 1 + noteCount = list(noteCount) + #features = np.array(list(features)) + # pos: [1,0] , argmax: 0 + # neg: [0,1] , argmax: 1 + result = (sess.run(tf.argmax(prediction.eval(feed_dict={x:[noteCount]}),1))) + if result[0] == 0: + output.write("Happy") + elif result[0] == 1: + output.write("Sad") + # with open('mood.txt', 'w') as outfile: + # mood_dict = dict() + # if result[0] == 0: + # mood_dict = {'Mood': "Happy"} + # elif result[0] == 1: + # mood_dict = {'Mood': "Sad"} + # json.dump(mood_dict, outfile) + output.seek(0) #resets the pointer to the data of the file to the start + return output + diff --git a/static/src/ai/AI.js b/static/src/ai/AI.js index 326e121..41fe88e 100644 --- a/static/src/ai/AI.js +++ b/static/src/ai/AI.js @@ -63,6 +63,16 @@ class AI extends events.EventEmitter{ this.emit('keyUp', note.midi, note.noteOff + now) } }) + // We added another track to the midi file that contains data we want to send back to the client + // the first note of this new track is used as a bool value for happy/sad + if (response.tracks[2]['notes'][0]['midi'] === 0) { + console.log('happy') + window.isSad = 0; + } + else if (response.tracks[2]['notes'][0]['midi'] === 1) { + console.log('sad') + window.isSad = 1; + } }) this._lastPhrase = -1 this.emit('sent') diff --git a/static/src/interface/Glow.js b/static/src/interface/Glow.js index 87d850c..dc43a4d 100644 --- a/static/src/interface/Glow.js +++ b/static/src/interface/Glow.js @@ -45,12 +45,30 @@ class Glow { this._aiVisible = false this._aiGlow.classList.remove('visible') this._userGlow.classList.add('visible') + // Update the class of glow to show the correct color + if (window.isSad) { + this._userGlow.classList.add('sad') + this._userGlow.classList.remove('happy') + } + else { + this._userGlow.classList.add('happy') + this._userGlow.classList.remove('sad') + } } } else { if (!this._aiVisible){ this._aiVisible = true this._aiGlow.classList.add('visible') this._userGlow.classList.remove('visible') + // Update the class of glow to show the correct color + if (window.isSad) { + this._userGlow.classList.add('sad') + this._userGlow.classList.remove('happy') + } + else { + this._userGlow.classList.add('happy') + this._userGlow.classList.remove('sad') + } } } } diff --git a/static/style/common.scss b/static/style/common.scss index 8ff849e..4093f07 100644 --- a/static/style/common.scss +++ b/static/style/common.scss @@ -1,4 +1,6 @@ $blue : rgb(30, 183, 235); $orange : rgb(249, 187, 45); +$green : rgb(0, 225, 0); +$red: rgb(255, 0, 0); $font-family: 'Quicksand', sans-serif; \ No newline at end of file diff --git a/static/style/glow.css b/static/style/glow.css index 31b4256..d485a20 100644 --- a/static/style/glow.css +++ b/static/style/glow.css @@ -15,9 +15,10 @@ } } - $glowReach : 60%; + /*$glowReach : 60%;*/ + $glowReach : 80%; - #ai { + /*#ai { opacity: 0; background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $orange 0%, black $glowReach); } @@ -25,5 +26,15 @@ #user { opacity: 0; background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $blue 0%, black $glowReach); + }*/ + + .happy { + opacity: 0; + background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $green 0%, black $glowReach); + } + + .sad { + opacity: 0; + background-image: radial-gradient(ellipse farthest-corner at 50% 0px, $red 0%, black $glowReach); } } \ No newline at end of file