ale/aiexperiments-ai-duet
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

renaming app->src

Browse Source
This commit is contained in:
Yotam Mann 2017-01-13 16:49:19 -05:00
parent f881b2f6db
commit 2c2621985d
10 changed files with 128 additions and 682 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
static/app/Main.js
View File

@ -1,68 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Keyboard} from 'keyboard/Keyboard'
import domReady from 'domready'
import 'style/main.css'
import {AI} from 'ai/AI'
import {Sound} from 'sound/Sound'
import {Glow} from 'interface/Glow'
import {Splash} from 'interface/Splash'
domReady(() => {
const container = document.createElement('div')
container.id = 'container'
document.body.appendChild(container)
const ai = new AI()
const glow = new Glow(container)
const keyboard = new Keyboard(container)
const sound = new Sound()
const splash = new Splash(document.body)
splash.on('click', () => {
container.classList.add('focus')
keyboard.activate()
})
sound.load()
keyboard.on('keyDown', (note) => {
sound.keyDown(note)
ai.keyDown(note)
glow.user()
})
keyboard.on('keyUp', (note) => {
sound.keyUp(note)
ai.keyUp(note)
glow.user()
})
ai.on('keyDown', (note, time) => {
sound.keyDown(note, time, true)
keyboard.keyDown(note, time, true)
glow.ai(time)
})
ai.on('keyUp', (note, time) => {
sound.keyUp(note, time, true)
keyboard.keyUp(note, time, true)
glow.ai(time)
})
})

157
static/app/keyboard/Element.js
View File

@ -1,157 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import events from 'events'
import 'style/keyboard.css'
import 'pepjs'
import {Roll} from 'roll/Roll'
const offsets = [0, 0.5, 1, 1.5, 2, 3, 3.5, 4, 4.5, 5, 5.5, 6]
class KeyboardElement extends events.EventEmitter {
constructor(container, lowest=36, octaves=4){
super()
this._container = document.createElement('div')
this._container.id = 'keyboard'
container.setAttribute('touch-action', 'none')
container.addEventListener('pointerup', () => this._mousedown = false)
container.appendChild(this._container)
this._keys = {}
this._mousedown = false
this.resize(lowest, octaves)
/**
* The piano roll
* @type {Roll}
*/
this._roll = new Roll(container)
}
resize(lowest, octaves){
this._keys = {}
// clear the previous ones
this._container.innerHTML = ''
// each of the keys
const keyWidth = (1 / 7) / octaves
for (let i = lowest; i < lowest + octaves * 12; i++){
let key = document.createElement('div')
key.classList.add('key')
let isSharp = ([1, 3, 6, 8, 10].indexOf(i % 12) !== -1)
key.classList.add(isSharp ? 'black' : 'white')
this._container.appendChild(key)
// position the element
let noteOctave = Math.floor(i / 12) - Math.floor(lowest / 12)
let offset = offsets[i % 12] + noteOctave * 7
key.style.width = `${keyWidth * 100}%`
key.style.left = `${offset * keyWidth * 100}%`
key.id = i.toString()
key.setAttribute('touch-action', 'none')
const fill = document.createElement('div')
fill.id = 'fill'
key.appendChild(fill)
this._bindKeyEvents(key)
this._keys[i] = key
}
}
_bindKeyEvents(key){
key.addEventListener('pointerover', (e) => {
if (this._mousedown){
const noteNum = parseInt(e.target.id)
// this.keyDown(noteNum, false)
this.emit('keyDown', noteNum)
} else {
key.classList.add('hover')
}
})
key.addEventListener('pointerout', (e) => {
if (this._mousedown){
const noteNum = parseInt(e.target.id)
// this.keyUp(noteNum, false)
this.emit('keyUp', noteNum)
} else {
key.classList.remove('hover')
}
})
key.addEventListener('pointerdown', (e) => {
e.preventDefault()
const noteNum = parseInt(e.target.id)
// this.keyDown(noteNum, false)
this.emit('keyDown', noteNum)
this._mousedown = true
})
key.addEventListener('pointerup', (e) => {
e.preventDefault()
const noteNum = parseInt(e.target.id)
// this.keyUp(noteNum, false)
this.emit('keyUp', noteNum)
this._mousedown = false
})
}
keyDown(noteNum, ai=false){
// console.log('down', noteNum, ai)
if (this._keys.hasOwnProperty(noteNum)){
const key = this._keys[noteNum]
key.classList.remove('hover')
const highlight = document.createElement('div')
highlight.classList.add('highlight')
highlight.classList.add('active')
if (ai){
highlight.classList.add('ai')
}
key.querySelector('#fill').appendChild(highlight)
this._roll.keyDown(noteNum, this._getNotePosition(noteNum), ai)
}
}
keyUp(noteNum, ai=false){
// console.log('up', noteNum, ai)
if (this._keys.hasOwnProperty(noteNum)){
const query = ai ? '.highlight.active.ai' : '.highlight.active'
const highlight = this._keys[noteNum].querySelector(query)
if (highlight){
highlight.classList.remove('active')
setTimeout(() => highlight.remove(), 2000)
//and up on the roll
} else {
//try again
this.keyUp(noteNum)
}
}
this._roll.keyUp(noteNum, ai)
}
_getNotePosition(key){
if (this._keys.hasOwnProperty(key)){
return this._keys[key].querySelector('#fill').getBoundingClientRect()
}
}
}
export {KeyboardElement}

149
static/app/keyboard/Keyboard.js
View File

@ -1,149 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import AudioKeys from 'audiokeys'
import Tone from 'Tone/core/Tone'
import events from 'events'
import {KeyboardElement} from 'keyboard/Element'
import buckets from 'buckets-js'
import {Midi} from 'keyboard/Midi'
import Buffer from 'Tone/core/Buffer'
class Keyboard extends events.EventEmitter{
constructor(container){
super()
this._active = false
/**
* The audio key keyboard
* @type {AudioKeys}
*/
this._keyboard = new AudioKeys({polyphony : 88, rows : 1, octaveControls : false, rootNote : 48})
this._keyboard.down((e) => {
this.keyDown(e.note)
this._emitKeyDown(e.note)
})
this._keyboard.up((e) => {
this.keyUp(e.note)
this._emitKeyUp(e.note)
})
/**
* The piano interface
*/
this._keyboardInterface = new KeyboardElement(container, 36, 2)
this._keyboardInterface.on('keyDown', (note) => {
this.keyDown(note)
this._emitKeyDown(note)
})
this._keyboardInterface.on('keyUp', (note) => {
this.keyUp(note)
this._emitKeyUp(note)
})
window.addEventListener('resize', this._resize.bind(this))
//size initially
this._resize()
//make sure they don't get double clicked
this._currentKeys = {}
//a queue of all of the events
this._eventQueue = new buckets.PriorityQueue((a, b) => b.time - a.time)
this._boundLoop = this._loop.bind(this)
this._loop()
const bottom = document.createElement('div')
bottom.id = 'bottom'
container.appendChild(bottom)
//the midi input
this._midi = new Midi()
this._midi.on('keyDown', (note) => {
this.keyDown(note)
this._emitKeyDown(note)
})
this._midi.on('keyUp', (note) => {
this.keyUp(note)
this._emitKeyUp(note)
})
}
_loop(){
requestAnimationFrame(this._boundLoop)
const now = Tone.now()
while(!this._eventQueue.isEmpty() && this._eventQueue.peek().time <= now){
const event = this._eventQueue.dequeue()
event.callback()
}
}
_emitKeyDown(note){
if (this._active){
this.emit('keyDown', note)
}
}
_emitKeyUp(note){
if (this._active){
this.emit('keyUp', note)
}
}
keyDown(note, time=Tone.now(), ai=false){
if (!this._active){
return
}
if (!this._currentKeys[note]){
this._currentKeys[note] = 0
}
this._currentKeys[note] += 1
this._eventQueue.add({
time : time,
callback : this._keyboardInterface.keyDown.bind(this._keyboardInterface, note, ai)
})
}
keyUp(note, time=Tone.now(), ai=false){
if (!this._active){
return
}
if (this._currentKeys[note]){
this._currentKeys[note] -= 1
this._currentKeys[note] = Math.max(this._currentKeys[note], 0)
this._eventQueue.add({
time : time,
callback : this._keyboardInterface.keyUp.bind(this._keyboardInterface, note, ai)
})
}
}
_resize(){
const keyWidth = 24
let octaves = Math.round((window.innerWidth / keyWidth) / 12)
octaves = Math.max(octaves, 2)
this._keyboardInterface.resize(36, octaves)
}
activate(){
this._active = true
}
}
export {Keyboard}

59
static/app/keyboard/Midi.js
View File

@ -1,59 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import events from 'events'
import WebMidi from 'webmidi'
class Midi extends events.EventEmitter{
constructor(){
super()
this._isEnabled = false
WebMidi.enable((err) => {
if (!err){
this._isEnabled = true
if (WebMidi.inputs){
WebMidi.inputs.forEach((input) => this._bindInput(input))
}
WebMidi.addListener('connected', (device) => {
if (device.input){
this._bindInput(device.input)
}
})
}
})
}
_bindInput(inputDevice){
if (this._isEnabled){
WebMidi.addListener('disconnected', (device) => {
if (device.input){
device.input.removeListener('noteOn')
device.input.removeListener('noteOff')
}
})
inputDevice.addListener('noteon', 'all', (event) => {
this.emit('keyDown', event.note.number)
})
inputDevice.addListener('noteoff', 'all', (event) => {
this.emit('keyUp', event.note.number)
})
}
}
}
export {Midi}

123
static/app/roll/Roll.js
View File

@ -1,123 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const THREE = require('three')
var geometry = new THREE.PlaneGeometry( 1, 1, 1 )
var material = new THREE.MeshBasicMaterial( {color: 0x1FB7EC, side: THREE.DoubleSide} )
var aiMaterial = new THREE.MeshBasicMaterial( {color: 0xFFB729, side: THREE.DoubleSide} )
window.zero = new THREE.Vector3(0, 0, 0)
function scale(value, inMin, inMax, min, max){
return ((value - inMin) / (inMax - inMin)) * (max - min) + min
}
class Roll {
constructor(container){
this._element = document.createElement('div')
this._element.id = 'roll'
container.appendChild(this._element)
this._camera = new THREE.OrthographicCamera(0, 1, 1, 0, 1, 1000 )
this._camera.position.z = 1
this._camera.lookAt(new THREE.Vector3(0, 0, 0))
this._scene = new THREE.Scene()
this._renderer = new THREE.WebGLRenderer({alpha: true})
this._renderer.setClearColor(0x000000, 0)
this._renderer.setPixelRatio( window.devicePixelRatio )
this._renderer.sortObjects = false
this._element.appendChild(this._renderer.domElement)
this._currentNotes = {}
window.camera = this._camera
//set the size initially
this._resize()
//start the loop
this._boundLoop = this._loop.bind(this)
this._boundLoop()
window.addEventListener('resize', this._resize.bind(this))
}
keyDown(midi, box, ai=false){
const selector = ai ? `ai${midi}` : midi
if (!this._currentNotes.hasOwnProperty(selector)){
this._currentNotes[selector] = []
}
if (midi && box){
//translate the box coords to this space
const initialScaling = 10000
const plane = new THREE.Mesh( geometry, ai ? aiMaterial : material )
const margin = 4
const width = box.width - margin * 2
plane.scale.set(width, initialScaling, 1)
plane.position.z = 0
plane.position.x = box.left + margin + width / 2
plane.position.y = this._element.clientHeight + this._camera.position.y + initialScaling / 2
this._scene.add(plane)
this._currentNotes[selector].push({
plane : plane,
position: this._camera.position.y
})
}
}
keyUp(midi, ai=false){
const selector = ai ? `ai${midi}` : midi
if (this._currentNotes[selector] && this._currentNotes[selector].length){
const note = this._currentNotes[selector].shift()
const plane = note.plane
const position = note.position
// get the distance covered
plane.scale.y = Math.max(this._camera.position.y - position, 5)
plane.position.y = this._element.clientHeight + position + plane.scale.y / 2
} else {
// console.log(midi)
// setTimeout(() => this.keyUp(midi, ai), 100)
}
}
_resize(){
var frustumSize = 1000
var aspect = this._element.clientWidth / this._element.clientHeight
//make it match the screen pixesl
this._camera.left = 0
this._camera.bottom = this._element.clientHeight
this._camera.right = this._element.clientWidth
this._camera.top = 0
//update things
this._camera.updateProjectionMatrix()
this._renderer.setSize( this._element.clientWidth, this._element.clientHeight )
}
_loop(){
requestAnimationFrame(this._boundLoop)
this._renderer.render( this._scene, this._camera )
this._camera.position.y += 2
}
}
export {Roll}

101
static/app/sound/Sound.js
View File

@ -1,101 +0,0 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import Piano from 'Piano/src/Piano'
import Tone from 'Tone/core/Tone'
import PolySynth from 'Tone/instrument/PolySynth'
import Frequency from 'Tone/type/Frequency'
import MonoSynth from 'Tone/instrument/MonoSynth'
class Sound {
constructor(){
this._range = [36, 108]
/**
* The piano audio
* @type {Piano}
*/
this._piano = new Piano(this._range, 1, false).toMaster().setVolume('release', -Infinity)
/**
* The piano audio
* @type {Piano}
*/
this._aipiano = new Piano(this._range, 1, false).toMaster().setVolume('release', -Infinity)
this._synth = new PolySynth(8, MonoSynth).toMaster()
this._synth.set({
oscillator : {
type : 'pwm',
modulationFrequency : 3
},
envelope : {
attackCurve : 'linear',
attack : 0.05,
decay : 0.3,
sustain : 0.8,
release : 3,
},
filter : {
type : 'lowpass'
},
filterEnvelope : {
baseFrequency : 800,
octaves : 1,
attack : 0.3,
decay : 0.1,
sustain : 1,
release : 3,
}
})
this._synth.volume.value = -36
window.synth = this._synth
}
load(){
const salamanderPath = 'audio/Salamander/'
return Promise.all([this._piano.load(salamanderPath), this._aipiano.load(salamanderPath)])
}
keyDown(note, time=Tone.now(), ai=false){
if (note >= this._range[0] && note <= this._range[1]){
if (ai){
this._aipiano.keyDown(note, 1, time)
this._synth.triggerAttack(Frequency(note, 'midi').toNote(), time)
} else {
this._piano.keyDown(note, 1, time)
}
}
}
keyUp(note, time=Tone.now(), ai=false){
if (note >= this._range[0] && note <= this._range[1]){
if (ai){
this._aipiano.keyUp(note, time)
this._synth.triggerRelease(Frequency(note, 'midi').toNote(), time)
} else {
this._piano.keyUp(note, time)
}
}
}
}
export {Sound}

111
static/src/Main.js Normal file
View File

@ -0,0 +1,111 @@
/**
* Copyright 2016 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Keyboard} from 'keyboard/Keyboard'
import {AI} from 'ai/AI'
import {Sound} from 'sound/Sound'
import {Glow} from 'interface/Glow'
import {Splash} from 'interface/Splash'
import {About} from 'interface/About'
import {Tutorial} from 'ai/Tutorial'
import 'babel-polyfill'
/////////////// SPLASH ///////////////////
const about = new About(document.body)
const splash = new Splash(document.body)
splash.on('click', () => {
keyboard.activate()
tutorial.start()
})
about.on('close', () => {
if (!splash.loaded || splash.isOpen()){
splash.show()
} else {
keyboard.activate()
}
})
about.on('open', () => {
keyboard.deactivate()
if (splash.isOpen()){
splash.hide()
}
})
/////////////// PIANO ///////////////////
const container = document.createElement('div')
container.id = 'container'
document.body.appendChild(container)
const glow = new Glow(container)
const keyboard = new Keyboard(container)
const sound = new Sound()
sound.load()
keyboard.on('keyDown', (note) => {
sound.keyDown(note)
ai.keyDown(note)
glow.user()
})
keyboard.on('keyUp', (note) => {
sound.keyUp(note)
ai.keyUp(note)
glow.user()
})
/////////////// AI ///////////////////
const ai = new AI()
ai.on('keyDown', (note, time) => {
sound.keyDown(note, time, true)
keyboard.keyDown(note, time, true)
glow.ai(time)
})
ai.on('keyUp', (note, time) => {
sound.keyUp(note, time, true)
keyboard.keyUp(note, time, true)
glow.ai(time)
})
/////////////// TUTORIAL ///////////////////
const tutorial = new Tutorial(container)
tutorial.on('keyDown', (note, time) => {
sound.keyDown(note, time)
keyboard.keyDown(note, time)
glow.user()
})
tutorial.on('keyUp', (note, time) => {
sound.keyUp(note, time)
keyboard.keyUp(note, time)
glow.user()
})
tutorial.on('aiKeyDown', (note, time) => {
ai.keyDown(note, time)
})
tutorial.on('aiKeyUp', (note, time) => {
ai.keyUp(note, time)
})

38
static/app/ai/AI.js → static/src/ai/AI.js
View File

@ -14,13 +14,10 @@
* limitations under the License.
*/
import {Midi} from 'MidiConvert/src/Midi'
import Tone from 'Tone/core/Tone'
import MidiConvert from 'MidiConvert/src/MidiConvert'
import MidiConvert from 'midiconvert'
import events from 'events'
window.generator = 'pop'
class AI extends events.EventEmitter{
constructor(){
super()
@ -33,22 +30,11 @@ class AI extends events.EventEmitter{
this._lastPhrase = -1
/*setInterval(() => {
//wait a max of 10 seconds before sending an event
if (Date.now() - this._phraseStart > 5000){
for (let note in this._heldNotes){
this._track.noteOff(note, Tone.now())
delete this._heldNotes[note]
}
this.send()
}
}, 200)*/
this._aiEndTime = 0
}
_newTrack(){
this._midi = new Midi()
this._midi = new MidiConvert.create()
this._track = this._midi.track()
}
@ -66,40 +52,42 @@ class AI extends events.EventEmitter{
let additional = endTime
additional = Math.min(additional, 8)
additional = Math.max(additional, 1)
request.load(`/predict?duration=${endTime + additional}&generator=${generator}`, JSON.stringify(request.toArray()), 'POST').then((response) => {
request.load(`/predict?duration=${endTime + additional}`, JSON.stringify(request.toArray()), 'POST').then((response) => {
response.slice(endTime / 2).tracks[1].notes.forEach((note) => {
const now = Tone.now()
const now = Tone.now() + 0.05
if (note.noteOn + now > this._aiEndTime){
this._aiEndTime = note.noteOn + now
this.emit('keyDown', note.midi, note.noteOn + now)
note.duration = note.duration * 0.9
note.duration = Math.min(note.duration, 4)
this.emit('keyUp', note.midi, note.noteOff + now)
}
})
})
this._lastPhrase = -1
this.emit('sent')
}
}
keyDown(note){
keyDown(note, time=Tone.now()){
if (this._track.length === 0 && this._lastPhrase === -1){
this._lastPhrase = Date.now()
}
this._track.noteOn(note, Tone.now())
this._track.noteOn(note, time)
clearTimeout(this._sendTimeout)
this._heldNotes[note] = true
}
keyUp(note){
this._track.noteOff(note, Tone.now())
keyUp(note, time=Tone.now()){
this._track.noteOff(note, time)
delete this._heldNotes[note]
// send something if there are no events for a moment
if (Object.keys(this._heldNotes).length === 0){
if (this._lastPhrase !== -1 && Date.now() - this._lastPhrase > 5000){
//do it immediately
if (this._lastPhrase !== -1 && Date.now() - this._lastPhrase > 3000){
//just send it
this.send()
} else {
this._sendTimeout = setTimeout(this.send.bind(this), 600)
this._sendTimeout = setTimeout(this.send.bind(this), 600 + (time - Tone.now()) * 1000)
}
}
}

0
static/app/interface/Glow.js → static/src/interface/Glow.js
View File

4
static/app/interface/Loader.js → static/src/interface/Loader.js
View File

@ -44,8 +44,12 @@ export default class Loader extends EventEmitter{
StartAudioContext(Tone.context, loader)
this.loaded = false
Buffer.on('load', () => {
this.loaded = true
fillText.innerHTML = '<div id="piano"></div> <div id="play">PLAY</div>'
loader.classList.add('clickable')