From 09bb68f7ad28ff116543b6be0e1032bf172f28d5 Mon Sep 17 00:00:00 2001 From: Cameron Taylor Date: Thu, 7 Oct 2021 21:01:46 -0400 Subject: [PATCH] a bot FFT stuf fin progress MESSY LOL --- source/ABotVis.hx | 230 ++++++++++++++++++++++++++++++++++++- source/ChartingState.hx | 3 + source/SpectogramSprite.hx | 37 +++--- source/VisShit.hx | 3 +- 4 files changed, 249 insertions(+), 24 deletions(-) diff --git a/source/ABotVis.hx b/source/ABotVis.hx index 52f0d3aa6..279c5918d 100644 --- a/source/ABotVis.hx +++ b/source/ABotVis.hx @@ -1,10 +1,238 @@ +import dsp.FFT; import flixel.FlxSprite; +import flixel.addons.plugin.taskManager.FlxTask; +import flixel.graphics.frames.FlxAtlasFrames; import flixel.group.FlxSpriteGroup.FlxTypedSpriteGroup; +import flixel.math.FlxMath; +import flixel.system.FlxSound; +import ui.PreferencesMenu.CheckboxThingie; + +using Lambda; class ABotVis extends FlxTypedSpriteGroup { - public function new() + public var vis:VisShit; + + var volumes:Array = []; + + public function new(snd:FlxSound) { super(); + + vis = new VisShit(snd); + // vis.snd = snd; + + var visFrms:FlxAtlasFrames = Paths.getSparrowAtlas('aBotViz'); + + for (lol in 1...8) + { + // pushes initial value + volumes.push(0.0); + + var viz:FlxSprite = new FlxSprite(50 * lol, 0); + viz.frames = visFrms; + add(viz); + + var visStr = 'VIZ'; + if (lol == 5) + visStr = 'viz'; // lol makes it lowercase, accomodates for art that I dont wanna rename! + + viz.animation.addByPrefix('VIZ', visStr + lol, 0); + viz.animation.play('VIZ', false, false, -1); + } + } + + override function update(elapsed:Float) + { + // updateViz(); + + updateFFT(); + + super.update(elapsed); + } + + function updateFFT() + { + if (vis.snd != null) + { + vis.checkAndSetBuffer(); + + if (vis.setBuffer) + { + var remappedShit:Int = 0; + + if (vis.snd.playing) + remappedShit = Std.int(FlxMath.remapToRange(vis.snd.time, 0, vis.snd.length, 0, vis.numSamples)); + else + remappedShit = Std.int(FlxMath.remapToRange(Conductor.songPosition, 0, vis.snd.length, 0, vis.numSamples)); + + var fftSamples:Array = []; + + var swagBucks = remappedShit; + + for (i in remappedShit...remappedShit + (Std.int((44100 * (1 / 144))))) + { + var left = vis.audioData[swagBucks] / 32767; + var right = vis.audioData[swagBucks + 1] / 32767; + + var balanced = (left + right) / 2; + + swagBucks += 2; + + fftSamples.push(balanced); + } + + var freqShit = funnyFFT(fftSamples); + + for (i in 0...group.members.length) + { + var sliceLength:Int = Std.int(freqShit[0].length / group.members.length); + + var volSlice = freqShit[0].slice(Std.int(sliceLength * i), Std.int(sliceLength * i) + sliceLength); + + var avgVel:Float = 0; + + for (slice in volSlice) + { + avgVel += slice; + } + + avgVel /= volSlice.length; + + avgVel *= 10000000; + + volumes[i] += avgVel - (FlxG.elapsed * (volumes[i] * 50)); + + var animFrame:Int = Std.int(volumes[i]); + + animFrame = Math.floor(Math.min(5, animFrame)); + animFrame = Math.floor(Math.max(0, animFrame)); + + animFrame = Std.int(Math.abs(animFrame - 5)); // shitty dumbass flip, cuz dave got da shit backwards lol! + + group.members[i].animation.curAnim.curFrame = animFrame; + if (FlxG.keys.justPressed.U) + { + trace(avgVel); + trace(group.members[i].animation.curAnim.curFrame); + } + } + + // group.members[0].animation.curAnim.curFrame = + } + } + } + + public function updateViz() + { + if (vis.snd != null) + { + var remappedShit:Int = 0; + vis.checkAndSetBuffer(); + + if (vis.setBuffer) + { + // var startingSample:Int = Std.int(FlxMath.remapToRange) + + if (vis.snd.playing) + remappedShit = Std.int(FlxMath.remapToRange(vis.snd.time, 0, vis.snd.length, 0, vis.numSamples)); + + for (i in 0...group.members.length) + { + var sampleApprox:Int = Std.int(FlxMath.remapToRange(i, 0, group.members.length, remappedShit, remappedShit + 500)); + + var left = vis.audioData[sampleApprox] / 32767; + + var animFrame:Int = Std.int(FlxMath.remapToRange(left, -1, 1, 0, 6)); + + group.members[i].animation.curAnim.curFrame = animFrame; + } + } + } + } + + function funnyFFT(samples:Array, ?skipped:Int = 1):Array> + { + // nab multiple samples at once in while / for loops? + + var fs:Float = 44100 / skipped; // sample rate shit? + + final fftN = 1024; + final halfN = Std.int(fftN / 2); + final overlap = 0.5; + final hop = Std.int(fftN * (1 - overlap)); + + // window function to compensate for overlapping + final a0 = 0.5; // => Hann(ing) window + final window = (n:Int) -> a0 - (1 - a0) * Math.cos(2 * Math.PI * n / fftN); + + // NOTE TO SELF FOR WHEN I WAKE UP + + // helpers, note that spectrum indexes suppose non-negative frequencies + final binSize = fs / fftN; + final indexToFreq = function(k:Int) + { + var powShit:Float = FlxMath.remapToRange(k, 0, halfN, 0, 4.3); // 4.3 is almost 20khz + + return 1.0 * (Math.pow(10, powShit)); // we need the `1.0` to avoid overflows + }; + + // "melodic" band-pass filter + final minFreq = 20.70; + final maxFreq = 4000.01; + final melodicBandPass = function(k:Int, s:Float) + { + // final freq = indexToFreq(k); + // final filter = freq > minFreq - binSize && freq < maxFreq + binSize ? 1 : 0; + return s; + }; + + var freqOutput:Array> = []; + + var c = 0; // index where each chunk begins + var indexOfArray:Int = 0; + while (c < samples.length) + { + // take a chunk (zero-padded if needed) and apply the window + final chunk = [ + for (n in 0...fftN) + (c + n < samples.length ? samples[c + n] : 0.0) * window(n) + ]; + + // compute positive spectrum with sampling correction and BP filter + final freqs = FFT.rfft(chunk).map(z -> z.scale(1 / fftN).magnitude).mapi(melodicBandPass); + + freqOutput.push([]); + + // if (FlxG.keys.justPressed.M) + // trace(FFT.rfft(chunk).map(z -> z.scale(1 / fs).magnitude)); + + // find spectral peaks and their instantaneous frequencies + for (k => s in freqs) + { + final time = c / fs; + final freq = indexToFreq(k); + final power = s * s; + if (FlxG.keys.justPressed.I) + { + trace(k); + + haxe.Log.trace('${time};${freq};${power}', null); + } + if (freq < maxFreq) + freqOutput[indexOfArray].push(power); + // + } + // haxe.Log.trace("", null); + + indexOfArray++; + // move to next (overlapping) chunk + c += hop; + } + + if (FlxG.keys.justPressed.C) + trace(freqOutput.length); + + return freqOutput; } } diff --git a/source/ChartingState.hx b/source/ChartingState.hx index 5874d769c..77b205bb4 100644 --- a/source/ChartingState.hx +++ b/source/ChartingState.hx @@ -417,6 +417,9 @@ class ChartingState extends MusicBeatState staticSpecGrp = new FlxTypedGroup(); add(staticSpecGrp); + var aBoy:ABotVis = new ABotVis(FlxG.sound.music); + add(aBoy); + for (index => voc in vocals.members) { var vocalSpec:SpectogramSprite = new SpectogramSprite(voc, FlxG.random.color(0xFFAAAAAA, FlxColor.WHITE, 100)); diff --git a/source/SpectogramSprite.hx b/source/SpectogramSprite.hx index 4f041a98f..0f8921528 100644 --- a/source/SpectogramSprite.hx +++ b/source/SpectogramSprite.hx @@ -141,7 +141,7 @@ class SpectogramSprite extends FlxTypedSpriteGroup { frameCounter++; - if (frameCounter >= 3) + if (frameCounter >= 0) { frameCounter = 0; doAnim = true; @@ -180,20 +180,20 @@ class SpectogramSprite extends FlxTypedSpriteGroup // trace(audioData.subarray(remappedShit, remappedShit + lengthOfShit).buffer); } - for (sample in remappedShit...remappedShit + (Std.int((44100 * (10 / 60)) / 4))) + for (sample in remappedShit...remappedShit + (Std.int((44100 * (1 / 144))))) { var left = audioData[i] / 32767; var right = audioData[i + 1] / 32767; var balanced = (left + right) / 2; - i += 8; + i += 2; // var remappedSample:Float = FlxMath.remapToRange(sample, remappedShit, remappedShit + lengthOfShit, 0, lengthOfShit - 1); fftSamples.push(balanced); } - var freqShit = funnyFFT(fftSamples, 4); + var freqShit = funnyFFT(fftSamples); for (i in 0...group.members.length) { @@ -208,7 +208,7 @@ class SpectogramSprite extends FlxTypedSpriteGroup var hzPicker:Float = Math.pow(10, powedShit); // var sampleApprox:Int = Std.int(FlxMath.remapToRange(i, 0, group.members.length, startingSample, startingSample + samplesToGen)); - var remappedFreq:Int = Std.int(FlxMath.remapToRange(hzPicker + 50, 0, 10000, 0, freqShit[0].length - 1)); + var remappedFreq:Int = Std.int(FlxMath.remapToRange(hzPicker, 0, 10000, 0, freqShit[0].length - 1)); group.members[i].x = prevLine.x; group.members[i].y = prevLine.y; @@ -219,7 +219,6 @@ class SpectogramSprite extends FlxTypedSpriteGroup freqPower += freqShit[pow][remappedFreq]; freqPower /= freqShit.length; - // freqShit[remappedFreq] var freqIDK:Float = FlxMath.remapToRange(freqPower, 0, 0.000005, 0, 50); prevLine.x = (freqIDK * swagheight / 2 + swagheight / 2) + x; @@ -227,22 +226,10 @@ class SpectogramSprite extends FlxTypedSpriteGroup var line = FlxVector.get(prevLine.x - group.members[i].x, prevLine.y - group.members[i].y); + // dont draw a line until i figure out a nicer way to view da spikes and shit idk lol! // group.members[i].setGraphicSize(Std.int(Math.max(line.length, 1)), Std.int(1)); // group.members[i].angle = line.degrees; } - /* - for (freq in 0...freqShit.length) - { - var remappedFreq:Float = FlxMath.remapToRange(freq, 0, freqShit.length, 0, lengthOfShit - 1); - - group.members[Std.int(remappedFreq)].x = prevLine.x; - group.members[Std.int(remappedFreq)].y = prevLine.y; - - var freqShit:Float = FlxMath.remapToRange(freqShit[freq], 0, 0.002, 0, 20); - - prevLine.x = (freqShit * swagheight / 2 + swagheight / 2) + x; - prevLine.y = (Math.ceil(remappedFreq) / lengthOfShit * daHeight) + y; - }*/ } } } @@ -301,7 +288,7 @@ class SpectogramSprite extends FlxTypedSpriteGroup var fs:Float = 44100 / skipped; // sample rate shit? - final fftN = 2048; + final fftN = 1024; final halfN = Std.int(fftN / 2); final overlap = 0.5; final hop = Std.int(fftN * (1 - overlap)); @@ -312,7 +299,12 @@ class SpectogramSprite extends FlxTypedSpriteGroup // helpers, note that spectrum indexes suppose non-negative frequencies final binSize = fs / fftN; - final indexToFreq = (k:Int) -> 1.0 * k * binSize; // we need the `1.0` to avoid overflows + final indexToFreq = function(k:Int) + { + var powShit:Float = FlxMath.remapToRange(k, 0, halfN, 0, 4.3); // 4.3 is almost 20khz + + return 1.0 * (Math.pow(10, powShit)); // we need the `1.0` to avoid overflows + }; // "melodic" band-pass filter final minFreq = 20.70; @@ -337,7 +329,7 @@ class SpectogramSprite extends FlxTypedSpriteGroup ]; // compute positive spectrum with sampling correction and BP filter - final freqs = FFT.rfft(chunk).map(z -> z.scale(1 / fs).magnitude).mapi(melodicBandPass); + final freqs = FFT.rfft(chunk).map(z -> z.scale(1 / fftN).magnitude).mapi(melodicBandPass); freqOutput.push([]); @@ -349,6 +341,7 @@ class SpectogramSprite extends FlxTypedSpriteGroup final power = s * s; if (FlxG.keys.justPressed.N) { + trace(k); haxe.Log.trace('${time};${freq};${power}', null); } if (freq < maxFreq) diff --git a/source/VisShit.hx b/source/VisShit.hx index cf453e9c6..b0d4ef3aa 100644 --- a/source/VisShit.hx +++ b/source/VisShit.hx @@ -9,6 +9,7 @@ class VisShit public var setBuffer:Bool = false; public var audioData:Int16Array; public var sampleRate:Int = 44100; // default, ez? + public var numSamples:Int = 0; public function new(snd:FlxSound) { @@ -34,7 +35,7 @@ class VisShit trace(buf.bitsPerSample); setBuffer = true; - // numSamples = Std.int(audioData.length / 2); + numSamples = Std.int(audioData.length / 2); } } }