From c3d25822522cb6b3e0cd37f4b7672b8fc9256954 Mon Sep 17 00:00:00 2001 From: EliteMasterEric Date: Sat, 20 Jan 2024 14:07:48 -0500 Subject: [PATCH] Work in progress on new Waveform renderer --- .vscode/settings.json | 5 + assets | 2 +- source/funkin/audio/waveform/WaveformData.hx | 243 ++++++++++++++++++ .../audio/waveform/WaveformDataParser.hx | 32 +++ source/funkin/ui/debug/WaveformTestState.hx | 202 +++++++++++++++ 5 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 source/funkin/audio/waveform/WaveformData.hx create mode 100644 source/funkin/audio/waveform/WaveformDataParser.hx create mode 100644 source/funkin/ui/debug/WaveformTestState.hx diff --git a/.vscode/settings.json b/.vscode/settings.json index cefbadcf6..8979e4de6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -125,6 +125,11 @@ "target": "windows", "args": ["-debug", "-DLATENCY"] }, + { + "label": "Windows / Debug (Waveform Test)", + "target": "windows", + "args": ["-debug", "-DWAVEFORM"] + }, { "label": "HTML5 / Debug", "target": "html5", diff --git a/assets b/assets index 7e19c4cfa..3d92b4976 160000 --- a/assets +++ b/assets @@ -1 +1 @@ -Subproject commit 7e19c4cfa7db57178f03ed4a58a9fd4d2b93dea7 +Subproject commit 3d92b497682727d34eaa55e564e0bd9faea1c9d7 diff --git a/source/funkin/audio/waveform/WaveformData.hx b/source/funkin/audio/waveform/WaveformData.hx new file mode 100644 index 000000000..525d1bd5f --- /dev/null +++ b/source/funkin/audio/waveform/WaveformData.hx @@ -0,0 +1,243 @@ +package funkin.audio.waveform; + +import funkin.util.MathUtil; + +@:nullSafety +class WaveformData +{ + /** + * The version of the waveform data format. + * @default `2` (-1 if not specified/invalid) + */ + public var version(default, null):Int = -1; + + /** + * The number of channels in the waveform. + */ + public var channels(default, null):Int = 1; + + @:alias('sample_rate') + public var sampleRate(default, null):Int = 44100; + + /** + * Number of input audio samples per output waveform data point. + * At base zoom level this is number of samples per pixel. + * Lower values can more accurately represent the waveform when zoomed in, but take more data. + */ + @:alias('samples_per_pixel') + public var samplesPerPixel(default, null):Int = 256; + + /** + * Number of bits to use for each sample value. Valid values are `8` and `16`. + */ + public var bits(default, null):Int = 16; + + /** + * Number of output waveform data points. + */ + public var length(default, null):Int = 0; // Array size is (4 * length) + + /** + * Array of Int16 values representing the waveform. + * TODO: Use an `openfl.Vector` for performance. + */ + public var data(default, null):Array = []; + + @:jignored + var channelData:Null> = null; + + public function new() {} + + function buildChannelData():Array + { + channelData = []; + for (i in 0...channels) + { + channelData.push(new WaveformDataChannel(this, i)); + } + return channelData; + } + + public function channel(index:Int) + { + return (channelData == null) ? buildChannelData()[index] : channelData[index]; + } + + public function get(index:Int):Int + { + return data[index] ?? 0; + } + + public function set(index:Int, value:Int) + { + data[index] = value; + } + + /** + * Maximum possible value for a waveform data point. + * The minimum possible value is (-1 * maxSampleValue) + */ + public function maxSampleValue():Int + { + if (_maxSampleValue != -1) return _maxSampleValue; + return _maxSampleValue = Std.int(Math.pow(2, bits)); + } + + /** + * Cache the value because `Math.pow` is expensive and the value gets used a lot. + */ + @:jignored + var _maxSampleValue:Int = -1; + + /** + * @return The length of the waveform in samples. + */ + public function lenSamples():Int + { + return length * samplesPerPixel; + } + + /** + * @return The length of the waveform in seconds. + */ + public function lenSeconds():Float + { + return lenSamples() / sampleRate; + } + + /** + * Given the time in seconds, return the waveform data point index. + */ + public function secondsToIndex(seconds:Float):Int + { + return Std.int(seconds * sampleRate / samplesPerPixel); + } + + /** + * Given a waveform data point index, return the time in seconds. + */ + public function indexToSeconds(index:Int):Float + { + return index * samplesPerPixel / sampleRate; + } + + /** + * Given the percentage progress through the waveform, return the waveform data point index. + */ + public function percentToIndex(percent:Float):Int + { + return Std.int(percent * length); + } + + /** + * Given a waveform data point index, return the percentage progress through the waveform. + */ + public function indexToPercent(index:Int):Float + { + return index / length; + } +} + +class WaveformDataChannel +{ + var parent:WaveformData; + var channelId:Int; + + public function new(parent:WaveformData, channelId:Int) + { + this.parent = parent; + this.channelId = channelId; + } + + public function minSample(i:Int) + { + var offset = (i * parent.channels + this.channelId) * 2; + return parent.get(offset); + } + + /** + * Mapped to a value between 0 and 1. + */ + public function minSampleMapped(i:Int) + { + return minSample(i) / parent.maxSampleValue(); + } + + /** + * Minimum value within the range of samples. + * @param i + */ + public function minSampleRange(start:Int, end:Int) + { + var min = parent.maxSampleValue(); + for (i in start...end) + { + var sample = minSample(i); + if (sample < min) min = sample; + } + return min; + } + + /** + * Maximum value within the range of samples, mapped to a value between 0 and 1. + * @param i + */ + public function minSampleRangeMapped(start:Int, end:Int) + { + return minSampleRange(start, end) / parent.maxSampleValue(); + } + + public function maxSample(i:Int) + { + var offset = (i * parent.channels + this.channelId) * 2 + 1; + return parent.get(offset); + } + + /** + * Mapped to a value between 0 and 1. + */ + public function maxSampleMapped(i:Int) + { + return maxSample(i) / parent.maxSampleValue(); + } + + /** + * Maximum value within the range of samples. + * @param i + */ + public function maxSampleRange(start:Int, end:Int) + { + var max = -parent.maxSampleValue(); + for (i in start...end) + { + var sample = maxSample(i); + if (sample > max) max = sample; + } + return max; + } + + /** + * Maximum value within the range of samples, mapped to a value between 0 and 1. + * @param i + */ + public function maxSampleRangeMapped(start:Int, end:Int) + { + return maxSampleRange(start, end) / parent.maxSampleValue(); + } + + /** + * Maximum value within the range of samples, mapped to a value between 0 and 1. + * @param i + */ + public function setMinSample(i:Int, value:Int) + { + var offset = (i * parent.channels + this.channelId) * 2; + parent.set(offset, value); + } + + public function setMaxSample(i:Int, value:Int) + { + var offset = (i * parent.channels + this.channelId) * 2 + 1; + parent.set(offset, value); + } +} diff --git a/source/funkin/audio/waveform/WaveformDataParser.hx b/source/funkin/audio/waveform/WaveformDataParser.hx new file mode 100644 index 000000000..fb453cf1b --- /dev/null +++ b/source/funkin/audio/waveform/WaveformDataParser.hx @@ -0,0 +1,32 @@ +package funkin.audio.waveform; + +class WaveformDataParser +{ + public static function parseWaveformData(path:String):Null + { + var rawJson:String = openfl.Assets.getText(path).trim(); + return parseWaveformDataString(rawJson, path); + } + + public static function parseWaveformDataString(contents:String, ?fileName:String):Null + { + var parser = new json2object.JsonParser(); + parser.ignoreUnknownVariables = false; + parser.fromJson(contents, fileName); + + if (parser.errors.length > 0) + { + printErrors(parser.errors, fileName); + return null; + } + return parser.value; + } + + static function printErrors(errors:Array, id:String = ''):Void + { + trace('[WAVEFORM] Failed to parse waveform data: ${id}'); + + for (error in errors) + funkin.data.DataError.printError(error); + } +} diff --git a/source/funkin/ui/debug/WaveformTestState.hx b/source/funkin/ui/debug/WaveformTestState.hx new file mode 100644 index 000000000..b202116b9 --- /dev/null +++ b/source/funkin/ui/debug/WaveformTestState.hx @@ -0,0 +1,202 @@ +package funkin.ui.debug; + +import flixel.FlxSprite; +import flixel.util.FlxColor; +import funkin.audio.FunkinSound; +import funkin.audio.waveform.WaveformData; +import funkin.audio.waveform.WaveformDataParser; +import funkin.graphics.rendering.MeshRender; + +class WaveformTestState extends MusicBeatState +{ + public function new() + { + super(); + } + + var waveformData:WaveformData; + + var waveformAudio:FunkinSound; + + var meshRender:MeshRender; + + var timeMarker:FlxSprite; + + public override function create():Void + { + super.create(); + + waveformData = WaveformDataParser.parseWaveformData(Paths.json("waveform/dadbattle-erect/dadbattle-erect.waveform")); + + waveformAudio = FunkinSound.load(Paths.music('dadbattle-erect/dadbattle-erect')); + + var lightBlue:FlxColor = FlxColor.fromString("#ADD8E6"); + meshRender = new MeshRender(0, 0, lightBlue); + add(meshRender); + + timeMarker = new FlxSprite(0, FlxG.height * 1 / 6); + timeMarker.makeGraphic(1, Std.int(FlxG.height * 2 / 3), FlxColor.RED); + add(timeMarker); + + drawWaveform(time, duration); + } + + /** + * @param offsetX Horizontal offset to draw the waveform at, in samples. + */ + function drawWaveform(timeSeconds:Float, duration:Float):Void + { + meshRender.clear(); + + var offsetX:Int = waveformData.secondsToIndex(timeSeconds); + + var waveformHeight:Int = Std.int(FlxG.height * (2 / 3)); + var waveformWidth:Int = FlxG.width; + var waveformCenterPos:Int = Std.int(FlxG.height / 2); + + var oneSecondInIndices:Int = waveformData.secondsToIndex(1); + + var startTime:Float = -1.0; + var endTime:Float = startTime + duration; + + var startIndex:Int = Std.int(offsetX + (oneSecondInIndices * startTime)); + var endIndex:Int = Std.int(offsetX + (oneSecondInIndices * (startTime + duration))); + + var pixelsPerIndex:Float = waveformWidth / (endIndex - startIndex); + var indexesPerPixel:Float = (endIndex - startIndex) / waveformWidth; + + if (pixelsPerIndex >= 1.0) + { + // Each index is at least one pixel wide, so we render each index. + var prevVertexTopIndex:Int = -1; + var prevVertexBottomIndex:Int = -1; + for (i in startIndex...endIndex) + { + var pixelPos:Int = Std.int((i - startIndex) * pixelsPerIndex); + + var vertexTopY:Int = Std.int(waveformCenterPos - (waveformData.channel(0).maxSampleMapped(i) * waveformHeight / 2)); + var vertexBottomY:Int = Std.int(waveformCenterPos + (-waveformData.channel(0).minSampleMapped(i) * waveformHeight / 2)); + + var vertexTopIndex:Int = meshRender.build_vertex(pixelPos, vertexTopY); + var vertexBottomIndex:Int = meshRender.build_vertex(pixelPos, vertexBottomY); + + if (prevVertexTopIndex != -1 && prevVertexBottomIndex != -1) + { + meshRender.add_quad(prevVertexTopIndex, vertexTopIndex, vertexBottomIndex, prevVertexBottomIndex); + } + else + { + trace('Skipping quad at index ${i}'); + } + + prevVertexTopIndex = vertexTopIndex; + prevVertexBottomIndex = vertexBottomIndex; + } + } + else + { + // Indexes are less than one pixel wide, so for each pixel we render the maximum of the samples that fall within it. + var prevVertexTopIndex:Int = -1; + var prevVertexBottomIndex:Int = -1; + for (i in 0...waveformWidth) + { + // Wrap Std.int around the whole range calculation, not just indexesPerPixel, otherwise you get weird issues with zooming. + var rangeStart:Int = Std.int(i * indexesPerPixel + startIndex); + var rangeEnd:Int = Std.int((i + 1) * indexesPerPixel + startIndex); + + var vertexTopY:Int = Std.int(waveformCenterPos - (waveformData.channel(0).maxSampleRangeMapped(rangeStart, rangeEnd) * waveformHeight / 2)); + var vertexBottomY:Int = Std.int(waveformCenterPos + (-waveformData.channel(0).minSampleRangeMapped(rangeStart, rangeEnd) * waveformHeight / 2)); + + // trace('Drawing index ${rangeStart} at pixel ${i} with MAX ${vertexTopY} and MIN ${vertexBottomY}'); + + var vertexTopIndex:Int = meshRender.build_vertex(i, vertexTopY); + var vertexBottomIndex:Int = meshRender.build_vertex(i, vertexBottomY); + + if (prevVertexTopIndex != -1 && prevVertexBottomIndex != -1) + { + meshRender.add_quad(prevVertexTopIndex, vertexTopIndex, vertexBottomIndex, prevVertexBottomIndex); + } + else + { + trace('Skipping quad at index ${i}'); + } + + prevVertexTopIndex = vertexTopIndex; + prevVertexBottomIndex = vertexBottomIndex; + } + } + + trace('Drawing ${duration} seconds of waveform with ${meshRender.vertex_count} vertices'); + + var oneSecondInPixels:Float = waveformWidth / duration; + + timeMarker.x = Std.int(oneSecondInPixels); + + // For each sample in the waveform... + // Add a MAX vertex and a MIN vertex. + // If previous MAX/MIN is empty, store. + // If previous MAX/MIN is not empty, draw a quad using current and previous MAX/MIN. Then store current MAX/MIN. + // Continue until end of waveform. + } + + public override function update(elapsed:Float):Void + { + super.update(elapsed); + + if (FlxG.keys.justPressed.SPACE) + { + if (waveformAudio.isPlaying) + { + waveformAudio.stop(); + } + else + { + waveformAudio.play(); + } + } + + if (waveformAudio.isPlaying) + { + var songTimeSeconds:Float = waveformAudio.time / 1000; + drawWaveform(songTimeSeconds, duration); + } + + if (FlxG.keys.justPressed.UP) + { + trace('Zooming out'); + duration += 1.0; + drawTheWaveform(); + } + if (FlxG.keys.justPressed.DOWN) + { + trace('Zooming in'); + duration -= 1.0; + drawTheWaveform(); + } + if (FlxG.keys.justPressed.LEFT) + { + trace('Seeking back'); + time -= 1.0; + drawTheWaveform(); + } + if (FlxG.keys.justPressed.RIGHT) + { + trace('Seeking forward'); + time += 1.0; + drawTheWaveform(); + } + } + + var time:Float = 0.0; + var duration:Float = 5.0; + + function drawTheWaveform():Void + { + drawWaveform(time, duration); + } + + public override function destroy():Void + { + super.destroy(); + } +}