Work in progress on new Waveform renderer

.vscode/settings.json

@@ -125,6 +125,11 @@
       "target": "windows",
       "args": ["-debug", "-DLATENCY"]
     },
+    {
+      "label": "Windows / Debug (Waveform Test)",
+      "target": "windows",
+      "args": ["-debug", "-DWAVEFORM"]
+    },
     {
       "label": "HTML5 / Debug",
       "target": "html5",

assets

@@ -1 +1 @@
-Subproject commit 7e19c4cfa7db57178f03ed4a58a9fd4d2b93dea7
+Subproject commit 3d92b497682727d34eaa55e564e0bd9faea1c9d7

source/funkin/audio/waveform/WaveformData.hx

@@ -0,0 +1,243 @@
+package funkin.audio.waveform;
+
+import funkin.util.MathUtil;
+
+@:nullSafety
+class WaveformData
+{
+  /**
+   * The version of the waveform data format.
+   * @default `2` (-1 if not specified/invalid)
+   */
+  public var version(default, null):Int = -1;
+
+  /**
+   * The number of channels in the waveform.
+   */
+  public var channels(default, null):Int = 1;
+
+  @:alias('sample_rate')
+  public var sampleRate(default, null):Int = 44100;
+
+  /**
+   * Number of input audio samples per output waveform data point.
+   * At base zoom level this is number of samples per pixel.
+   * Lower values can more accurately represent the waveform when zoomed in, but take more data.
+   */
+  @:alias('samples_per_pixel')
+  public var samplesPerPixel(default, null):Int = 256;
+
+  /**
+   * Number of bits to use for each sample value. Valid values are `8` and `16`.
+   */
+  public var bits(default, null):Int = 16;
+
+  /**
+   * Number of output waveform data points.
+   */
+  public var length(default, null):Int = 0; // Array size is (4 * length)
+
+  /**
+   * Array of Int16 values representing the waveform.
+   * TODO: Use an `openfl.Vector` for performance.
+   */
+  public var data(default, null):Array<Int> = [];
+
+  @:jignored
+  var channelData:Null<Array<WaveformDataChannel>> = null;
+
+  public function new() {}
+
+  function buildChannelData():Array<WaveformDataChannel>
+  {
+    channelData = [];
+    for (i in 0...channels)
+    {
+      channelData.push(new WaveformDataChannel(this, i));
+    }
+    return channelData;
+  }
+
+  public function channel(index:Int)
+  {
+    return (channelData == null) ? buildChannelData()[index] : channelData[index];
+  }
+
+  public function get(index:Int):Int
+  {
+    return data[index] ?? 0;
+  }
+
+  public function set(index:Int, value:Int)
+  {
+    data[index] = value;
+  }
+
+  /**
+   * Maximum possible value for a waveform data point.
+   * The minimum possible value is (-1 * maxSampleValue)
+   */
+  public function maxSampleValue():Int
+  {
+    if (_maxSampleValue != -1) return _maxSampleValue;
+    return _maxSampleValue = Std.int(Math.pow(2, bits));
+  }
+
+  /**
+   * Cache the value because `Math.pow` is expensive and the value gets used a lot.
+   */
+  @:jignored
+  var _maxSampleValue:Int = -1;
+
+  /**
+   * @return The length of the waveform in samples.
+   */
+  public function lenSamples():Int
+  {
+    return length * samplesPerPixel;
+  }
+
+  /**
+   * @return The length of the waveform in seconds.
+   */
+  public function lenSeconds():Float
+  {
+    return lenSamples() / sampleRate;
+  }
+
+  /**
+   * Given the time in seconds, return the waveform data point index.
+   */
+  public function secondsToIndex(seconds:Float):Int
+  {
+    return Std.int(seconds * sampleRate / samplesPerPixel);
+  }
+
+  /**
+   * Given a waveform data point index, return the time in seconds.
+   */
+  public function indexToSeconds(index:Int):Float
+  {
+    return index * samplesPerPixel / sampleRate;
+  }
+
+  /**
+   * Given the percentage progress through the waveform, return the waveform data point index.
+   */
+  public function percentToIndex(percent:Float):Int
+  {
+    return Std.int(percent * length);
+  }
+
+  /**
+   * Given a waveform data point index, return the percentage progress through the waveform.
+   */
+  public function indexToPercent(index:Int):Float
+  {
+    return index / length;
+  }
+}
+
+class WaveformDataChannel
+{
+  var parent:WaveformData;
+  var channelId:Int;
+
+  public function new(parent:WaveformData, channelId:Int)
+  {
+    this.parent = parent;
+    this.channelId = channelId;
+  }
+
+  public function minSample(i:Int)
+  {
+    var offset = (i * parent.channels + this.channelId) * 2;
+    return parent.get(offset);
+  }
+
+  /**
+   * Mapped to a value between 0 and 1.
+   */
+  public function minSampleMapped(i:Int)
+  {
+    return minSample(i) / parent.maxSampleValue();
+  }
+
+  /**
+   * Minimum value within the range of samples.
+   * @param i
+   */
+  public function minSampleRange(start:Int, end:Int)
+  {
+    var min = parent.maxSampleValue();
+    for (i in start...end)
+    {
+      var sample = minSample(i);
+      if (sample < min) min = sample;
+    }
+    return min;
+  }
+
+  /**
+   * Maximum value within the range of samples, mapped to a value between 0 and 1.
+   * @param i
+   */
+  public function minSampleRangeMapped(start:Int, end:Int)
+  {
+    return minSampleRange(start, end) / parent.maxSampleValue();
+  }
+
+  public function maxSample(i:Int)
+  {
+    var offset = (i * parent.channels + this.channelId) * 2 + 1;
+    return parent.get(offset);
+  }
+
+  /**
+   * Mapped to a value between 0 and 1.
+   */
+  public function maxSampleMapped(i:Int)
+  {
+    return maxSample(i) / parent.maxSampleValue();
+  }
+
+  /**
+   * Maximum value within the range of samples.
+   * @param i
+   */
+  public function maxSampleRange(start:Int, end:Int)
+  {
+    var max = -parent.maxSampleValue();
+    for (i in start...end)
+    {
+      var sample = maxSample(i);
+      if (sample > max) max = sample;
+    }
+    return max;
+  }
+
+  /**
+   * Maximum value within the range of samples, mapped to a value between 0 and 1.
+   * @param i
+   */
+  public function maxSampleRangeMapped(start:Int, end:Int)
+  {
+    return maxSampleRange(start, end) / parent.maxSampleValue();
+  }
+
+  /**
+   * Maximum value within the range of samples, mapped to a value between 0 and 1.
+   * @param i
+   */
+  public function setMinSample(i:Int, value:Int)
+  {
+    var offset = (i * parent.channels + this.channelId) * 2;
+    parent.set(offset, value);
+  }
+
+  public function setMaxSample(i:Int, value:Int)
+  {
+    var offset = (i * parent.channels + this.channelId) * 2 + 1;
+    parent.set(offset, value);
+  }
+}

source/funkin/audio/waveform/WaveformDataParser.hx

@@ -0,0 +1,32 @@
+package funkin.audio.waveform;
+
+class WaveformDataParser
+{
+  public static function parseWaveformData(path:String):Null<WaveformData>
+  {
+    var rawJson:String = openfl.Assets.getText(path).trim();
+    return parseWaveformDataString(rawJson, path);
+  }
+
+  public static function parseWaveformDataString(contents:String, ?fileName:String):Null<WaveformData>
+  {
+    var parser = new json2object.JsonParser<WaveformData>();
+    parser.ignoreUnknownVariables = false;
+    parser.fromJson(contents, fileName);
+
+    if (parser.errors.length > 0)
+    {
+      printErrors(parser.errors, fileName);
+      return null;
+    }
+    return parser.value;
+  }
+
+  static function printErrors(errors:Array<json2object.Error>, id:String = ''):Void
+  {
+    trace('[WAVEFORM] Failed to parse waveform data: ${id}');
+
+    for (error in errors)
+      funkin.data.DataError.printError(error);
+  }
+}

source/funkin/ui/debug/WaveformTestState.hx

@@ -0,0 +1,202 @@
+package funkin.ui.debug;
+
+import flixel.FlxSprite;
+import flixel.util.FlxColor;
+import funkin.audio.FunkinSound;
+import funkin.audio.waveform.WaveformData;
+import funkin.audio.waveform.WaveformDataParser;
+import funkin.graphics.rendering.MeshRender;
+
+class WaveformTestState extends MusicBeatState
+{
+  public function new()
+  {
+    super();
+  }
+
+  var waveformData:WaveformData;
+
+  var waveformAudio:FunkinSound;
+
+  var meshRender:MeshRender;
+
+  var timeMarker:FlxSprite;
+
+  public override function create():Void
+  {
+    super.create();
+
+    waveformData = WaveformDataParser.parseWaveformData(Paths.json("waveform/dadbattle-erect/dadbattle-erect.waveform"));
+
+    waveformAudio = FunkinSound.load(Paths.music('dadbattle-erect/dadbattle-erect'));
+
+    var lightBlue:FlxColor = FlxColor.fromString("#ADD8E6");
+    meshRender = new MeshRender(0, 0, lightBlue);
+    add(meshRender);
+
+    timeMarker = new FlxSprite(0, FlxG.height * 1 / 6);
+    timeMarker.makeGraphic(1, Std.int(FlxG.height * 2 / 3), FlxColor.RED);
+    add(timeMarker);
+
+    drawWaveform(time, duration);
+  }
+
+  /**
+   * @param offsetX Horizontal offset to draw the waveform at, in samples.
+   */
+  function drawWaveform(timeSeconds:Float, duration:Float):Void
+  {
+    meshRender.clear();
+
+    var offsetX:Int = waveformData.secondsToIndex(timeSeconds);
+
+    var waveformHeight:Int = Std.int(FlxG.height * (2 / 3));
+    var waveformWidth:Int = FlxG.width;
+    var waveformCenterPos:Int = Std.int(FlxG.height / 2);
+
+    var oneSecondInIndices:Int = waveformData.secondsToIndex(1);
+
+    var startTime:Float = -1.0;
+    var endTime:Float = startTime + duration;
+
+    var startIndex:Int = Std.int(offsetX + (oneSecondInIndices * startTime));
+    var endIndex:Int = Std.int(offsetX + (oneSecondInIndices * (startTime + duration)));
+
+    var pixelsPerIndex:Float = waveformWidth / (endIndex - startIndex);
+    var indexesPerPixel:Float = (endIndex - startIndex) / waveformWidth;
+
+    if (pixelsPerIndex >= 1.0)
+    {
+      // Each index is at least one pixel wide, so we render each index.
+      var prevVertexTopIndex:Int = -1;
+      var prevVertexBottomIndex:Int = -1;
+      for (i in startIndex...endIndex)
+      {
+        var pixelPos:Int = Std.int((i - startIndex) * pixelsPerIndex);
+
+        var vertexTopY:Int = Std.int(waveformCenterPos - (waveformData.channel(0).maxSampleMapped(i) * waveformHeight / 2));
+        var vertexBottomY:Int = Std.int(waveformCenterPos + (-waveformData.channel(0).minSampleMapped(i) * waveformHeight / 2));
+
+        var vertexTopIndex:Int = meshRender.build_vertex(pixelPos, vertexTopY);
+        var vertexBottomIndex:Int = meshRender.build_vertex(pixelPos, vertexBottomY);
+
+        if (prevVertexTopIndex != -1 && prevVertexBottomIndex != -1)
+        {
+          meshRender.add_quad(prevVertexTopIndex, vertexTopIndex, vertexBottomIndex, prevVertexBottomIndex);
+        }
+        else
+        {
+          trace('Skipping quad at index ${i}');
+        }
+
+        prevVertexTopIndex = vertexTopIndex;
+        prevVertexBottomIndex = vertexBottomIndex;
+      }
+    }
+    else
+    {
+      // Indexes are less than one pixel wide, so for each pixel we render the maximum of the samples that fall within it.
+      var prevVertexTopIndex:Int = -1;
+      var prevVertexBottomIndex:Int = -1;
+      for (i in 0...waveformWidth)
+      {
+        // Wrap Std.int around the whole range calculation, not just indexesPerPixel, otherwise you get weird issues with zooming.
+        var rangeStart:Int = Std.int(i * indexesPerPixel + startIndex);
+        var rangeEnd:Int = Std.int((i + 1) * indexesPerPixel + startIndex);
+
+        var vertexTopY:Int = Std.int(waveformCenterPos - (waveformData.channel(0).maxSampleRangeMapped(rangeStart, rangeEnd) * waveformHeight / 2));
+        var vertexBottomY:Int = Std.int(waveformCenterPos + (-waveformData.channel(0).minSampleRangeMapped(rangeStart, rangeEnd) * waveformHeight / 2));
+
+        // trace('Drawing index ${rangeStart} at pixel ${i} with MAX ${vertexTopY} and MIN ${vertexBottomY}');
+
+        var vertexTopIndex:Int = meshRender.build_vertex(i, vertexTopY);
+        var vertexBottomIndex:Int = meshRender.build_vertex(i, vertexBottomY);
+
+        if (prevVertexTopIndex != -1 && prevVertexBottomIndex != -1)
+        {
+          meshRender.add_quad(prevVertexTopIndex, vertexTopIndex, vertexBottomIndex, prevVertexBottomIndex);
+        }
+        else
+        {
+          trace('Skipping quad at index ${i}');
+        }
+
+        prevVertexTopIndex = vertexTopIndex;
+        prevVertexBottomIndex = vertexBottomIndex;
+      }
+    }
+
+    trace('Drawing ${duration} seconds of waveform with ${meshRender.vertex_count} vertices');
+
+    var oneSecondInPixels:Float = waveformWidth / duration;
+
+    timeMarker.x = Std.int(oneSecondInPixels);
+
+    // For each sample in the waveform...
+    // Add a MAX vertex and a MIN vertex.
+    //   If previous MAX/MIN is empty, store.
+    //   If previous MAX/MIN is not empty, draw a quad using current and previous MAX/MIN. Then store current MAX/MIN.
+    // Continue until end of waveform.
+  }
+
+  public override function update(elapsed:Float):Void
+  {
+    super.update(elapsed);
+
+    if (FlxG.keys.justPressed.SPACE)
+    {
+      if (waveformAudio.isPlaying)
+      {
+        waveformAudio.stop();
+      }
+      else
+      {
+        waveformAudio.play();
+      }
+    }
+
+    if (waveformAudio.isPlaying)
+    {
+      var songTimeSeconds:Float = waveformAudio.time / 1000;
+      drawWaveform(songTimeSeconds, duration);
+    }
+
+    if (FlxG.keys.justPressed.UP)
+    {
+      trace('Zooming out');
+      duration += 1.0;
+      drawTheWaveform();
+    }
+    if (FlxG.keys.justPressed.DOWN)
+    {
+      trace('Zooming in');
+      duration -= 1.0;
+      drawTheWaveform();
+    }
+    if (FlxG.keys.justPressed.LEFT)
+    {
+      trace('Seeking back');
+      time -= 1.0;
+      drawTheWaveform();
+    }
+    if (FlxG.keys.justPressed.RIGHT)
+    {
+      trace('Seeking forward');
+      time += 1.0;
+      drawTheWaveform();
+    }
+  }
+
+  var time:Float = 0.0;
+  var duration:Float = 5.0;
+
+  function drawTheWaveform():Void
+  {
+    drawWaveform(time, duration);
+  }
+
+  public override function destroy():Void
+  {
+    super.destroy();
+  }
+}