From 194d8e6ce68882b0b32fce48d6a26a4fe34de478 Mon Sep 17 00:00:00 2001 From: Eric Myllyoja Date: Tue, 13 Sep 2022 01:09:30 -0400 Subject: [PATCH] WIP on song data loading --- source/funkin/InitState.hx | 8 +- source/funkin/modding/PolymodHandler.hx | 2 + source/funkin/play/song/ScriptedSong.hx | 1 + source/funkin/play/song/Song.hx | 119 ++++++++- source/funkin/play/song/SongData.hx | 314 ++++++++++++++++++++++- source/funkin/play/song/SongMigrator.hx | 79 ++++++ source/funkin/play/song/SongMitrator.hx | 12 - source/funkin/play/song/SongValidator.hx | 123 +++++++++ source/funkin/util/assets/DataAssets.hx | 6 +- 9 files changed, 621 insertions(+), 43 deletions(-) create mode 100644 source/funkin/play/song/SongMigrator.hx delete mode 100644 source/funkin/play/song/SongMitrator.hx create mode 100644 source/funkin/play/song/SongValidator.hx diff --git a/source/funkin/InitState.hx b/source/funkin/InitState.hx index cba1afb6f..26c6e3a64 100644 --- a/source/funkin/InitState.hx +++ b/source/funkin/InitState.hx @@ -9,9 +9,9 @@ import flixel.math.FlxRect; import flixel.util.FlxColor; import funkin.charting.ChartingState; import funkin.modding.module.ModuleHandler; -import funkin.play.PicoFight; import funkin.play.PlayState; import funkin.play.character.CharacterData.CharacterDataParser; +import funkin.play.song.SongData.SongDataParser; import funkin.play.stage.StageData; import funkin.ui.PreferencesMenu; import funkin.ui.animDebugShit.DebugBoundingState; @@ -28,11 +28,6 @@ import io.colyseus.Room; #if discord_rpc import Discord.DiscordClient; #end -#if desktop -import sys.FileSystem; -import sys.io.File; -import sys.thread.Thread; -#end /** * Initializes the game state using custom defines. @@ -123,6 +118,7 @@ class InitState extends FlxTransitionableState // FlxTransitionableState.skipNextTransOut = true; FlxTransitionableState.skipNextTransIn = true; + SongDataParser.loadSongCache(); StageDataParser.loadStageCache(); CharacterDataParser.loadCharacterCache(); ModuleHandler.buildModuleCallbacks(); diff --git a/source/funkin/modding/PolymodHandler.hx b/source/funkin/modding/PolymodHandler.hx index d08e8c8f6..ec14537fe 100644 --- a/source/funkin/modding/PolymodHandler.hx +++ b/source/funkin/modding/PolymodHandler.hx @@ -2,6 +2,7 @@ package funkin.modding; import funkin.modding.module.ModuleHandler; import funkin.play.character.CharacterData.CharacterDataParser; +import funkin.play.song.SongData; import funkin.play.stage.StageData; import polymod.Polymod; import polymod.backends.PolymodAssets.PolymodAssetType; @@ -231,6 +232,7 @@ class PolymodHandler // Reload everything that is cached. // Currently this freezes the game for a second but I guess that's tolerable? + SongDataParser.loadSongCache(); StageDataParser.loadStageCache(); CharacterDataParser.loadCharacterCache(); ModuleHandler.loadModuleCache(); diff --git a/source/funkin/play/song/ScriptedSong.hx b/source/funkin/play/song/ScriptedSong.hx index e89f68596..600b4f9bc 100644 --- a/source/funkin/play/song/ScriptedSong.hx +++ b/source/funkin/play/song/ScriptedSong.hx @@ -1,5 +1,6 @@ package funkin.play.song; +import funkin.play.song.Song; import polymod.hscript.HScriptedClass; @:hscriptClass diff --git a/source/funkin/play/song/Song.hx b/source/funkin/play/song/Song.hx index a287bc048..77ae1bd56 100644 --- a/source/funkin/play/song/Song.hx +++ b/source/funkin/play/song/Song.hx @@ -2,6 +2,8 @@ package funkin.play.song; import funkin.play.song.SongData.SongDataParser; import funkin.play.song.SongData.SongMetadata; +import funkin.play.song.SongData.SongTimeChange; +import funkin.play.song.SongData.SongTimeFormat; /** * This is a data structure managing information about the current song. @@ -14,30 +16,85 @@ import funkin.play.song.SongData.SongMetadata; */ class Song // implements IPlayStateScriptedClass { - public var songId(default, null):String; + public final songId:String; - public var songName(get, null):String; + final _metadata:Array; - final _metadata:SongMetadata; - - // final _chartData:SongChartData; + final difficulties:Map; public function new(id:String) { this.songId = id; + difficulties = new Map(); + _metadata = SongDataParser.parseSongMetadata(songId); - if (_metadata == null) + if (_metadata == null || _metadata.length == 0) { throw 'Could not find song data for songId: $songId'; } + + populateFromMetadata(); } - function get_songName():String + function populateFromMetadata() { - if (_metadata == null) - return null; - return _metadata.name; + // Variations may have different artist, time format, generatedBy, etc. + for (metadata in _metadata) + { + for (diffId in metadata.playData.difficulties) + { + var difficulty = new SongDifficulty(diffId, metadata.variation); + + difficulty.songName = metadata.songName; + difficulty.songArtist = metadata.artist; + difficulty.timeFormat = metadata.timeFormat; + difficulty.divisions = metadata.divisions; + difficulty.timeChanges = metadata.timeChanges; + difficulty.loop = metadata.loop; + difficulty.generatedBy = metadata.generatedBy; + + difficulties.set(diffId, difficulty); + } + } + } + + /** + * Parse and cache the chart for a specific difficulty. + */ + public function cacheChart(diffId:String) + { + getDifficulty(diffId).cacheChart(); + } + + /** + * Parse and cache the chart for all difficulties of this song. + */ + public function cacheCharts() + { + for (difficulty in difficulties) + { + difficulty.cacheChart(); + } + } + + /** + * Retrieve the metadata for a specific difficulty, including the chart if it is loaded. + */ + public function getDifficulty(diffId:String):SongDifficulty + { + return difficulties.get(diffId); + } + + /** + * Purge the cached chart data for each difficulty of this song. + */ + public function clearCharts() + { + for (diff in difficulties) + { + diff.clearChart(); + } } public function toString():String @@ -45,3 +102,45 @@ class Song // implements IPlayStateScriptedClass return 'Song($songId)'; } } + +class SongDifficulty +{ + /** + * The difficulty ID, such as `easy` or `hard`. + */ + public final difficulty:String; + + /** + * The metadata file that contains this difficulty. + */ + public final variation:String; + + public var songName:String = SongValidator.DEFAULT_SONGNAME; + public var songArtist:String = SongValidator.DEFAULT_ARTIST; + public var timeFormat:SongTimeFormat = SongValidator.DEFAULT_TIMEFORMAT; + public var divisions:Int = SongValidator.DEFAULT_DIVISIONS; + public var loop:Bool = SongValidator.DEFAULT_LOOP; + public var generatedBy:String = SongValidator.DEFAULT_GENERATEDBY; + + public var timeChanges:Array = []; + + public var scrollSpeed(default, null):Float = SongValidator.DEFAULT_SCROLLSPEED; + + // public var notes(default, null):Array<; + + public function new(diffId:String, variation:String) + { + this.difficulty = diffId; + this.variation = variation; + } + + public function cacheChart():Void + { + // TODO: Parse chart data + } + + public function clearChart():Void + { + // notes = null; + } +} diff --git a/source/funkin/play/song/SongData.hx b/source/funkin/play/song/SongData.hx index 14758bfcb..64f04c825 100644 --- a/source/funkin/play/song/SongData.hx +++ b/source/funkin/play/song/SongData.hx @@ -1,6 +1,10 @@ package funkin.play.song; +import flixel.util.typeLimit.OneOfTwo; +import funkin.play.song.ScriptedSong; import funkin.util.assets.DataAssets; +import haxe.DynamicAccess; +import haxe.Json; import openfl.utils.Assets; import thx.semver.Version; @@ -11,19 +15,14 @@ using StringTools; */ class SongDataParser { - /** - * The current version string for the stage data format. - * Handle breaking changes by incrementing this value - * and adding migration to the SongMigrator class. - */ - public static final CHART_VERSION:String = "2.0.0"; - /** * A list containing all the songs available to the game. */ static final songCache:Map = new Map(); static final DEFAULT_SONG_ID = 'UNKNOWN'; + static final SONG_DATA_PATH = 'songs/'; + static final SONG_DATA_SUFFIX = '/metadata.json'; /** * Parses and preloads the game's song metadata and scripts when the game starts. @@ -57,7 +56,7 @@ class SongDataParser // // UNSCRIPTED SONGS // - var songIdList:Array = DataAssets.listDataFilesInPath('songs/'); + var songIdList:Array = DataAssets.listDataFilesInPath(SONG_DATA_PATH, SONG_DATA_SUFFIX); var unscriptedSongIds:Array = songIdList.filter(function(songId:String):Bool { return !songCache.exists(songId); @@ -77,6 +76,7 @@ class SongDataParser catch (e) { trace(' An error occurred while loading song data: ${songId}'); + trace(e); // Assume error was already logged. continue; } @@ -111,14 +111,50 @@ class SongDataParser } } - public static function parseSongMetadata(songId:String):Null + public static function parseSongMetadata(songId:String):Array { - return null; + var result:Array = []; + + var rawJson:String = loadSongMetadataFile(songId); + var jsonData:Dynamic = null; + try + { + jsonData = Json.parse(rawJson); + } + catch (e) + { + } + + var songMetadata:SongMetadata = SongMigrator.migrateSongMetadata(jsonData, songId); + songMetadata = SongValidator.validateSongMetadata(songMetadata, songId); + + if (songMetadata == null) + { + return result; + } + + result.push(songMetadata); + + var variations = songMetadata.playData.songVariations; + + for (variation in variations) + { + var variationRawJson:String = loadSongMetadataFile(songId, variation); + var variationSongMetadata:SongMetadata = SongMigrator.migrateSongMetadata(variationRawJson, '${songId}_${variation}'); + variationSongMetadata = SongValidator.validateSongMetadata(variationSongMetadata, '${songId}_${variation}'); + if (variationSongMetadata != null) + { + variationSongMetadata.variation = variation; + result.push(variationSongMetadata); + } + } + + return result; } - static function loadSongMetadataFile(songPath:String, variant:String = ''):String + static function loadSongMetadataFile(songPath:String, variation:String = ''):String { - var songMetadataFilePath:String = (variant != '') ? Paths.json('songs/${songPath}') : Paths.json('songs/${songPath}'); + var songMetadataFilePath:String = (variation != '') ? Paths.json('$SONG_DATA_PATH$songPath/metadata-$variation') : Paths.json('$SONG_DATA_PATH$songPath/metadata'); var rawJson:String = Assets.getText(songMetadataFilePath).trim(); @@ -129,10 +165,52 @@ class SongDataParser return rawJson; } + + public static function parseSongChartData(songId:String, variation:String = ""):SongChartData + { + var rawJson:String = loadSongChartDataFile(songId, variation); + var jsonData:Dynamic = null; + try + { + jsonData = Json.parse(rawJson); + } + catch (e) + { + } + + var songChartData:SongChartData = SongMigrator.migrateSongChartData(jsonData, songId); + songChartData = SongValidator.validateSongChartData(songChartData, songId); + + if (songChartData == null) + { + trace('Failed to validate song chart data: ${songId}'); + return null; + } + + return songChartData; + } + + static function loadSongChartDataFile(songPath:String, variation:String = ''):String + { + var songChartDataFilePath:String = (variation != '') ? Paths.json('$SONG_DATA_PATH$songPath/chart-$variation') : Paths.json('$SONG_DATA_PATH$songPath/chart'); + + var rawJson:String = Assets.getText(songChartDataFilePath).trim(); + + while (!rawJson.endsWith("}")) + { + rawJson = rawJson.substr(0, rawJson.length - 1); + } + + return rawJson; + } } typedef SongMetadata = { + /** + * A semantic versioning string for the song data format. + * + */ var version:Version; var songName:String; @@ -140,12 +218,224 @@ typedef SongMetadata = var timeFormat:SongTimeFormat; var divisions:Int; var timeChanges:Array; + var loop:Bool; + var playData:SongPlayData; + var generatedBy:String; + + /** + * Defaults to ''. Populated later. + */ + var variation:String; }; +typedef SongPlayData = +{ + var songVariations:Array; + var difficulties:Array; + + /** + * Keys are the player characters and the values give info on what opponent/GF/inst to use. + */ + var playableChars:DynamicAccess; + + var stage:String; + var noteSkin:String; +} + +typedef RawSongPlayableChar = +{ + var g:String; + var o:String; + var i:String; +} + +abstract SongPlayableChar(RawSongPlayableChar) +{ + public function new(girlfriend:String, opponent:String, inst:String = "") + { + this = { + g: girlfriend, + o: opponent, + i: inst + }; + } + + public var girlfriend(get, set):String; + + public function get_girlfriend():String + { + return this.g; + } + + public function set_girlfriend(value:String):String + { + return this.g = value; + } + + public var opponent(get, set):String; + + public function get_opponent():String + { + return this.o; + } + + public function set_opponent(value:String):String + { + return this.o = value; + } + + public var inst(get, set):String; + + public function get_inst():String + { + return this.i; + } + + public function set_inst(value:String):String + { + return this.i = value; + } +} + typedef SongChartData = { }; +typedef RawSongTimeChange = +{ + /** + * Timestamp in specified `timeFormat`. + */ + var t:Float; + + /** + * Time in beats (int). The game will calculate further beat values based on this one, + * so it can do it in a simple linear fashion. + */ + var b:Int; + + /** + * Quarter notes per minute (float). Cannot be empty in the first element of the list, + * but otherwise it's optional, and defaults to the value of the previous element. + */ + var bpm:Float; + + /** + * Time signature numerator (int). Optional, defaults to 4. + */ + var n:Int; + + /** + * Time signature denominator (int). Optional, defaults to 4. Should only ever be a power of two. + */ + var d:Int; + + /** + * Beat tuplets (Array or int). This defines how many steps each beat is divided into. + * It can either be an array of length `n` (see above) or a single integer number. + * Optional, defaults to `[4]`. + */ + var bt:OneOfTwo>; +} + +/** + * Add aliases to the minimalized property names of the typedef, + * to improve readability. + */ +abstract SongTimeChange(RawSongTimeChange) +{ + public function new(timeStamp:Float, beatTime:Int, bpm:Float, timeSignatureNum:Int = 4, timeSignatureDen:Int = 4, beatTuplets:Array) + { + this = { + t: timeStamp, + b: beatTime, + bpm: bpm, + n: timeSignatureNum, + d: timeSignatureDen, + bt: beatTuplets, + } + } + + public var timeStamp(get, set):Float; + + public function get_timeStamp():Float + { + return this.t; + } + + public function set_timeStamp(value:Float):Float + { + return this.t = value; + } + + public var beatTime(get, set):Int; + + public function get_beatTime():Int + { + return this.b; + } + + public function set_beatTime(value:Int):Int + { + return this.b = value; + } + + public var bpm(get, set):Float; + + public function get_bpm():Float + { + return this.bpm; + } + + public function set_bpm(value:Float):Float + { + return this.bpm = value; + } + + public var timeSignatureNum(get, set):Int; + + public function get_timeSignatureNum():Int + { + return this.n; + } + + public function set_timeSignatureNum(value:Int):Int + { + return this.n = value; + } + + public var timeSignatureDen(get, set):Int; + + public function get_timeSignatureDen():Int + { + return this.d; + } + + public function set_timeSignatureDen(value:Int):Int + { + return this.d = value; + } + + public var beatTuplets(get, set):Array; + + public function get_beatTuplets():Array + { + if (Std.isOfType(this.bt, Int)) + { + return [this.bt]; + } + else + { + return this.bt; + } + } + + public function set_beatTuplets(value:Array):Array + { + return this.bt = value; + } +} + enum abstract SongTimeFormat(String) from String to String { var TICKS = "ticks"; diff --git a/source/funkin/play/song/SongMigrator.hx b/source/funkin/play/song/SongMigrator.hx new file mode 100644 index 000000000..36573a9fd --- /dev/null +++ b/source/funkin/play/song/SongMigrator.hx @@ -0,0 +1,79 @@ +package funkin.play.song; + +import funkin.play.song.SongData.SongChartData; +import funkin.play.song.SongData.SongMetadata; +import funkin.util.VersionUtil; + +class SongMigrator +{ + /** + * The current latest version string for the song data format. + * Handle breaking changes by incrementing this value + * and adding migration to the SongMigrator class. + */ + public static final CHART_VERSION:String = "2.0.0"; + + public static final CHART_VERSION_RULE:String = "2.0.x"; + + public static function migrateSongMetadata(jsonData:Dynamic, songId:String):SongMetadata + { + if (jsonData.version) + { + if (VersionUtil.validateVersion(jsonData.version, CHART_VERSION_RULE)) + { + trace('[SONGDATA] Song (${songId}) metadata version (${jsonData.version}) is valid and up-to-date.'); + + var songMetadata:SongMetadata = cast jsonData; + + return songMetadata; + } + else + { + trace('[SONGDATA] Song (${songId}) metadata version (${jsonData.version}) is outdated.'); + switch (jsonData.version) + { + // TODO: Add migration functions as cases here. + default: + // Unknown version. + trace('[SONGDATA] Song (${songId}) unknown metadata version: ${jsonData.version}'); + } + } + } + else + { + trace('[SONGDATA] Song metadata version is missing.'); + } + return null; + } + + public static function migrateSongChartData(jsonData:Dynamic, songId:String):SongChartData + { + if (jsonData.version) + { + if (VersionUtil.validateVersion(jsonData.version, CHART_VERSION_RULE)) + { + trace('[SONGDATA] Song (${songId}) chart version (${jsonData.version}) is valid and up-to-date.'); + + var songMetadata:SongMetadata = cast jsonData; + + return songMetadata; + } + else + { + trace('[SONGDATA] Song (${songId}) chart version (${jsonData.version}) is outdated.'); + switch (jsonData.version) + { + // TODO: Add migration functions as cases here. + default: + // Unknown version. + trace('[SONGDATA] Song (${songId}) unknown chart version: ${jsonData.version}'); + } + } + } + else + { + trace('[SONGDATA] Song chart version is missing.'); + } + return null; + } +} diff --git a/source/funkin/play/song/SongMitrator.hx b/source/funkin/play/song/SongMitrator.hx deleted file mode 100644 index cad03b8ff..000000000 --- a/source/funkin/play/song/SongMitrator.hx +++ /dev/null @@ -1,12 +0,0 @@ -package funkin.play.song; - -class SongMigrator -{ - public static function migrateSongMetadata(song:Song, jsonData:Dynamic) - { - } - - public static function migrateSongChart(song:Song, jsonData:Dynamic) - { - } -} diff --git a/source/funkin/play/song/SongValidator.hx b/source/funkin/play/song/SongValidator.hx new file mode 100644 index 000000000..413c28e1a --- /dev/null +++ b/source/funkin/play/song/SongValidator.hx @@ -0,0 +1,123 @@ +package funkin.play.song; + +import funkin.play.song.SongData.SongChartData; +import funkin.play.song.SongData.SongMetadata; +import funkin.play.song.SongData.SongPlayData; +import funkin.play.song.SongData.SongTimeChange; +import funkin.play.song.SongData.SongTimeFormat; + +/** + * For SongMetadata and SongChartData objects, + * ensures mandatory fields are present and populates optional fields with default values. + */ +class SongValidator +{ + public static final DEFAULT_SONGNAME:String = "Unknown"; + public static final DEFAULT_ARTIST:String = "Unknown"; + public static final DEFAULT_TIMEFORMAT:SongTimeFormat = SongTimeFormat.MILLISECONDS; + public static final DEFAULT_DIVISIONS:Int = -1; + public static final DEFAULT_LOOP:Bool = false; + public static final DEFAULT_GENERATEDBY:String = "Unknown"; + public static final DEFAULT_SCROLLSPEED:Float = 1.0; + + /** + * Validates the fields of a SongMetadata object (excluding the version field). + * + * @param input The SongMetadata object to validate. + * @param songId The ID of the song being validated. Only used for error messages. + * @return The validated SongMetadata object. + */ + public static function validateSongMetadata(input:SongMetadata, songId:String = 'unknown'):SongMetadata + { + if (input == null) + { + trace('[SONGDATA] Could not parse metadata for song ${songId}'); + return null; + } + + if (input.songName == null) + { + trace('[SONGDATA] Song ${songId} is missing a songName field. '); + input.songName = DEFAULT_SONGNAME; + } + if (input.artist == null) + { + trace('[SONGDATA] Song ${songId} is missing an artist field. '); + input.artist = DEFAULT_ARTIST; + } + if (input.timeFormat == null) + { + trace('[SONGDATA] Song ${songId} is missing a timeFormat field. '); + input.timeFormat = DEFAULT_TIMEFORMAT; + } + if (input.generatedBy == null) + { + input.generatedBy = DEFAULT_GENERATEDBY; + } + + input.timeChanges = validateTimeChanges(input.timeChanges, songId); + input.playData = validatePlayData(input.playData, songId); + + input.variation = ''; + + return input; + } + + /** + * Validates the fields of a SongPlayData object. + * + * @param input The SongPlayData object to validate. + * @param songId The ID of the song being validated. Only used for error messages. + * @return The validated SongPlayData object. + */ + public static function validatePlayData(input:SongPlayData, songId:String = 'unknown'):SongPlayData + { + return input; + } + + /** + * Validates the fields of a TimeChange object. + * + * @param input The TimeChange object to validate. + * @param songId The ID of the song being validated. Only used for error messages. + * @return The validated TimeChange object. + */ + public static function validateTimeChange(input:SongTimeChange, songId:String = 'unknown'):SongTimeChange + { + return input; + } + + /** + * Validates multiple TimeChange objects in an array. + */ + public static function validateTimeChanges(input:Array, songId:String = 'unknown'):Array + { + if (input == null) + { + trace('[SONGDATA] Song ${songId} is missing a timeChanges field. '); + return []; + } + + input = input.map((timeChange) -> validateTimeChange(timeChange, songId)); + + return input; + } + + /** + * Validates the fields of a SongChartData object (excluding the version field). + * + * @param input The SongChartData object to validate. + * @param songId The ID of the song being validated. Only used for error messages. + * @return The validated SongChartData object. + */ + public static function validateSongChartData(input:SongChartData, songId:String = 'unknown'):SongChartData + { + if (input == null) + { + trace('[SONGDATA] Could not parse chart data for song ${songId}'); + return null; + } + + return input; + } +} diff --git a/source/funkin/util/assets/DataAssets.hx b/source/funkin/util/assets/DataAssets.hx index ed4805276..a110ecc2f 100644 --- a/source/funkin/util/assets/DataAssets.hx +++ b/source/funkin/util/assets/DataAssets.hx @@ -9,7 +9,7 @@ class DataAssets return 'assets/data/${path}'; } - public static function listDataFilesInPath(path:String, ?ext:String = '.json'):Array + public static function listDataFilesInPath(path:String, ?suffix:String = '.json'):Array { var textAssets = openfl.utils.Assets.list(); var queryPath = buildDataPath(path); @@ -17,9 +17,9 @@ class DataAssets var results:Array = []; for (textPath in textAssets) { - if (textPath.startsWith(queryPath) && textPath.endsWith(ext)) + if (textPath.startsWith(queryPath) && textPath.endsWith(suffix)) { - var pathNoSuffix = textPath.substring(0, textPath.length - ext.length); + var pathNoSuffix = textPath.substring(0, textPath.length - suffix.length); var pathNoPrefix = pathNoSuffix.substring(queryPath.length); // No duplicates! Why does this happen?