support custom text encoding for .tsc and stage table (#259)

* feat: support optional custom encoding
* feat: support custom encoding for stage table

Cargo.toml

@@ -66,6 +66,7 @@ cpal = { git = "https://github.com/doukutsu-rs/cpal", rev = "9d269d8724102404e73
 directories = "3"
 discord-rich-presence = { version = "0.2", optional = true }
 downcast = "0.11"
+encoding_rs = "0.8.33"
 fern = "0.6.2"
 glutin = { git = "https://github.com/doukutsu-rs/glutin.git", rev = "2dd95f042e6e090d36f577cbea125560dd99bd27", optional = true, default_features = false, features = ["x11"] }
 imgui = "0.8"

src/engine_constants/mod.rs

@@ -277,6 +277,7 @@ pub struct EngineConstants {
     pub missile_flags: Vec<u16>,
     pub locales: Vec<Locale>,
     pub gamepad: GamepadConsts,
+    pub stage_encoding: Option<TextScriptEncoding>,
 }
 
 impl EngineConstants {
@@ -1600,6 +1601,7 @@ impl EngineConstants {
                     (Axis::TriggerRight, GamepadConsts::rects(Rect::new(32, 80, 64, 96))),
                 ]),
             },
+            stage_encoding: None,
         }
     }
 

src/game/map.rs

@@ -1,18 +1,17 @@
 use std::collections::HashMap;
 use std::io;
-use std::io::{BufRead, BufReader, Cursor, Read};
+use std::io::{BufRead, BufReader, Read};
 use std::sync::Arc;
 
-use byteorder::{LE, ReadBytesExt};
+use byteorder::{ReadBytesExt, LE};
 
 use crate::common::{Color, Rect};
 use crate::framework::context::Context;
-use crate::framework::error::{GameError, GameResult};
 use crate::framework::error::GameError::ResourceLoadError;
+use crate::framework::error::{GameError, GameResult};
 use crate::framework::filesystem;
 use crate::game::shared_game_state::TileSize;
 use crate::game::stage::{PxPackScroll, PxPackStageData, StageData};
-use crate::util::encoding::read_cur_shift_jis;
 
 static SUPPORTED_PXM_VERSIONS: [u8; 1] = [0x10];
 static SUPPORTED_PXE_VERSIONS: [u8; 2] = [0, 0x10];
@@ -84,22 +83,11 @@ impl Map {
         }
 
         fn read_string<R: io::Read>(map_data: &mut R) -> GameResult<String> {
-            let mut bytes = map_data.read_u8()? as u32;
+            let bytes = map_data.read_u8()? as u32;
             let mut raw_chars = Vec::new();
             raw_chars.resize(bytes as usize, 0u8);
             map_data.read(&mut raw_chars)?;
-            let mut raw_chars = Cursor::new(raw_chars);
+            Ok(encoding_rs::SHIFT_JIS.decode_without_bom_handling(&raw_chars).0.into_owned())
-
-            let mut chars = Vec::new();
-            chars.reserve(bytes as usize);
-
-            while bytes > 0 {
-                let (consumed, chr) = read_cur_shift_jis(&mut raw_chars, bytes);
-                chars.push(chr);
-                bytes -= consumed;
-            }
-
-            Ok(chars.iter().collect())
         }
 
         fn skip_string<R: io::Read>(map_data: &mut R) -> GameResult {
@@ -549,7 +537,7 @@ impl WaterParams {
     }
 
     pub fn load_from<R: io::Read>(&mut self, data: R) -> GameResult {
-        fn next_u8<'a>(s: &mut impl Iterator<Item=&'a str>, error_msg: &str) -> GameResult<u8> {
+        fn next_u8<'a>(s: &mut impl Iterator<Item = &'a str>, error_msg: &str) -> GameResult<u8> {
             match s.next() {
                 None => Err(GameError::ParseError("Out of range.".to_string())),
                 Some(v) => v.trim().parse::<u8>().map_err(|_| GameError::ParseError(error_msg.to_string())),

src/game/scripting/tsc/bytecode_utils.rs

@@ -3,7 +3,6 @@ use std::io::{Cursor, Read};
 use crate::framework::error::GameError::ParseError;
 use crate::framework::error::GameResult;
 use crate::game::scripting::tsc::text_script::TextScriptEncoding;
-use crate::util::encoding::{read_cur_shift_jis, read_cur_wtf8};
 
 pub fn put_varint(val: i32, out: &mut Vec<u8>) {
     let mut x = ((val as u32) >> 31) ^ ((val as u32) << 1);
@@ -43,7 +42,7 @@ pub fn read_cur_varint(cursor: &mut Cursor<&[u8]>) -> GameResult<i32> {
 }
 
 #[allow(unused)]
-pub fn read_varint<I: Iterator<Item=u8>>(iter: &mut I) -> GameResult<i32> {
+pub fn read_varint<I: Iterator<Item = u8>>(iter: &mut I) -> GameResult<i32> {
     let mut result = 0u32;
 
     for o in 0..5 {
@@ -62,27 +61,21 @@ pub fn put_string(buffer: &mut Vec<u8>, out: &mut Vec<u8>, encoding: TextScriptE
     if buffer.is_empty() {
         return;
     }
+    let mut chars_count = 0;
 
-    let mut cursor: Cursor<&Vec<u8>> = Cursor::new(buffer);
     let mut tmp_buf = Vec::new();
-    let mut remaining = buffer.len() as u32;
-    let mut chars = 0;
 
-    while remaining > 0 {
+    let encoding: &encoding_rs::Encoding = encoding.into();
-        let (consumed, chr) = match encoding {
-            TextScriptEncoding::UTF8 => read_cur_wtf8(&mut cursor, remaining),
-            TextScriptEncoding::ShiftJIS => read_cur_shift_jis(&mut cursor, remaining),
-        };
 
-        remaining -= consumed;
+    let decoded_text = encoding.decode_without_bom_handling(&buffer).0;
-        chars += 1;
+    for chr in decoded_text.chars() {
-
+        chars_count += 1;
-        put_varint(chr as i32, &mut tmp_buf);
+        put_varint(chr as _, &mut tmp_buf);
     }
 
     buffer.clear();
 
-    put_varint(chars, out);
+    put_varint(chars_count, out);
     out.append(&mut tmp_buf);
 }
 

src/game/scripting/tsc/text_script.rs

@@ -44,23 +44,144 @@ bitfield! {
 }
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone)]
+#[allow(non_camel_case_types)]
 #[repr(u8)]
 pub enum TextScriptEncoding {
     UTF8 = 0,
     ShiftJIS,
+    UTF16BE,
+    UTF16LE,
+    ISO_2022_JP,
+    ISO_8859_2,
+    ISO_8859_3,
+    ISO_8859_4,
+    ISO_8859_5,
+    ISO_8859_6,
+    ISO_8859_7,
+    ISO_8859_8,
+    ISO_8859_8_I,
+    ISO_8859_10,
+    ISO_8859_13,
+    ISO_8859_14,
+    ISO_8859_15,
+    ISO_8859_16,
+    KOI8_R,
+    KOI8_U,
+    MACINTOSH,
+    EUC_JP,
+    EUC_KR,
+    GB18030,
+    GBK,
+    BIG5,
+    Win1250,
+    Win1251,
+    Win1252,
+    Win1253,
+    Win1254,
+    Win1255,
+    Win1256,
+    Win1257,
+    Win1258,
 }
 
 impl From<&str> for TextScriptEncoding {
     fn from(s: &str) -> Self {
         match s {
             "utf-8" => Self::UTF8,
+
+            "iso-2022-jp" => Self::ISO_2022_JP,
+            "iso-8859-2" => Self::ISO_8859_2,
+            "iso-8859-3" => Self::ISO_8859_3,
+            "iso-8859-4" => Self::ISO_8859_4,
+            "iso-8859-5" => Self::ISO_8859_5,
+            "iso-8859-6" => Self::ISO_8859_6,
+            "iso-8859-7" => Self::ISO_8859_7,
+            "iso-8859-8" => Self::ISO_8859_8,
+            "iso-8859-8-i" => Self::ISO_8859_8_I,
+            "iso-8859-10" => Self::ISO_8859_10,
+            "iso-8859-13" => Self::ISO_8859_13,
+            "iso-8859-14" => Self::ISO_8859_14,
+            "iso-8859-15" => Self::ISO_8859_15,
+            "iso-8859-16" => Self::ISO_8859_16,
+
+            "koi8-r" => Self::KOI8_R,
+            "koi8-u" => Self::KOI8_U,
+
+            "macintosh" => Self::MACINTOSH,
+
+            "euc-jp" => Self::EUC_JP,
+            "euc-kr" => Self::EUC_KR,
+
+            "gb18030" => Self::GB18030,
+            "gbk" => Self::GBK,
+            "big5" => Self::BIG5,
+
+            "windows-1250" => Self::Win1250,
+            "windows-1251" => Self::Win1251,
+            "windows-1252" => Self::Win1252,
+            "windows-1253" => Self::Win1253,
+            "windows-1254" => Self::Win1254,
+            "windows-1255" => Self::Win1255,
+            "windows-1256" => Self::Win1256,
+            "windows-1257" => Self::Win1257,
+            "windows-1258" => Self::Win1258,
+
+            "utf-16be" => Self::UTF16BE,
+            "utf-16le" => Self::UTF16LE,
+
             _ => Self::ShiftJIS,
         }
     }
 }
 
+impl From<TextScriptEncoding> for &'static encoding_rs::Encoding {
+    fn from(value: TextScriptEncoding) -> Self {
+        match value {
+            TextScriptEncoding::ShiftJIS => encoding_rs::SHIFT_JIS,
+            TextScriptEncoding::UTF8 => encoding_rs::UTF_8,
+            TextScriptEncoding::UTF16BE => encoding_rs::UTF_16BE,
+            TextScriptEncoding::UTF16LE => encoding_rs::UTF_16LE,
+            TextScriptEncoding::ISO_2022_JP => encoding_rs::ISO_2022_JP,
+            TextScriptEncoding::ISO_8859_2 => encoding_rs::ISO_8859_2,
+            TextScriptEncoding::ISO_8859_3 => encoding_rs::ISO_8859_3,
+            TextScriptEncoding::ISO_8859_4 => encoding_rs::ISO_8859_4,
+            TextScriptEncoding::ISO_8859_5 => encoding_rs::ISO_8859_5,
+            TextScriptEncoding::ISO_8859_6 => encoding_rs::ISO_8859_6,
+            TextScriptEncoding::ISO_8859_7 => encoding_rs::ISO_8859_7,
+            TextScriptEncoding::ISO_8859_8 => encoding_rs::ISO_8859_8,
+            TextScriptEncoding::ISO_8859_8_I => encoding_rs::ISO_8859_8_I,
+            TextScriptEncoding::ISO_8859_10 => encoding_rs::ISO_8859_10,
+            TextScriptEncoding::ISO_8859_13 => encoding_rs::ISO_8859_13,
+            TextScriptEncoding::ISO_8859_14 => encoding_rs::ISO_8859_14,
+            TextScriptEncoding::ISO_8859_15 => encoding_rs::ISO_8859_15,
+            TextScriptEncoding::ISO_8859_16 => encoding_rs::ISO_8859_16,
+            TextScriptEncoding::KOI8_R => encoding_rs::KOI8_R,
+            TextScriptEncoding::KOI8_U => encoding_rs::KOI8_U,
+            TextScriptEncoding::MACINTOSH => encoding_rs::MACINTOSH,
+            TextScriptEncoding::EUC_JP => encoding_rs::EUC_JP,
+            TextScriptEncoding::EUC_KR => encoding_rs::EUC_KR,
+            TextScriptEncoding::GB18030 => encoding_rs::GB18030,
+            TextScriptEncoding::GBK => encoding_rs::GBK,
+            TextScriptEncoding::BIG5 => encoding_rs::BIG5,
+            TextScriptEncoding::Win1250 => encoding_rs::WINDOWS_1250,
+            TextScriptEncoding::Win1251 => encoding_rs::WINDOWS_1251,
+            TextScriptEncoding::Win1252 => encoding_rs::WINDOWS_1252,
+            TextScriptEncoding::Win1253 => encoding_rs::WINDOWS_1253,
+            TextScriptEncoding::Win1254 => encoding_rs::WINDOWS_1254,
+            TextScriptEncoding::Win1255 => encoding_rs::WINDOWS_1255,
+            TextScriptEncoding::Win1256 => encoding_rs::WINDOWS_1256,
+            TextScriptEncoding::Win1257 => encoding_rs::WINDOWS_1257,
+            TextScriptEncoding::Win1258 => encoding_rs::WINDOWS_1258,
+        }
+    }
+}
+
 impl TextScriptEncoding {
     pub fn invalid_encoding(encoding: TextScriptEncoding, state: &SharedGameState) -> bool {
+        if state.loc.encoding.is_some_and(|e| e == encoding) {
+            return true;
+        }
+
         let required_encoding = if (state.loc.code == "jp" || state.loc.code == "en") && state.constants.is_base() {
             TextScriptEncoding::ShiftJIS
         } else {
@@ -798,8 +919,10 @@ impl TextScriptVM {
                         // The vanilla game treats this as a 1-byte value lol
                         //if npc.event_num == (new_direction & 0xFF) as u16 {
                         if npc.event_num == new_direction as u16 {
-                            game_scene.player1.direction = if game_scene.player1.x > npc.x { Direction::Left } else { Direction::Right };
+                            game_scene.player1.direction =
-                            game_scene.player2.direction = if game_scene.player2.x > npc.x { Direction::Left } else { Direction::Right };
+                                if game_scene.player1.x > npc.x { Direction::Left } else { Direction::Right };
+                            game_scene.player2.direction =
+                                if game_scene.player2.x > npc.x { Direction::Left } else { Direction::Right };
                         }
                     }
                 }

src/game/shared_game_state.rs

@@ -421,16 +421,7 @@ impl SharedGameState {
         constants.load_locales(ctx)?;
 
         let locale = SharedGameState::get_locale(&constants, &settings.locale).unwrap_or_default();
-        if (locale.code == "jp" || locale.code == "en") && constants.is_base() {
+        let font = Self::try_update_locale(&mut constants, &locale, ctx).unwrap();
-            constants.textscript.encoding = TextScriptEncoding::ShiftJIS
-        } else {
-            constants.textscript.encoding = TextScriptEncoding::UTF8
-        }
-
-        let font = BMFont::load(&constants.base_paths, &locale.font.path, ctx, locale.font.scale).or_else(|e| {
-            log::warn!("Failed to load font, using built-in: {}", e);
-            BMFont::load(&vec!["/".to_owned()], "builtin/builtin_font.fnt", ctx, 1.0)
-        })?;
 
         let mod_list = ModList::load(ctx, &constants.string_table)?;
 
@@ -522,6 +513,17 @@ impl SharedGameState {
         })
     }
 
+    pub fn reload_stage_table(&mut self, ctx: &mut Context) -> GameResult {
+        let stages = StageData::load_stage_table(
+            ctx,
+            &self.constants.base_paths,
+            self.constants.is_switch,
+            self.constants.stage_encoding,
+        )?;
+        self.stages = stages;
+        Ok(())
+    }
+
     pub fn reload_resources(&mut self, ctx: &mut Context) -> GameResult {
         self.constants.rebuild_path_list(self.mod_path.clone(), self.season, &self.settings);
         if !self.constants.is_demo {
@@ -531,8 +533,7 @@ impl SharedGameState {
         self.constants.load_csplus_tables(ctx)?;
         self.constants.load_animated_faces(ctx)?;
         self.constants.load_texture_size_hints(ctx)?;
-        let stages = StageData::load_stage_table(ctx, &self.constants.base_paths, self.constants.is_switch)?;
+        self.reload_stage_table(ctx)?;
-        self.stages = stages;
 
         let npc_tbl = filesystem::open_find(ctx, &self.constants.base_paths, "npc.tbl")?;
         let npc_table = NPCTable::load_from(npc_tbl)?;
@@ -571,24 +572,39 @@ impl SharedGameState {
         self.texture_set.unload_all();
     }
 
-    pub fn update_locale(&mut self, ctx: &mut Context) {
+    pub fn try_update_locale(
-        if let Some(locale) = SharedGameState::get_locale(&self.constants, &self.settings.locale) {
+        constants: &mut EngineConstants,
-            self.loc = locale;
+        locale: &Locale,
-            if (self.loc.code == "jp" || self.loc.code == "en") && self.constants.is_base() {
+        ctx: &mut Context,
-                self.constants.textscript.encoding = TextScriptEncoding::ShiftJIS
+    ) -> GameResult<BMFont> {
+        if let Some(encoding) = locale.encoding {
+            constants.textscript.encoding = encoding
         } else {
-                self.constants.textscript.encoding = TextScriptEncoding::UTF8
+            if (locale.code == "jp" || locale.code == "en") && constants.is_base() {
+                constants.textscript.encoding = TextScriptEncoding::ShiftJIS
+            } else {
+                constants.textscript.encoding = TextScriptEncoding::UTF8
             }
         }
 
-        let font = BMFont::load(&self.constants.base_paths, &self.loc.font.path, ctx, self.loc.font.scale)
+        constants.stage_encoding = locale.stage_encoding;
-            .or_else(|e| {
+
+        let font = BMFont::load(&constants.base_paths, &locale.font.path, ctx, locale.font.scale).or_else(|e| {
             log::warn!("Failed to load font, using built-in: {}", e);
             BMFont::load(&vec!["/".to_owned()], "builtin/builtin_font.fnt", ctx, 1.0)
-            })
+        })?;
-            .unwrap();
 
+        Ok(font)
+    }
+
+    pub fn update_locale(&mut self, ctx: &mut Context) {
+        let Some(locale) = SharedGameState::get_locale(&self.constants, &self.settings.locale) else {
+            return;
+        };
+        let font = Self::try_update_locale(&mut self.constants, &locale, ctx).unwrap();
+        self.loc = locale;
         self.font = font;
+        let _ = self.reload_stage_table(ctx);
     }
 
     pub fn graphics_reset(&mut self) {

src/game/stage.rs

@@ -1,19 +1,17 @@
 use std::io::{Cursor, Read};
 use std::str::from_utf8;
 
-use byteorder::LE;
 use byteorder::ReadBytesExt;
-use log::info;
+use byteorder::LE;
 
 use crate::common::Color;
 use crate::engine_constants::EngineConstants;
 use crate::framework::context::Context;
-use crate::framework::error::{GameError, GameResult};
 use crate::framework::error::GameError::ResourceLoadError;
+use crate::framework::error::{GameError, GameResult};
 use crate::framework::filesystem;
 use crate::game::map::{Map, NPCData};
-use crate::game::scripting::tsc::text_script::TextScript;
+use crate::game::scripting::tsc::text_script::{TextScript, TextScriptEncoding};
-use crate::util::encoding::read_cur_shift_jis;
 
 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
 pub struct NpcType {
@@ -225,21 +223,25 @@ fn zero_index(s: &[u8]) -> usize {
     s.iter().position(|&c| c == b'\0').unwrap_or(s.len())
 }
 
-fn from_shift_jis(s: &[u8]) -> String {
+fn from_encoding(s: &[u8], encoding: Option<TextScriptEncoding>) -> String {
-    let mut cursor = Cursor::new(s);
+    if let Some(encoding) = encoding {
-    let mut chars = Vec::new();
+        let encoding: &encoding_rs::Encoding = encoding.into();
-    let mut bytes = s.len() as u32;
+        return encoding.decode_without_bom_handling(s).0.into_owned();
-
-    while bytes > 0 {
-        let (consumed, chr) = read_cur_shift_jis(&mut cursor, bytes);
-        chars.push(chr);
-        bytes -= consumed;
     }
 
-    chars.iter().collect()
+    from_shift_jis(s)
 }
 
-fn from_csplus_stagetbl(s: &[u8], is_switch: bool) -> String {
+fn from_shift_jis(s: &[u8]) -> String {
+    encoding_rs::SHIFT_JIS.decode_without_bom_handling(s).0.into_owned()
+}
+
+fn from_csplus_stagetbl(s: &[u8], is_switch: bool, encoding: Option<TextScriptEncoding>) -> String {
+    if let Some(encoding) = encoding {
+        let encoding: &encoding_rs::Encoding = encoding.into();
+        return encoding.decode_without_bom_handling(s).0.into_owned();
+    }
+
     if is_switch {
         from_utf8(s).unwrap_or("").trim_matches('\0').to_string()
     } else {
@@ -248,7 +250,12 @@ fn from_csplus_stagetbl(s: &[u8], is_switch: bool) -> String {
 }
 
 impl StageData {
-    pub fn load_stage_table(ctx: &mut Context, roots: &Vec<String>, is_switch: bool) -> GameResult<Vec<Self>> {
+    pub fn load_stage_table(
+        ctx: &mut Context,
+        roots: &Vec<String>,
+        is_switch: bool,
+        encoding: Option<TextScriptEncoding>,
+    ) -> GameResult<Vec<Self>> {
         let stage_tbl_path = "/stage.tbl";
         let stage_sect_path = "/stage.sect";
         let mrmap_bin_path = "/mrmap.bin";
@@ -261,7 +268,7 @@ impl StageData {
 
             for path in roots.iter().rev() {
                 if let Ok(mut file) = filesystem::open(ctx, [path, stage_tbl_path].join("")) {
-                    info!("Loading Cave Story+ stage table from {}", &path);
+                    log::info!("Loading Cave Story+ stage table from {}", &path);
 
                     let mut new_stages = Vec::new();
 
@@ -289,13 +296,14 @@ impl StageData {
                         f.read_exact(&mut name_jap_buf)?;
                         f.read_exact(&mut name_buf)?;
 
-                        let tileset = from_csplus_stagetbl(&ts_buf[0..zero_index(&ts_buf)], is_switch);
+                        let tileset = from_csplus_stagetbl(&ts_buf[0..zero_index(&ts_buf)], is_switch, encoding);
-                        let map = from_csplus_stagetbl(&map_buf[0..zero_index(&map_buf)], is_switch);
+                        let map = from_csplus_stagetbl(&map_buf[0..zero_index(&map_buf)], is_switch, encoding);
-                        let background = from_csplus_stagetbl(&back_buf[0..zero_index(&back_buf)], is_switch);
+                        let background = from_csplus_stagetbl(&back_buf[0..zero_index(&back_buf)], is_switch, encoding);
-                        let npc1 = from_csplus_stagetbl(&npc1_buf[0..zero_index(&npc1_buf)], is_switch);
+                        let npc1 = from_csplus_stagetbl(&npc1_buf[0..zero_index(&npc1_buf)], is_switch, encoding);
-                        let npc2 = from_csplus_stagetbl(&npc2_buf[0..zero_index(&npc2_buf)], is_switch);
+                        let npc2 = from_csplus_stagetbl(&npc2_buf[0..zero_index(&npc2_buf)], is_switch, encoding);
-                        let name = from_csplus_stagetbl(&name_buf[0..zero_index(&name_buf)], is_switch);
+                        let name = from_csplus_stagetbl(&name_buf[0..zero_index(&name_buf)], is_switch, encoding);
-                        let name_jp = from_csplus_stagetbl(&name_jap_buf[0..zero_index(&name_jap_buf)], is_switch);
+                        let name_jp =
+                            from_csplus_stagetbl(&name_jap_buf[0..zero_index(&name_jap_buf)], is_switch, encoding);
 
                         let stage = StageData {
                             name: name.clone(),
@@ -326,7 +334,7 @@ impl StageData {
             // Cave Story freeware executable dump.
             let mut stages = Vec::new();
 
-            info!("Loading Cave Story freeware exe dump stage table from {}", &stage_sect_path);
+            log::info!("Loading Cave Story freeware exe dump stage table from {}", &stage_sect_path);
 
             let mut data = Vec::new();
             file.read_to_end(&mut data)?;
@@ -355,12 +363,12 @@ impl StageData {
                     let _ = f.read(&mut lol)?;
                 }
 
-                let tileset = from_shift_jis(&ts_buf[0..zero_index(&ts_buf)]);
+                let tileset = from_encoding(&ts_buf[0..zero_index(&ts_buf)], encoding);
-                let map = from_shift_jis(&map_buf[0..zero_index(&map_buf)]);
+                let map = from_encoding(&map_buf[0..zero_index(&map_buf)], encoding);
-                let background = from_shift_jis(&back_buf[0..zero_index(&back_buf)]);
+                let background = from_encoding(&back_buf[0..zero_index(&back_buf)], encoding);
-                let npc1 = from_shift_jis(&npc1_buf[0..zero_index(&npc1_buf)]);
+                let npc1 = from_encoding(&npc1_buf[0..zero_index(&npc1_buf)], encoding);
-                let npc2 = from_shift_jis(&npc2_buf[0..zero_index(&npc2_buf)]);
+                let npc2 = from_encoding(&npc2_buf[0..zero_index(&npc2_buf)], encoding);
-                let name = from_shift_jis(&name_buf[0..zero_index(&name_buf)]);
+                let name = from_encoding(&name_buf[0..zero_index(&name_buf)], encoding);
 
                 let stage = StageData {
                     name: name.clone(),
@@ -383,7 +391,7 @@ impl StageData {
             // Moustache Rider stage table
             let mut stages = Vec::new();
 
-            info!("Loading Moustache Rider stage table from {}", &mrmap_bin_path);
+            log::info!("Loading Moustache Rider stage table from {}", &mrmap_bin_path);
 
             let mut data = Vec::new();
 
@@ -414,12 +422,12 @@ impl StageData {
                 let boss_no = f.read_u8()?;
                 f.read_exact(&mut name_buf)?;
 
-                let tileset = from_shift_jis(&ts_buf[0..zero_index(&ts_buf)]);
+                let tileset = from_encoding(&ts_buf[0..zero_index(&ts_buf)], encoding);
-                let map = from_shift_jis(&map_buf[0..zero_index(&map_buf)]);
+                let map = from_encoding(&map_buf[0..zero_index(&map_buf)], encoding);
-                let background = from_shift_jis(&back_buf[0..zero_index(&back_buf)]);
+                let background = from_encoding(&back_buf[0..zero_index(&back_buf)], encoding);
-                let npc1 = from_shift_jis(&npc1_buf[0..zero_index(&npc1_buf)]);
+                let npc1 = from_encoding(&npc1_buf[0..zero_index(&npc1_buf)], encoding);
-                let npc2 = from_shift_jis(&npc2_buf[0..zero_index(&npc2_buf)]);
+                let npc2 = from_encoding(&npc2_buf[0..zero_index(&npc2_buf)], encoding);
-                let name = from_shift_jis(&name_buf[0..zero_index(&name_buf)]);
+                let name = from_encoding(&name_buf[0..zero_index(&name_buf)], encoding);
 
                 let stage = StageData {
                     name: name.clone(),
@@ -441,7 +449,7 @@ impl StageData {
         } else if let Ok(mut file) = filesystem::open_find(ctx, roots, stage_dat_path) {
             let mut stages = Vec::new();
 
-            info!("Loading NXEngine stage table from {}", &stage_dat_path);
+            log::info!("Loading NXEngine stage table from {}", &stage_dat_path);
 
             let mut data = Vec::new();
 

src/i18n.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 
 use crate::framework::context::Context;
 use crate::framework::filesystem;
+use crate::game::scripting::tsc::text_script::TextScriptEncoding;
 use crate::game::shared_game_state::FontData;
 
 #[derive(Debug, Clone)]
@@ -9,6 +10,8 @@ pub struct Locale {
     pub code: String,
     pub name: String,
     pub font: FontData,
+    pub encoding: Option<TextScriptEncoding>,
+    pub stage_encoding: Option<TextScriptEncoding>,
     strings: HashMap<String, String>,
 }
 
@@ -18,6 +21,8 @@ impl Default for Locale {
             code: "en".to_owned(),
             name: "English".to_owned(),
             font: FontData { path: String::new(), scale: 1.0, space_offset: 0.0 },
+            encoding: None,
+            stage_encoding: None,
             strings: HashMap::new(),
         }
     }
@@ -25,7 +30,7 @@ impl Default for Locale {
 
 impl Locale {
     pub fn new(ctx: &mut Context, base_paths: &Vec<String>, code: &str) -> Locale {
-        let file = filesystem::open_find(ctx, base_paths, &format!("locale/{}.json", code)).unwrap();
+        let file = filesystem::open_find(ctx, base_paths, &format!("locale/{code}.json")).unwrap();
         let json: serde_json::Value = serde_json::from_reader(file).unwrap();
 
         let strings = Locale::flatten(&json);
@@ -36,7 +41,18 @@ impl Locale {
         let font_scale = strings["font_scale"].parse::<f32>().unwrap_or(1.0);
         let font = FontData::new(font_name, font_scale, 0.0);
 
-        Locale { code: code.to_string(), name, font, strings }
+        let encoding = if let Some(enc) = strings.get("encoding").clone() {
+            Some(TextScriptEncoding::from(enc.as_str()))
+        } else {
+            None
+        };
+        let stage_encoding = if let Some(enc) = strings.get("stage_encoding").clone() {
+            Some(TextScriptEncoding::from(enc.as_str()))
+        } else {
+            None
+        };
+
+        Locale { code: code.to_string(), name, font, encoding, stage_encoding, strings }
     }
 
     fn flatten(json: &serde_json::Value) -> HashMap<String, String> {

src/util/mod.rs

@@ -1,4 +1,3 @@
 pub mod bitvec;
-pub mod encoding;
-pub mod rng;
 pub mod browser;
+pub mod rng;