Strip UTF-8 BOM in M2dReader and add test

Move UTF-8 BOM removal to the IO layer: M2dReader.GetString() now strips a leading U+FEFF if present. Remove the now-unused Sanitizer.RemoveUtf8Bom method and delete calls to it in several parsers (ItemOptionParser, TableParser). Add SanitizerBomTest to verify XML data with and without a BOM deserializes correctly. This centralizes BOM handling at read-time and removes duplicate sanitizer logic.

Author: AngeloTadeucci

Maple2.File.IO/M2dReader.cs

@@ -63,7 +63,12 @@ public byte[] GetBytes(PackFileEntry entry) {
 
         public string GetString(PackFileEntry entry) {
             byte[] data = CryptoManager.DecryptData(entry.FileHeader, m2dFile);
-            return Encoding.Default.GetString(data);
+            string result = Encoding.Default.GetString(data);
+            // Remove UTF-8 BOM if present
+            if (result.Length > 0 && result[0] == '\uFEFF') {
+                return result[1..];
+            }
+            return result;
         }
 
         public void Dispose() {

Maple2.File.Parser/ItemOptionParser.cs

@@ -131,7 +131,6 @@ public IEnumerable<ItemOptionConstantData> ParseConstant() {
 
     public IEnumerable<ItemOptionConstant> ParseConstantNew() {
         string xml = Sanitizer.RemoveEmpty(xmlReader.GetString(xmlReader.GetEntry("table/itemoptionconstant.xml")));
-        xml = Sanitizer.RemoveUtf8Bom(xml);
         var reader = XmlReader.Create(new StringReader(xml));
         var root = itemOptionConstantNewSerializer.Deserialize(reader) as ItemOptionConstantRootNew;
         Debug.Assert(root != null);
@@ -161,7 +160,6 @@ public IEnumerable<ItemOptionData> ParseRandom() {
 
     public IEnumerable<ItemOptionRandomNew> ParseRandomNew() {
         string xml = Sanitizer.RemoveEmpty(xmlReader.GetString(xmlReader.GetEntry("table/itemoptionrandom.xml")));
-        xml = Sanitizer.RemoveUtf8Bom(xml);
         var reader = XmlReader.Create(new StringReader(xml));
         var root = itemOptionNewSerializer.Deserialize(reader) as ItemOptionRandomRootNew;
         Debug.Assert(root != null);
@@ -192,7 +190,6 @@ public IEnumerable<ItemOptionData> ParseStatic() {
 
     public IEnumerable<MergeOptionNew> ParseMergeOptionBaseNew() {
         string xml = Sanitizer.RemoveEmpty(xmlReader.GetString(xmlReader.GetEntry("table/itemmergeoptionbase.xml")));
-        xml = Sanitizer.RemoveUtf8Bom(xml);
         var reader = XmlReader.Create(new StringReader(xml));
         var root = itemMergeOptionNewSerializer.Deserialize(reader) as ItemMergeOptionRootNew;
         Debug.Assert(root != null);

Maple2.File.Parser/TableParser.cs

@@ -815,7 +815,6 @@ public IEnumerable<JobTableNew> ParseJobTableNew() {
     public IEnumerable<(int Id, IDictionary<byte, List<IndividualItemDropItem>>)> ParseIndividualItemDropFinal() {
         string xml = Sanitizer.RemoveEmpty(xmlReader.GetString(xmlReader.GetEntry("table/individualitemdrop_final.xml")));
         xml = Sanitizer.SanitizeBool(xml);
-        xml = Sanitizer.RemoveUtf8Bom(xml);
         var reader = XmlReader.Create(new StringReader(xml));
         var data = individualItemDropNewSerializer.Deserialize(reader) as IndividualItemDropRootNew;
         Debug.Assert(data != null);
@@ -1000,7 +999,6 @@ public IEnumerable<JobTableNew> ParseJobTableNew() {
 
     public IEnumerable<(int ItemId, ShopFurnishing UgcItem)> ParseFurnishingShopUgcAll() {
         string xml = Sanitizer.RemoveEmpty(xmlReader.GetString(xmlReader.GetEntry($"table/{locale}/shop_ugcall.xml")));
-        xml = Sanitizer.RemoveUtf8Bom(xml);
 
         var reader = XmlReader.Create(new StringReader(xml));
         var data = shopFurnishingSerializer.Deserialize(reader) as ShopFurnishingRoot;

Maple2.File.Parser/Tools/Sanitizer.cs

@@ -95,11 +95,4 @@ private static string FixCommaFloats(string xml, params string[] attributes) {
         string pattern = $"({string.Join('|', attributes)})=\"(-?\\d+)(?:,(\\d+))\"";
         return Regex.Replace(xml, pattern, "$1=\"$2.$3\"");
     }
-
-    public static string RemoveUtf8Bom(string xml) {
-        if (!string.IsNullOrEmpty(xml) && xml[0] == '\uFEFF') {
-            return xml[1..];
-        }
-        return xml;
-    }
 }

Maple2.File.Tests/SanitizerBomTest.cs

@@ -0,0 +1,61 @@
+using System.Text;
+using System.Xml.Serialization;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Maple2.File.Tests;
+
+[TestClass]
+public class SanitizerBomTest {
+    private const string XML_CONTENT = "<?xml version=\"1.0\" encoding=\"utf-8\"?><ms2><item id=\"1\" name=\"TestItem\" /></ms2>";
+
+    [XmlRoot("ms2")]
+    public class Ms2Root {
+        [XmlElement("item")]
+        public ItemElement? Item { get; set; }
+    }
+
+    public class ItemElement {
+        [XmlAttribute("id")]
+        public int Id { get; set; }
+
+        [XmlAttribute("name")]
+        public string Name { get; set; } = string.Empty;
+    }
+
+    [TestMethod]
+    public void TestParseXmlWithoutBom() {
+        byte[] data = Encoding.UTF8.GetBytes(XML_CONTENT);
+        string xml = Encoding.Default.GetString(data);
+
+        var serializer = new XmlSerializer(typeof(Ms2Root));
+        using var reader = new StringReader(xml);
+        var result = (Ms2Root?) serializer.Deserialize(reader);
+
+        Assert.IsNotNull(result);
+        Assert.IsNotNull(result.Item);
+        Assert.AreEqual(1, result.Item.Id);
+        Assert.AreEqual("TestItem", result.Item.Name);
+    }
+
+    [TestMethod]
+    public void TestParseXmlWithBom() {
+        byte[] bom = [0xEF, 0xBB, 0xBF];
+        byte[] content = Encoding.UTF8.GetBytes(XML_CONTENT);
+        byte[] data = [..bom, ..content];
+
+        // Simulate M2dReader.GetString() which strips BOM
+        string xml = Encoding.Default.GetString(data);
+        if (xml.Length > 0 && xml[0] == '\uFEFF') {
+            xml = xml[1..];
+        }
+
+        var serializer = new XmlSerializer(typeof(Ms2Root));
+        using var reader = new StringReader(xml);
+        var result = (Ms2Root?) serializer.Deserialize(reader);
+
+        Assert.IsNotNull(result);
+        Assert.IsNotNull(result.Item);
+        Assert.AreEqual(1, result.Item.Id);
+        Assert.AreEqual("TestItem", result.Item.Name);
+    }
+}