[core] Support module patches for zone data

modules/README.md

@@ -29,6 +29,16 @@ Command modules register with `xi.module.registerCommand('name', commandTable)`
 
 Mutating game data at require time is forbidden. Data changes are declared as an override of `xi.server.onServerStart` that calls `super()` first and then applies the changes.
 
+## Zone data
+
+A module can patch a zone data file by mirroring its path under the module's `data` directory. For example,
+`modules/example/data/zones/west_ronfaure/mobs.yaml` patches `data/zones/west_ronfaure/mobs.yaml`.
+
+Nested data components may be selected directly. For example, an `init.txt` entry for
+`era/data/abyssea/advanced_job_quest` loads patches relative to that directory.
+
+Patches use RFC 7396 semantics and are applied in `init.txt` order. Objects merge by key, lists are replaced whole, and `null` removes a key.
+
 ## Era Accuracy Modules
 
 Lua era-accuracy modules live under `era/lua/` and mirror the main `scripts/` tree where practical.

modules/era/data/abyssea/advanced_job_quest/zones/ghelsba_outpost/mobs.yaml

@@ -0,0 +1,12 @@
+# DRG quest mob before the May 10, 2011 version update.
+# Source: https://forum.square-enix.com/ffxi/threads/7257
+
+templates:
+  Cyranuce_M_Cutauleon:
+    attributes:
+      stats:
+        hp: 2700
+
+spawns:
+  17350928:
+    level: [32, 32]

modules/era/data/abyssea/advanced_job_quest/zones/konschtat_highlands/mobs.yaml

@@ -0,0 +1,12 @@
+# SAM quest mob before the May 10, 2011 version update.
+# Source: https://forum.square-enix.com/ffxi/threads/7257
+
+templates:
+  Forger:
+    attributes:
+      stats:
+        hp: 2000
+
+spawns:
+  17219999:
+    level: [32, 32]

modules/era/data/abyssea/advanced_job_quest/zones/korroloka_tunnel/mobs.yaml

@@ -0,0 +1,16 @@
+# NIN quest mobs before the May 10, 2011 version update.
+# Source: https://forum.square-enix.com/ffxi/threads/7257
+
+templates:
+  Korroloka_Leech:
+    attributes:
+      stats:
+        hp: 900
+
+spawns:
+  17486187:
+    level: [32, 32]
+  17486188:
+    level: [32, 32]
+  17486189:
+    level: [32, 32]

modules/era/data/abyssea/advanced_job_quest/zones/the_sanctuary_of_zitah/mobs.yaml

@@ -0,0 +1,12 @@
+# SAM quest mob before the May 10, 2011 version update.
+# Source: https://forum.square-enix.com/ffxi/threads/7257
+
+templates:
+  Guardian_Treant:
+    attributes:
+      stats:
+        hp: 2000
+
+spawns:
+  17272838:
+    level: [32, 32]

src/map/data/loader.h

@@ -24,6 +24,7 @@
 #include "common/enum_traits.h"
 #include "common/logging.h"
 #include "data/enums/zone.h"
+#include "data/yaml/merge.h"
 
 #include <cstdlib>
 #include <exception>
@@ -33,16 +34,22 @@
 #include <iterator>
 #include <optional>
 #include <string>
+#include <string_view>
 
 namespace xi::data
 {
 
+inline auto zoneDataName(const xi::ZoneId zoneId, const std::string_view name) -> std::string
+{
+    return fmt::format("zones/{}/{}", EnumTraits<xi::ZoneId>::toName(zoneId), name);
+}
+
 inline auto zoneFilePath(const xi::ZoneId zoneId, const std::string_view name) -> std::string
 {
-    return fmt::format("data/zones/{}/{}.yaml", EnumTraits<xi::ZoneId>::toName(zoneId), name);
+    return fmt::format("data/{}.yaml", zoneDataName(zoneId, name));
 }
 
-// Per-zone data file. No file means the zone declares none of this kind.
+// Per-zone data file. No core file means the zone declares none of this kind.
 template <class Dataset>
 auto loadZoneFile(const xi::ZoneId zoneId) -> std::optional<typename Dataset::Records>
 {
@@ -52,10 +59,20 @@ auto loadZoneFile(const xi::ZoneId zoneId) -> std::optional<typename Dataset::Re
         return std::nullopt;
     }
 
+    const auto modules = getDataModulePaths(zoneDataName(zoneId, Dataset::kDataPath), ".yaml");
+
     try
     {
-        std::ifstream     input(path, std::ios::binary);
-        const std::string text{ std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>() };
+        const auto text = [&]() -> std::string
+        {
+            if (!modules.empty())
+            {
+                return loadPatchedZoneYaml(path, modules);
+            }
+
+            std::ifstream input(path, std::ios::binary);
+            return { std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>() };
+        }();
 
         auto records = Dataset::decode(text);
         if constexpr (requires { Dataset::verifyZone(records, zoneId); })

src/map/data/yaml/merge.cpp

@@ -24,10 +24,14 @@
 #include <glaze/json/patch.hpp>
 #include <glaze/yaml.hpp>
 
+#include <algorithm>
+#include <filesystem>
+#include <fmt/format.h>
 #include <fstream>
 #include <sstream>
 #include <stdexcept>
 #include <string>
+#include <unordered_set>
 #include <vector>
 
 namespace xi::data
@@ -36,6 +40,18 @@ namespace xi::data
 namespace
 {
 
+auto trimLine(const std::string_view line) -> std::string_view
+{
+    const auto first = line.find_first_not_of(" \t\r\n");
+    if (first == std::string_view::npos)
+    {
+        return {};
+    }
+
+    const auto last = line.find_last_not_of(" \t\r\n");
+    return line.substr(first, last - first + 1);
+}
+
 auto readDocument(const std::string_view text) -> glz::generic_u64
 {
     glz::generic_u64 document;
@@ -47,6 +63,21 @@ auto readDocument(const std::string_view text) -> glz::generic_u64
     return document;
 }
 
+auto applyPatches(const std::string_view core, const std::span<const std::string> modules) -> glz::generic_u64
+{
+    auto document = readDocument(core);
+    for (const auto& module : modules)
+    {
+        const auto patch = readDocument(module);
+        if (const auto error = glz::merge_patch(document, patch))
+        {
+            throw std::runtime_error(glz::format_error(error));
+        }
+    }
+
+    return document;
+}
+
 auto slurp(const std::string_view path) -> std::string
 {
     const std::ifstream input(std::string{ path }, std::ios::binary);
@@ -62,24 +93,60 @@ auto slurp(const std::string_view path) -> std::string
 
 } // namespace
 
-auto mergeYaml(const std::string_view core, const std::span<const std::string> modules) -> std::string
+auto getDataModulePaths(const std::string_view name, const std::string_view extension) -> std::vector<std::string>
 {
-    if (modules.empty())
+    std::vector<std::string> modules;
+    std::ifstream            file("./modules/init.txt", std::ios_base::in);
+    if (!file)
     {
-        return std::string{ core };
+        return modules;
     }
 
-    auto document = readDocument(core);
-    for (const auto& module : modules)
+    std::unordered_set<std::string> seenPaths;
+    std::string                     line;
+    while (std::getline(file, line))
     {
-        const auto patch = readDocument(module);
-        if (const auto error = glz::merge_patch(document, patch))
+        const auto trimmed = trimLine(line);
+        if (trimmed.empty() || trimmed[0] == '#')
         {
-            throw std::runtime_error(glz::format_error(error));
+            continue;
+        }
+
+        const auto entry            = std::filesystem::path{ std::string{ trimmed } };
+        const auto explicitDataRoot = std::ranges::any_of(entry, [](const auto& component)
+                                                          {
+                                                              return component == "data";
+                                                          });
+
+        auto dataRoot = std::filesystem::path{ "./modules" };
+        if (explicitDataRoot)
+        {
+            dataRoot /= entry;
         }
+        else
+        {
+            dataRoot /= *entry.begin();
+            dataRoot /= "data";
+        }
+
+        const auto modulePath = (dataRoot / fmt::format("{}{}", name, extension)).generic_string();
+        if (seenPaths.insert(modulePath).second && std::filesystem::exists(modulePath))
+        {
+            modules.emplace_back(modulePath);
+        }
+    }
+
+    return modules;
+}
+
+auto mergeYaml(const std::string_view core, const std::span<const std::string> modules) -> std::string
+{
+    if (modules.empty())
+    {
+        return std::string{ core };
     }
 
-    auto output = glz::write_yaml(document);
+    auto output = glz::write_yaml(applyPatches(core, modules));
     if (!output)
     {
         throw std::runtime_error("Glaze could not serialize patched YAML");
@@ -105,4 +172,28 @@ auto loadMergedYaml(const std::string_view corePath, const std::span<const std::
     return mergeYaml(core, modules);
 }
 
+auto patchZoneYaml(const std::string_view core, const std::span<const std::string> modules) -> std::string
+{
+    auto output = glz::write_json(applyPatches(core, modules));
+    if (!output)
+    {
+        throw std::runtime_error("Could not serialize patched zone YAML");
+    }
+
+    return *output;
+}
+
+auto loadPatchedZoneYaml(const std::string_view corePath, const std::span<const std::string> modulePaths) -> std::string
+{
+    const auto               core = slurp(corePath);
+    std::vector<std::string> modules;
+    modules.reserve(modulePaths.size());
+    for (const auto& modulePath : modulePaths)
+    {
+        modules.emplace_back(slurp(modulePath));
+    }
+
+    return patchZoneYaml(core, modules);
+}
+
 } // namespace xi::data

src/map/data/yaml/merge.h

@@ -24,13 +24,19 @@
 #include <span>
 #include <string>
 #include <string_view>
+#include <vector>
 
 namespace xi::data
 {
 
 // Apply module YAML as RFC 7386 merge patches over the core document.
 auto mergeYaml(std::string_view core, std::span<const std::string> modules) -> std::string;
 
+auto getDataModulePaths(std::string_view name, std::string_view extension) -> std::vector<std::string>;
 auto loadMergedYaml(std::string_view corePath, std::span<const std::string> modulePaths) -> std::string;
 
+// Zone patches use JSON as private transport because Glaze's generic YAML tree cannot round-trip numeric mapping keys back into typed YAML maps.
+auto patchZoneYaml(std::string_view core, std::span<const std::string> modules) -> std::string;
+auto loadPatchedZoneYaml(std::string_view corePath, std::span<const std::string> modulePaths) -> std::string;
+
 } // namespace xi::data

src/map/data/yaml/read.h

@@ -21,6 +21,7 @@
 
 #pragma once
 
+#include <glaze/glaze.hpp>
 #include <glaze/yaml.hpp>
 
 #include <stdexcept>
@@ -36,13 +37,15 @@ inline constexpr glz::opts kStrictYaml{
 
 // Decode one YAML document with strict field checks.
 template <class T>
-auto read(const std::string_view yaml) -> T
+auto read(const std::string_view text) -> T
 {
     T          value{};
-    const auto error = glz::read_yaml<kStrictYaml>(value, yaml);
+    const auto first = text.find_first_not_of(" \t\r\n");
+    const auto json  = first != std::string_view::npos && (text[first] == '{' || text[first] == '[');
+    const auto error = json ? glz::read<kStrictYaml>(value, text) : glz::read_yaml<kStrictYaml>(value, text);
     if (error)
     {
-        throw std::runtime_error(glz::format_error(error, yaml));
+        throw std::runtime_error(glz::format_error(error, text));
     }
 
     return value;

src/map/utils/dataset_loader.h

@@ -21,12 +21,9 @@
 
 #pragma once
 
-// Datasets that merge module YAML over a core file. Kept out of loader.h because moduleutils pulls in the map's Lua layer.
-
 #include "common/logging.h"
 #include "data/loader.h"
 #include "data/yaml/merge.h"
-#include "utils/moduleutils.h"
 
 #include <chrono>
 #include <exception>
@@ -42,7 +39,7 @@ auto loadDataset() -> typename Dataset::Records
     const auto dataPath = Dataset::kDataPath;
     const auto start    = std::chrono::steady_clock::now();
     const auto corePath = fmt::format("data/{}.yaml", dataPath);
-    const auto modules  = moduleutils::GetDataModules(dataPath, ".yaml");
+    const auto modules  = getDataModulePaths(dataPath, ".yaml");
 
     try
     {