diff --git a/.gitignore b/.gitignore index 14083e8a78..66936d905f 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,7 @@ mop.sln graphify-out .claude CLAUDE.md +tools/db2tool/listfile.csv +tools/db2tool/DBDCache/ +tools/db2tool/dbfilesclient/ +tools/db2tool/caches/ diff --git a/docs/commands.md b/docs/commands.md index 864d1f99fc..742274807e 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -70,7 +70,7 @@ make wowsimmop make simdb # Generate data from WoW client files -# Requires dotnet 9 to run +# Requires a local WoW install; point Settings.BaseDir in the settings file at it # Uses tools/database/generator-settings.json for settings # Also runs make simdb # This is what you will use most of the time for generation diff --git a/makefile b/makefile index 97f08ff92d..c28d625d0f 100644 --- a/makefile +++ b/makefile @@ -248,15 +248,15 @@ CLIENTDATA_OUTPUT := $(shell realpath ./tools/database/wowsims.db) .PHONY: db db: - @echo "Running DB2ToSqlite for clientdata" - cd tools/DB2ToSqlite && dotnet run -- -s $(CLIENTDATA_SETTINGS) --output $(CLIENTDATA_OUTPUT) + @echo "Extracting client data" + go run ./tools/db2tool -s $(CLIENTDATA_SETTINGS) --output $(CLIENTDATA_OUTPUT) @echo "Running DBC generation tool" go run tools/database/gen_db/*.go -outDir=./assets -gen=db .PHONY: ptrdb ptrdb: - @echo "Running DB2ToSqlite for clientdata" - cd tools/DB2ToSqlite && dotnet run -- -s $(CLIENTDATAPTR_SETTINGS) --output $(CLIENTDATA_OUTPUT) + @echo "Extracting client data" + go run ./tools/db2tool -s $(CLIENTDATAPTR_SETTINGS) --output $(CLIENTDATA_OUTPUT) @echo "Running DBC generation tool" go run tools/database/gen_db/*.go -outDir=./assets -gen=db diff --git a/sim/common/mop/enchants_auto_gen.go b/sim/common/mop/enchants_auto_gen.go index d317bb35aa..3a01d4d62a 100644 --- a/sim/common/mop/enchants_auto_gen.go +++ b/sim/common/mop/enchants_auto_gen.go @@ -1,17 +1,17 @@ package mop import ( + "github.com/wowsims/mop/sim/common/shared" "github.com/wowsims/mop/sim/core" - "github.com/wowsims/mop/sim/common/shared" ) func RegisterAllEnchants() { // Enchants - + // Permanently attaches Lord Blastington's special scope to a ranged weapon, sometimes increasing Agility // by 1800 for 10s when dealing damage with ranged attacks. - // + // // Attaching this scope to a ranged weapon causes it to become soulbound. shared.NewProcStatBonusEffect(shared.ProcStatBonusEffect{ Name: "Lord Blastington's Scope of Doom", @@ -21,10 +21,10 @@ func RegisterAllEnchants() { Outcome: core.OutcomeLanded, RequireDamageDealt: true, }) - + // Permanently attaches a mirrored scope to a ranged weapon, sometimes increases critical strike by 900 for // 10s when dealing damage with ranged attacks. - // + // // Attaching this scope to a ranged weapon causes it to become soulbound. shared.NewProcStatBonusEffect(shared.ProcStatBonusEffect{ Name: "Mirror Scope", @@ -34,10 +34,10 @@ func RegisterAllEnchants() { Outcome: core.OutcomeLanded, RequireDamageDealt: true, }) - + // Embroiders a subtle pattern of light into your cloak, giving you a chance to increase your Intellect by // 2000 for 15s when casting a spell. - // + // // Embroidering your cloak will cause it to become soulbound and requires the Tailoring profession to remain // active. shared.NewProcStatBonusEffect(shared.ProcStatBonusEffect{ @@ -48,10 +48,10 @@ func RegisterAllEnchants() { Outcome: core.OutcomeLanded, RequireDamageDealt: false, }) - + // Embroiders a magical pattern into your cloak, giving you a chance to increase your Spirit by 3000 for // 15s when you cast a spell. - // + // // Embroidering your cloak will cause it to become soulbound and requires the Tailoring profession to remain // active. shared.NewProcStatBonusEffect(shared.ProcStatBonusEffect{ @@ -62,10 +62,10 @@ func RegisterAllEnchants() { Outcome: core.OutcomeLanded, RequireDamageDealt: false, }) - + // Embroiders a magical pattern into your cloak, causing your damaging melee and ranged attacks to sometimes // increase your attack power by 4000 for 15s. - // + // // Embroidering your cloak will cause it to become soulbound and requires the Tailoring profession to remain // active. shared.NewProcStatBonusEffect(shared.ProcStatBonusEffect{ @@ -76,4 +76,4 @@ func RegisterAllEnchants() { Outcome: core.OutcomeLanded, RequireDamageDealt: true, }) -} \ No newline at end of file +} diff --git a/sim/common/mop/stat_bonus_procs_auto_gen.go b/sim/common/mop/stat_bonus_procs_auto_gen.go index 095e68d410..0369ac879e 100644 --- a/sim/common/mop/stat_bonus_procs_auto_gen.go +++ b/sim/common/mop/stat_bonus_procs_auto_gen.go @@ -1417,9 +1417,9 @@ func RegisterAllProcs() { // This can be ignored if the effect has already been implemented. // With next db run the item will be removed if implemented. // - // Your helpful spells have a chance to grant you a Blessing of Zuldazar, which stacks up to 6 times. (Approximately - // 2.89 procs per minute) - // https://www.wowhead.com/mop/spell=138967 + // Consumes all Blessings of Zuldazar to shield the target, absorbing 1000 damage per Blessing consumed. + // Lasts 15s. + // https://www.wowhead.com/mop/spell=138925 // shared.NewProcStatBonusEffectWithVariants(shared.ProcStatBonusEffect{ // Callback: core.CallbackEmpty, // ProcMask: core.ProcMaskUnknown, @@ -1437,9 +1437,9 @@ func RegisterAllProcs() { // This can be ignored if the effect has already been implemented. // With next db run the item will be removed if implemented. // - // Consumes all Blessings of Zuldazar to shield the target, absorbing 1000 damage per Blessing consumed. - // Lasts 15s. - // https://www.wowhead.com/mop/spell=138925 + // Your helpful spells have a chance to grant you a Blessing of Zuldazar, which stacks up to 6 times. (Approximately + // 2.89 procs per minute) + // https://www.wowhead.com/mop/spell=138967 // shared.NewProcStatBonusEffectWithVariants(shared.ProcStatBonusEffect{ // Callback: core.CallbackEmpty, // ProcMask: core.ProcMaskUnknown, diff --git a/tools/DB2ToSqlite/.gitignore b/tools/DB2ToSqlite/.gitignore deleted file mode 100644 index 04e6b849d9..0000000000 --- a/tools/DB2ToSqlite/.gitignore +++ /dev/null @@ -1,422 +0,0 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.tlog -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files -*.ncb -*.aps - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider -*.sln.iml - -TactKey.csv - -# Other stuff -dbcs -cache -caches -temp -manifests -knownPushIDs.json -*.csv -*.csv.bak -*.txt -versionHistory.json -WTL.db -WTL.db-journal -extract -dump -db.db -listfile.csv -dbfilesclient -DBDCache -wowsims.db -.idea \ No newline at end of file diff --git a/tools/DB2ToSqlite/.vscode/launch.json b/tools/DB2ToSqlite/.vscode/launch.json deleted file mode 100644 index 9dae3d4450..0000000000 --- a/tools/DB2ToSqlite/.vscode/launch.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - - - ] -} \ No newline at end of file diff --git a/tools/DB2ToSqlite/DB2ToSqliteTool.csproj b/tools/DB2ToSqlite/DB2ToSqliteTool.csproj deleted file mode 100644 index aea61d1072..0000000000 --- a/tools/DB2ToSqlite/DB2ToSqliteTool.csproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - net9.0 - enable - enable - - - - - .\references\DBDC.IO.dll - - - \references\DBCD.dll - - - \references\TACTSharp.dll - - - \references\DBDefsLib.dll - - - - - - - - diff --git a/tools/DB2ToSqlite/Helpers/BindableSettings.cs b/tools/DB2ToSqlite/Helpers/BindableSettings.cs deleted file mode 100644 index f2a1b835f6..0000000000 --- a/tools/DB2ToSqlite/Helpers/BindableSettings.cs +++ /dev/null @@ -1,69 +0,0 @@ -using TACTSharp; - -namespace DB2ToSqliteTool.Helpers; - -public class BindableSettings : Settings -{ - public new string Region - { - get => base.Region; - set => base.Region = value; - } - - public new string Product - { - get => base.Product; - set => base.Product = value; - } - - public new RootInstance.LocaleFlags Locale - { - get => base.Locale; - set => base.Locale = value; - } - - public new RootInstance.LoadMode RootMode - { - get => base.RootMode; - set => base.RootMode = value; - } - - public new string? BaseDir - { - get => base.BaseDir; - set => base.BaseDir = value; - } - - public new string? BuildConfig - { - get => base.BuildConfig; - set => base.BuildConfig = value; - } - - public new string? CDNConfig - { - get => base.CDNConfig; - set => base.CDNConfig = value; - } - - public new string CacheDir - { - get => base.CacheDir; - set => base.CacheDir = value; - } - - public new bool ListfileFallback - { - get => base.ListfileFallback; - set => base.ListfileFallback = value; - } - - public new string ListfileURL - { - get => base.ListfileURL; - set => base.ListfileURL = value; - } - - public List GameTables { get; set; } = []; - public string GameTablesOutDirectory { get; set; } = ""; -} diff --git a/tools/DB2ToSqlite/Helpers/DBCacheParser.cs b/tools/DB2ToSqlite/Helpers/DBCacheParser.cs deleted file mode 100644 index 49ebfbcfb0..0000000000 --- a/tools/DB2ToSqlite/Helpers/DBCacheParser.cs +++ /dev/null @@ -1,51 +0,0 @@ -namespace DB2ToSqliteTool.Helpers; - -public struct HotfixEntry -{ - public uint regionID; - public int pushID; - public uint uniqueID; - public uint tableHash; - public uint recordID; - public int dataSize; - public byte status; - public byte[] data; -} - -//https://github.com/Marlamin/wow.tools.local/blob/main/Services/DBCacheParser.cs -public class DBCacheParser -{ - public int build; - public List hotfixes = []; - - public DBCacheParser(string filename) - { - using (var fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) - using (var bin = new BinaryReader(fs)) - { - var hotfix = new HotfixEntry(); - bin.ReadUInt32(); // Signature - var version = bin.ReadUInt32(); - if (version != 9) - //Console.WriteLine("Unsupported DBCache version " + version + ", skipping"); - return; - build = bin.ReadInt32(); - bin.BaseStream.Position += 32; - - while (bin.BaseStream.Position < bin.BaseStream.Length) - { - bin.ReadUInt32(); // Signature - hotfix.regionID = bin.ReadUInt32(); - hotfix.pushID = bin.ReadInt32(); - hotfix.uniqueID = bin.ReadUInt32(); - hotfix.tableHash = bin.ReadUInt32(); - hotfix.recordID = bin.ReadUInt32(); - hotfix.dataSize = bin.ReadInt32(); - hotfix.status = bin.ReadByte(); - bin.ReadBytes(3); - hotfix.data = bin.ReadBytes(hotfix.dataSize); - hotfixes.Add(hotfix); - } - } - } -} diff --git a/tools/DB2ToSqlite/Helpers/HotfixManager.cs b/tools/DB2ToSqlite/Helpers/HotfixManager.cs deleted file mode 100644 index d6996ad43b..0000000000 --- a/tools/DB2ToSqlite/Helpers/HotfixManager.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System.Text.Json; -using DBCD.IO; - -//https://github.com/Marlamin/wow.tools.local/blob/main/Services/HotfixManager.cs -namespace DB2ToSqliteTool.Helpers; - -public static class HotfixManager -{ - public static Dictionary HotfixReaders = []; - public static Dictionary> DbcacheParsers = []; - public static Dictionary? PushIdDetected = []; - - public static Dictionary TableNames = Directory.EnumerateFiles("DBDCache/") - .ToDictionary(x => Hash(Path.GetFileNameWithoutExtension(x).ToUpper()), - x => Path.GetFileNameWithoutExtension(x)); - - private static void LoadPushIDs() - { - PushIdDetected = - JsonSerializer.Deserialize>(File.ReadAllText("knownPushIDs.json")); - } - - private static void SavePushIDs() - { - File.WriteAllText("knownPushIDs.json", JsonSerializer.Serialize(PushIdDetected)); - } - - public static void LoadCaches(string wowLocation) - { - if (!File.Exists("knownPushIDs.json")) - SavePushIDs(); - - LoadPushIDs(); - - Console.WriteLine("Reloading all hotfixes.."); - HotfixReaders.Clear(); - - if (Directory.Exists("caches")) - foreach (var file in Directory.GetFiles("caches", "*.bin", SearchOption.AllDirectories)) - { - var reader = new HotfixReader(file); - if (!HotfixReaders.ContainsKey((uint)reader.BuildId)) - HotfixReaders.Add((uint)reader.BuildId, reader); - - HotfixReaders[(uint)reader.BuildId].CombineCache(file); - - if (!DbcacheParsers.ContainsKey((uint)reader.BuildId)) - DbcacheParsers.Add((uint)reader.BuildId, []); - - var newCache = new DBCacheParser(file); - DbcacheParsers[(uint)reader.BuildId].Add(newCache); - - var newPushIDs = newCache.hotfixes.Where(x => x.pushID > 0 && PushIdDetected != null && !PushIdDetected.ContainsKey(x.pushID)) - .Select(x => x.pushID).ToList(); - foreach (var newPushId in newPushIDs) - { - PushIdDetected?.TryAdd(newPushId, DateTime.Now); - Console.WriteLine("Detected new pushID " + newPushId + " at " + DateTime.Now.ToShortTimeString()); - } - - Console.WriteLine("Loaded hotfixes from caches directory for build " + reader.BuildId); - } - - - foreach (var file in Directory.GetFiles(wowLocation, "DBCache.bin", SearchOption.AllDirectories)) - { - var reader = new HotfixReader(file); - if (!HotfixReaders.ContainsKey((uint)reader.BuildId)) - HotfixReaders.Add((uint)reader.BuildId, reader); - - HotfixReaders[(uint)reader.BuildId].CombineCache(file); - - if (!DbcacheParsers.ContainsKey((uint)reader.BuildId)) - DbcacheParsers.Add((uint)reader.BuildId, []); - - var newCache = new DBCacheParser(file); - DbcacheParsers[(uint)reader.BuildId].Add(newCache); - - var newPushIDs = newCache.hotfixes.Where(x => x.pushID > 0 && PushIdDetected != null && !PushIdDetected.ContainsKey(x.pushID)) - .Select(x => x.pushID).ToList(); - foreach (var newPushId in newPushIDs) - { - PushIdDetected?.TryAdd(newPushId, DateTime.Now); - Console.WriteLine("Detected new pushID " + newPushId + " at " + DateTime.Now.ToShortTimeString()); - } - - Console.WriteLine("Loaded hotfixes from client for build " + reader.BuildId); - } - - SavePushIDs(); - } - - public static void Clear() - { - HotfixReaders.Clear(); - DbcacheParsers.Clear(); - } - - private static uint Hash(string s) - { - var sHashtable = new uint[] - { - 0x486E26EE, 0xDCAA16B3, 0xE1918EEF, 0x202DAFDB, - 0x341C7DC7, 0x1C365303, 0x40EF2D37, 0x65FD5E49, - 0xD6057177, 0x904ECE93, 0x1C38024F, 0x98FD323B, - 0xE3061AE7, 0xA39B0FA1, 0x9797F25F, 0xE4444563 - }; - - uint v = 0x7fed7fed; - var x = 0xeeeeeeee; - for (var i = 0; i < s.Length; i++) - { - var c = (byte)s[i]; - v += x; - v ^= sHashtable[(c >> 4) & 0xf] - sHashtable[c & 0xf]; - x = x * 33 + v + c + 3; - } - - return v; - } -} diff --git a/tools/DB2ToSqlite/Helpers/SQLiteDbCreator.cs b/tools/DB2ToSqlite/Helpers/SQLiteDbCreator.cs deleted file mode 100644 index 17409f4cd4..0000000000 --- a/tools/DB2ToSqlite/Helpers/SQLiteDbCreator.cs +++ /dev/null @@ -1,120 +0,0 @@ -using DBDefsLib; -using Microsoft.Data.Sqlite; - -namespace DB2ToSqliteTool.Helpers; - -public static class SqliteDbCreator -{ - public static void CreateDatabaseWithDefinitions(Dictionary definitions, - string sqliteFilePath, uint build) - { - if (File.Exists(sqliteFilePath)) File.Delete(sqliteFilePath); - - var connectionString = new SqliteConnectionStringBuilder - { - DataSource = sqliteFilePath, - Mode = SqliteOpenMode.ReadWriteCreate - }.ToString(); - - using var connection = new SqliteConnection(connectionString); - connection.Open(); - - using (var pragmaCommand = connection.CreateCommand()) - { - pragmaCommand.CommandText = "PRAGMA foreign_keys = ON;"; - pragmaCommand.ExecuteNonQuery(); - } - - using (var transaction = connection.BeginTransaction()) - { - foreach (var kvp in definitions) - { - var tableName = kvp.Key; - var dbDef = kvp.Value; - - var versionDef = dbDef.versionDefinitions.LastOrDefault(x => x.builds.Any(y => y.build == build)); - - var columnDefinitionsSql = new List(); - var foreignKeysSql = new List(); - - foreach (var def in versionDef.definitions) - { - if (!dbDef.columnDefinitions.TryGetValue(def.name, out var colDef)) - throw new Exception($"Column definition for {def.name} not found in table {tableName}"); - - if (def.arrLength == 0) - { - var sqliteType = MapToSqLiteType(colDef.type); - var nullability = !string.IsNullOrEmpty(colDef.foreignTable) && - !string.IsNullOrEmpty(colDef.foreignColumn) && - !def.isID - ? " NULL" - : ""; - var columnSql = $"[{def.name}] {sqliteType}{nullability}"; - if (def.isID) columnSql += " PRIMARY KEY"; - columnDefinitionsSql.Add(columnSql); - - if (!string.IsNullOrEmpty(colDef.foreignTable) && - !string.IsNullOrEmpty(colDef.foreignColumn)) - foreignKeysSql.Add( - $"CREATE INDEX IF NOT EXISTS IX_{tableName}_{def.name} ON [{tableName}] ([{def.name}])"); - } - else - { - var mainColumnSql = $"[{def.name}] TEXT"; - - columnDefinitionsSql.Add(mainColumnSql); - - // (For example, an "int" or "uint" becomes INTEGER, "float" becomes REAL, etc.) - var elementType = MapToSqLiteType(colDef.type); - - // Create a generated column for each array index. - for (var i = 0; i < def.arrLength; i++) - { - // Generated column syntax: - // [ColumnName_i] GENERATED ALWAYS AS (json_extract([ColumnName], '$[i]')) VIRTUAL - var genColumn = - $"[{def.name}_{i}] {elementType} GENERATED ALWAYS AS (json_extract([{def.name}], '$[{i}]')) VIRTUAL"; - columnDefinitionsSql.Add(genColumn); - } - } - } - - var allColumns = new List(columnDefinitionsSql); - - - var createTableSql = $"CREATE TABLE IF NOT EXISTS [{tableName}] ({string.Join(", ", allColumns)});"; - - using (var command = connection.CreateCommand()) - { - command.CommandText = createTableSql; - command.Transaction = transaction; - command.ExecuteNonQuery(); - } - - foreach (var indexSql in foreignKeysSql) - { - using var cmd = connection.CreateCommand(); - cmd.CommandText = indexSql; - cmd.Transaction = transaction; - cmd.ExecuteNonQuery(); - } - } - - transaction.Commit(); - } - - connection.Close(); - } - - private static string MapToSqLiteType(string type) - { - return type switch - { - "int" or "uint" => "INTEGER", - "float" => "REAL", - "string" or "locstring" => "TEXT", - _ => throw new Exception("Unsupported type: " + type) - }; - } -} diff --git a/tools/DB2ToSqlite/Helpers/SqliteDataInserter.cs b/tools/DB2ToSqlite/Helpers/SqliteDataInserter.cs deleted file mode 100644 index 25e438c67a..0000000000 --- a/tools/DB2ToSqlite/Helpers/SqliteDataInserter.cs +++ /dev/null @@ -1,104 +0,0 @@ -using System.Text.Json; -using DBCD; -using DBDefsLib; -using Microsoft.Data.Sqlite; - -namespace DB2ToSqliteTool.Helpers; - -public static class SqliteDataInserter -{ - public static void InsertRows(IDBCDStorage storage, string tableName, Structs.DBDefinition dbDef, - SqliteConnection connection, uint build) - { - var versionDef = dbDef.versionDefinitions.LastOrDefault(x => x.builds.Any(y => y.build == build)); - - var columnNames = versionDef.definitions.Select(def => def.name).ToList(); - - - var pkDefinition = versionDef.definitions.FirstOrDefault(def => def.isID); - - var pkColumn = pkDefinition.name; - - - var columnsPart = string.Join(", ", columnNames.Select(c => $"[{c}]")); - var valuesPart = string.Join(", ", columnNames.Select(c => "@" + c)); - - - var updateColumns = columnNames.Where(c => c != pkColumn).ToList(); - var updateClause = updateColumns.Any() - ? "DO UPDATE SET " + string.Join(", ", updateColumns.Select(c => $"[{c}] = excluded.[{c}]")) - : "DO NOTHING"; - - - var upsertSql = - $"INSERT INTO [{tableName}] ({columnsPart}) VALUES ({valuesPart}) ON CONFLICT([{pkColumn}]) {updateClause};"; - - - using var transaction = connection.BeginTransaction(); - using (var command = connection.CreateCommand()) - { - command.Transaction = transaction; - - // create indexes - foreach (var col in columnNames) - { - var colDef = versionDef.definitions.FirstOrDefault(d => d.name == col); - if (colDef.isRelation) - { - command.CommandText = $"CREATE INDEX IF NOT EXISTS idx_{col.ToLower()} ON {tableName} ({col});"; - try - { - command.ExecuteNonQuery(); - } - catch (SqliteException se) - { - Console.WriteLine("Error executing command:"); - Console.WriteLine(command.CommandText); - foreach (SqliteParameter param in command.Parameters) - Console.WriteLine($"{param.ParameterName} = {param.Value}"); - Console.WriteLine("Exception: " + se.Message); - throw; - } - } - } - - - command.CommandText = upsertSql; - - // store a list of all relations to create indexes for those columns for faster lookups - var indexList = new HashSet(); - foreach (var kvp in storage.Values) - { - command.Parameters.Clear(); - - foreach (var col in columnNames) - { - var value = kvp[col] ?? DBNull.Value; - var colDef = versionDef.definitions.FirstOrDefault(d => d.name == col); - if (colDef.isRelation && value == (object)0) value = DBNull.Value; - if (colDef.arrLength > 0) - if (value is Array arr) - value = JsonSerializer.Serialize(arr); - - command.Parameters.AddWithValue("@" + col, value); - } - - try - { - command.ExecuteNonQuery(); - } - catch (SqliteException se) - { - Console.WriteLine("Error executing command:"); - Console.WriteLine(command.CommandText); - foreach (SqliteParameter param in command.Parameters) - Console.WriteLine($"{param.ParameterName} = {param.Value}"); - Console.WriteLine("Exception: " + se.Message); - throw; - } - } - } - - transaction.Commit(); - } -} diff --git a/tools/DB2ToSqlite/Program.cs b/tools/DB2ToSqlite/Program.cs deleted file mode 100644 index b84029acc8..0000000000 --- a/tools/DB2ToSqlite/Program.cs +++ /dev/null @@ -1,126 +0,0 @@ -using System.Diagnostics; -using DB2ToSqliteTool.Helpers; -using DBCD; -using DBCD.Providers; -using DBDefsLib; -using Microsoft.Data.Sqlite; -using TACTSharp; - -var settingsFile = "appsettings.json"; -var databaseFile = "wowsims.db"; - -// Parse command-line arguments for settings and output file overrides. -for (var i = 0; i < args.Length; i++) -{ - if (args[i] == "--settings" || args[i] == "-s") - if (i + 1 < args.Length) - settingsFile = args[i + 1]; - - if (args[i] == "--output" || args[i] == "-output" || args[i] == "-o") - if (i + 1 < args.Length) - databaseFile = args[i + 1]; -} - -// Derive targetDirectory from the databaseFile’s directory (if specified) or fall back to configuration. - -var configuration = new ConfigurationBuilder() - .SetBasePath(Directory.GetCurrentDirectory()) - .AddJsonFile(settingsFile, false, true) - .Build(); - -var settings = configuration.GetSection("Settings").Get(); -if (settings == null) throw new Exception("Failed to load Settings from configuration."); - -var listFile = new Listfile(); - -listFile.Initialize(new CDN(settings), settings); - -Debug.Assert(settings.BaseDir != null); -var buildInfo = new BuildInfo(Path.Combine(settings.BaseDir, ".build.info"), settings, new CDN(settings)); - -var buildInstance = new BuildInstance(); -var entry = buildInfo.Entries.First(x => x.Product == settings.Product); - -buildInstance.Settings.BuildConfig ??= entry.BuildConfig; -buildInstance.Settings.CDNConfig ??= entry.CDNConfig; - -buildInstance.LoadConfigs(buildInstance.Settings.BuildConfig, buildInstance.Settings.CDNConfig); -buildInstance.Load(); - -var tables = configuration.GetSection("Tables").Get>(); -var gameTables = configuration.GetSection("GameTables").Get>(); - -var gameTablesOutDir = configuration.GetSection("GameTablesOutDirectory").Get() ?? "GameTables"; - -var targetDirectory = configuration.GetValue("TargetDirectory") ?? "dbfilesclient"; - -Directory.CreateDirectory(targetDirectory); -Directory.CreateDirectory(gameTablesOutDir); - -var fsProvider = new FilesystemDBCProvider(targetDirectory, true); -var githubDbdProvider = new GithubDBDProvider(true); - -var dbcd = new DBCD.DBCD(fsProvider, githubDbdProvider); - -var dbDefinitions = new Dictionary(); -var storageMap = new Dictionary(); -if (gameTables != null && gameTables.Any()) - foreach (var gameTable in gameTables) - { - var file = buildInstance.OpenFileByFDID(listFile.GetFDID($"gametables/{gameTable}.txt")); - await File.WriteAllBytesAsync($"{gameTablesOutDir}/{gameTable}.txt", file); - } - -if (tables != null) - foreach (var tableName in tables) - { - var file = buildInstance.OpenFileByFDID(listFile.GetFDID($"{targetDirectory}/{tableName}.db2")); - await File.WriteAllBytesAsync($"{targetDirectory}/{tableName}.db2", file); - - var tableDefStream = githubDbdProvider.StreamForTableName(tableName, entry.Version); - var dbReader = new DBDReader(); - var tableDefinition = dbReader.Read(tableDefStream, true); - - var storage = dbcd.Load(tableName, entry.Version); - - dbDefinitions.Add(tableName, tableDefinition); - storageMap.Add(tableName, storage); - } - -var splitBuild = entry.Version.Split('.'); -if (splitBuild.Length != 4) - throw new Exception("Invalid build!"); - -var buildNumber = uint.Parse(splitBuild[3]); -SqliteDbCreator.CreateDatabaseWithDefinitions(dbDefinitions, databaseFile, buildNumber); - - -if (HotfixManager.HotfixReaders.Count == 0) - HotfixManager.LoadCaches(settings.BaseDir); -if (!HotfixManager.HotfixReaders.TryGetValue(buildNumber, out var hotfixReader)) { - - //throw new Exception("No hotfix found for build " + buildNumber); -} - -var connectionString = new SqliteConnectionStringBuilder -{ - DataSource = databaseFile, - Mode = SqliteOpenMode.ReadWriteCreate -}.ToString(); - -await using var conn = new SqliteConnection(connectionString); - -conn.Open(); - -foreach (var tableName in tables) -{ - var storage = storageMap[tableName]; - if (hotfixReader != null) { - storage.ApplyingHotfixes(hotfixReader); - } - SqliteDataInserter.InsertRows(storage, tableName, dbDefinitions[tableName], conn, buildNumber); -} - -conn.Close(); - -Console.WriteLine("Processing completed."); diff --git a/tools/DB2ToSqlite/Properties/launchSettings.json b/tools/DB2ToSqlite/Properties/launchSettings.json deleted file mode 100644 index 266eaea0f0..0000000000 --- a/tools/DB2ToSqlite/Properties/launchSettings.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "$schema": "https://json.schemastore.org/launchsettings.json", - "profiles": { - "http": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": false, - "applicationUrl": "http://localhost:5221", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - }, - "https": { - "commandName": "Project", - "dotnetRunMessages": true, - "launchBrowser": false, - "applicationUrl": "https://localhost:7268;http://localhost:5221", - "environmentVariables": { - "ASPNETCORE_ENVIRONMENT": "Development" - } - } - } -} diff --git a/tools/DB2ToSqlite/appsettings.Development.json b/tools/DB2ToSqlite/appsettings.Development.json deleted file mode 100644 index a91944829f..0000000000 --- a/tools/DB2ToSqlite/appsettings.Development.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} diff --git a/tools/DB2ToSqlite/appsettings.json b/tools/DB2ToSqlite/appsettings.json deleted file mode 100644 index 066517b255..0000000000 --- a/tools/DB2ToSqlite/appsettings.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*", - "Settings": { - "BaseDir": "F:\\World of Warcraft", - "BuildConfig": "buildConfig", - "CDNConfig": "cdnConfig", - "Region": "us", - "Product": "wow_classic" - }, - "TargetDirectory": "dbfilesclient", - "DatabaseFile": "wowsims.db", - "Tables": [ - "Spell", - "Item", - "GemProperties", - "ItemSet", - "ItemSetSpell", - "ItemSubClass", - "ItemSubClassMask", - "ItemReforge", - "ItemBonus", - "ItemEffect", - "ItemClass", - "ItemRandomProperties", - "ItemExtendedCost", - "ItemRandomSuffix", - "RandPropPoints", - "ItemDamageAmmo", - "ItemDamageOneHand", - "ItemDamageOneHandCaster", - "ItemDamageRanged", - "ItemDamageThrown", - "ItemDamageTwoHand", - "ItemDamageTwoHandCaster", - "ItemDamageWand", - "ItemNameDescription", - "ItemSparse", - "ArmorLocation", - "ItemArmorTotal", - "ItemArmorShield", - "ItemArmorQuality", - "SkillLineAbility", - "Curve", - "CurvePoint", - "Difficulty", - "SpellEquippedItems", - "SpellRadius", - "SpellMisc", - "SpellLevels", - "SpellName", - "ScalingStatDistribution", - "SpellCooldowns", - "SpellScaling", - "SpellClassOptions", - "SpellItemEnchantment", - "SpellEffect" - ] -} diff --git a/tools/DB2ToSqlite/references/DBCD.IO.dll b/tools/DB2ToSqlite/references/DBCD.IO.dll deleted file mode 100644 index c16e55a117..0000000000 Binary files a/tools/DB2ToSqlite/references/DBCD.IO.dll and /dev/null differ diff --git a/tools/DB2ToSqlite/references/DBCD.dll b/tools/DB2ToSqlite/references/DBCD.dll deleted file mode 100644 index 4b7b59950e..0000000000 Binary files a/tools/DB2ToSqlite/references/DBCD.dll and /dev/null differ diff --git a/tools/DB2ToSqlite/references/DBDefsLib.dll b/tools/DB2ToSqlite/references/DBDefsLib.dll deleted file mode 100644 index 8c94ac7fc5..0000000000 Binary files a/tools/DB2ToSqlite/references/DBDefsLib.dll and /dev/null differ diff --git a/tools/DB2ToSqlite/references/TACTSharp.dll b/tools/DB2ToSqlite/references/TACTSharp.dll deleted file mode 100644 index 7e873dd5e1..0000000000 Binary files a/tools/DB2ToSqlite/references/TACTSharp.dll and /dev/null differ diff --git a/tools/database/database.go b/tools/database/database.go index d5a682411f..3be2e72f50 100644 --- a/tools/database/database.go +++ b/tools/database/database.go @@ -297,8 +297,22 @@ func ReadDatabaseFromJson(jsonStr string) *WowDatabase { } func (db *WowDatabase) WriteBinaryAndJson(binFilePath, jsonFilePath string) { + jsonBytes := db.toJsonBytes() + // The JSON covers every UIDatabase field, so unchanged JSON means unchanged + // contents — worth skipping because the binary's proto encoding is not + // byte-stable and would otherwise churn on every run. A missing binary must + // still be regenerated, though, so check for it before skipping. + if existing, err := os.ReadFile(jsonFilePath); err == nil && bytes.Equal(existing, jsonBytes) { + if _, err := os.Stat(binFilePath); err == nil { + log.Printf("No changes detected, skipping write of %s and %s", binFilePath, jsonFilePath) + return + } + log.Printf("%s is missing, regenerating it", binFilePath) + } db.WriteBinary(binFilePath) - db.WriteJson(jsonFilePath) + if err := os.WriteFile(jsonFilePath, jsonBytes, 0666); err != nil { + log.Fatalf("[ERROR] Failed to write %s: %s", jsonFilePath, err.Error()) + } } func (db *WowDatabase) WriteBinary(binFilePath string) { @@ -309,12 +323,20 @@ func (db *WowDatabase) WriteBinary(binFilePath string) { if err != nil { log.Fatalf("[ERROR] Failed to marshal db: %s", err.Error()) } - os.WriteFile(binFilePath, protoBytes, 0666) + if err := os.WriteFile(binFilePath, protoBytes, 0666); err != nil { + log.Fatalf("[ERROR] Failed to write %s: %s", binFilePath, err.Error()) + } } func (db *WowDatabase) WriteJson(jsonFilePath string) { - // Also write in JSON format, so we can manually inspect the contents. - // Write it out line-by-line, so we can have 1 line / item, making it more human-readable. + if err := os.WriteFile(jsonFilePath, db.toJsonBytes(), 0666); err != nil { + log.Fatalf("[ERROR] Failed to write %s: %s", jsonFilePath, err.Error()) + } +} + +// Serializes in JSON format, so we can manually inspect the contents. +// Written out line-by-line, so we can have 1 line / item, making it more human-readable. +func (db *WowDatabase) toJsonBytes() []byte { uidb := db.ToUIProto() buffer := new(bytes.Buffer) @@ -349,5 +371,5 @@ func (db *WowDatabase) WriteJson(jsonFilePath string) { tools.WriteProtoArrayToBuffer(uidb.SpellEffects, buffer, "spellEffects") buffer.WriteString("\n") buffer.WriteString("}") - os.WriteFile(jsonFilePath, buffer.Bytes(), 0666) + return buffer.Bytes() } diff --git a/tools/database/gen_db/main.go b/tools/database/gen_db/main.go index 586ce7c18e..ef8af3f8a6 100644 --- a/tools/database/gen_db/main.go +++ b/tools/database/gen_db/main.go @@ -26,7 +26,7 @@ import ( var outDir = flag.String("outDir", "assets", "Path to output directory for writing generated .go files.") var genAsset = flag.String("gen", "", "Asset to generate. Valid values are 'db', 'atlasloot', 'wowhead-items', 'wowhead-spells', 'wowhead-itemdb', 'mop-items', and 'wago-db2-items'") -var dbPath = flag.String("dbPath", "./tools/database/wowsims.db", "Location of wowsims.db file from the DB2ToSqliteTool") +var dbPath = flag.String("dbPath", "./tools/database/wowsims.db", "Location of the wowsims.db file produced by tools/db2tool") func main() { flag.Parse() @@ -150,7 +150,7 @@ func main() { db.Encounters = core.PresetEncounters db.ReforgeStats = reforgeStats.ToProto() - iconsMap, err := database.LoadArtTexturePaths("./tools/DB2ToSqlite/listfile.csv") + iconsMap, err := database.LoadArtTexturePaths(database.ListfilePath) if err != nil { panic(fmt.Sprintf("Error loading icon paths %v", err)) } diff --git a/tools/database/gen_effects.go b/tools/database/gen_effects.go index 0fdceb7ab1..815c2d0592 100644 --- a/tools/database/gen_effects.go +++ b/tools/database/gen_effects.go @@ -1,7 +1,9 @@ package database import ( + "bytes" "fmt" + "go/format" "os" "regexp" "slices" @@ -83,7 +85,13 @@ func GenerateEffectsFile(groups []*Group, outFile string, templateString string) return !grp.Entries[i].Supported } - return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + // Tie-break on SpellID: an item with multiple effects produces + // multiple entries with the same item ID. + if grp.Entries[i].Variants[0].ID != grp.Entries[j].Variants[0].ID { + return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + } + + return grp.Entries[i].Variants[0].SpellID < grp.Entries[j].Variants[0].SpellID }) } @@ -94,13 +102,21 @@ func GenerateEffectsFile(groups []*Group, outFile string, templateString string) "formatStrings": formatStrings, } tmpl := template.Must(template.New("effects").Funcs(funcMap).Parse(templateString)) - f, err := os.Create(outFile) + var buf bytes.Buffer + if err := tmpl.Execute(&buf, map[string]any{"Groups": groups}); err != nil { + return fmt.Errorf("failed to execute template: %w", err) + } + + // Format like gofmt so regeneration doesn't fight the pre-commit hook. + formatted, err := format.Source(buf.Bytes()) if err != nil { - return fmt.Errorf("failed to create file %s: %w", outFile, err) + // Write the raw output anyway so the invalid code can be inspected. + os.WriteFile(outFile, buf.Bytes(), 0o644) + return fmt.Errorf("generated code for %s does not parse: %w", outFile, err) } - defer f.Close() - if err := tmpl.Execute(f, map[string]interface{}{"Groups": groups}); err != nil { - return fmt.Errorf("failed to execute template: %w", err) + + if err := os.WriteFile(outFile, formatted, 0o644); err != nil { + return fmt.Errorf("failed to create file %s: %w", outFile, err) } return nil @@ -222,7 +238,10 @@ func GenerateItemEffects(instance *dbc.DBC, db *WowDatabase, itemSources map[int // sort entries first to make grouping consistent for variants sort.Slice(grp.Entries, func(i, j int) bool { - return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + if grp.Entries[i].Variants[0].ID != grp.Entries[j].Variants[0].ID { + return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + } + return grp.Entries[i].Variants[0].SpellID < grp.Entries[j].Variants[0].SpellID }) for _, entry := range grp.Entries { @@ -254,7 +273,10 @@ func GenerateItemEffects(instance *dbc.DBC, db *WowDatabase, itemSources map[int // sort entries first to make tooltip generation consistent for variants sort.Slice(grp.Entries, func(i, j int) bool { - return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + if grp.Entries[i].Variants[0].ID != grp.Entries[j].Variants[0].ID { + return grp.Entries[i].Variants[0].ID < grp.Entries[j].Variants[0].ID + } + return grp.Entries[i].Variants[0].SpellID < grp.Entries[j].Variants[0].SpellID }) for _, entry := range grp.Entries { diff --git a/tools/database/gen_protos.go b/tools/database/gen_protos.go index 861462c3ef..caa663d5d6 100644 --- a/tools/database/gen_protos.go +++ b/tools/database/gen_protos.go @@ -455,7 +455,7 @@ func GenerateProtos(dbcData *dbc.DBC, db *WowDatabase) { allGlyphSpellIds := []*proto.GlyphID{} var classesData []ClassData - iconsMap, _ := LoadArtTexturePaths("./tools/DB2ToSqlite/listfile.csv") + iconsMap, _ := LoadArtTexturePaths(ListfilePath) for _, dbcClass := range dbc.Classes { className := dbc.ClassNameFromDBC(dbcClass) data := ClassData{ diff --git a/tools/database/generator-settings.json b/tools/database/generator-settings.json index d35582928a..82be5f453c 100644 --- a/tools/database/generator-settings.json +++ b/tools/database/generator-settings.json @@ -1,13 +1,9 @@ { "Settings": { "BaseDir": "/mnt/c/Program Files/World of Warcraft", - "BuildConfig": "buildConfig", - "CDNConfig": "cdnConfig", - "Region": "us", "Product": "wow_classic" }, "TargetDirectory": "dbfilesclient", - "DatabaseFile": "wowsims.db", "GameTablesOutDirectory": "../../assets/db_inputs/basestats", "GameTables": [ "chancetomeleecrit", diff --git a/tools/database/icon_loader.go b/tools/database/icon_loader.go index eaf29bf054..a726af1d1c 100644 --- a/tools/database/icon_loader.go +++ b/tools/database/icon_loader.go @@ -10,6 +10,11 @@ import ( "strings" ) +// ListfilePath is where tools/db2tool caches the community listfile. It maps +// file data ids to game paths and is a second output contract of the extractor, +// alongside wowsims.db. +const ListfilePath = "./tools/db2tool/listfile.csv" + func LoadArtTexturePaths(filePath string) (map[int]string, error) { f, err := os.Open(filePath) if err != nil { diff --git a/tools/database/ptr-generator-settings.json b/tools/database/ptr-generator-settings.json index 45c8d7c87e..323e06da8c 100644 --- a/tools/database/ptr-generator-settings.json +++ b/tools/database/ptr-generator-settings.json @@ -1,13 +1,9 @@ { "Settings": { "BaseDir": "/mnt/c/Program Files/World of Warcraft", - "BuildConfig": "buildConfig", - "CDNConfig": "cdnConfig", - "Region": "us", "Product": "wow_classic_ptr" }, "TargetDirectory": "dbfilesclient", - "DatabaseFile": "wowsims.db", "GameTablesOutDirectory": "../../assets/db_inputs/basestats", "GameTables": [ "chancetomeleecrit", diff --git a/tools/database/tables.go b/tools/database/tables.go index db4e44606f..a4d64bbd7e 100644 --- a/tools/database/tables.go +++ b/tools/database/tables.go @@ -1120,7 +1120,7 @@ LEFT JOIN SpellName sn ON sn.ID = sm.SpellID return iconsByID, nil } -var iconsMap, _ = LoadArtTexturePaths("./tools/DB2ToSqlite/listfile.csv") +var iconsMap, _ = LoadArtTexturePaths(ListfilePath) func ScanSpells(rows *sql.Rows) (dbc.Spell, error) { var spell dbc.Spell diff --git a/tools/db2tool/NOTICES.md b/tools/db2tool/NOTICES.md new file mode 100644 index 0000000000..4a570937f2 --- /dev/null +++ b/tools/db2tool/NOTICES.md @@ -0,0 +1,74 @@ +# Third-party notices for `tools/db2tool` + +Several packages of this tool are Go translations (derivative works) of +upstream C# libraries. Each derived source file carries a short notice header +pointing here; this file is the authoritative list of upstreams, licenses, +and pinned revisions. + +| Package dir | Upstream | License | Pinned revision | +|---|---|---|---| +| `wdc/` | [wowdev/DBCD](https://github.com/wowdev/DBCD) (DBCD + DBCD.IO, v2.1.2) | MIT, Copyright (c) 2020 wowdev | `2180edb4d08b3822b3cfa964293ba8ccd4236ac0` | +| `dbd/` | [wowdev/WoWDBDefs](https://github.com/wowdev/WoWDBDefs) `code/C#/DBDefsLib` (**code** is BSD-3-Clause; the `.dbd` **data** files are CC BY-SA 4.0 and are fetched at build time, never vendored) | BSD-3-Clause, Copyright 2022 WoWDBDefs Contributors | `9002c532853a96d631c76dda50cb20189c27a173` (master at port time) | +| `tact/` | [wowdev/TACTSharp](https://github.com/wowdev/TACTSharp) v0.0.13-alpha | MIT, Copyright (c) 2024 Martin Benjamins | `d0ab516eb98b5db35682467b6e4977d88955046d` | +| `wdc/hotfix.go` (cache scanning + SStrHash; the XFTH reader itself derives from DBCD above) | [Marlamin/wow.tools.local](https://github.com/Marlamin/wow.tools.local) `Services/{HotfixManager,DBCacheParser}.cs` | MIT, Copyright (c) 2022 Martin Benjamins | `0aefbece74ef4e19ce67ebe91b51a8ae424c5c11` (upstream main at port time) | +| `sqlite/`, `config/`, `main.go` | original repo code | repo MIT | — | + +Runtime data dependencies (fetched, never vendored): + +- `.dbd` definitions from WoWDBDefs (`definitions/.dbd`) — CC BY-SA 4.0, + cached under a gitignored `DBDCache/`. +- `listfile.csv` (community listfile) — cached, gitignored. +- No TACT keys are used; encrypted DB2 sections are skipped. + +## MIT License (wowdev/DBCD, wowdev/TACTSharp, Marlamin/wow.tools.local) + +MIT License + +Copyright (c) 2020 wowdev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## BSD-3-Clause (WoWDBDefs code — applies to `dbd/`; these files stay BSD-3-Clause, not relicensed) + +Copyright 2022 WoWDBDefs Contributors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/tools/db2tool/config/config.go b/tools/db2tool/config/config.go new file mode 100644 index 0000000000..a92c19d6b8 --- /dev/null +++ b/tools/db2tool/config/config.go @@ -0,0 +1,45 @@ +// Package config binds the db2tool settings files, tools/database/ +// generator-settings.json and ptr-generator-settings.json. +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +// Settings is the settings file's "Settings" section: which product to extract +// and the install to read it from. +type Settings struct { + Product string `json:"Product"` + BaseDir string `json:"BaseDir"` +} + +type File struct { + Settings Settings `json:"Settings"` + // TargetDirectory does double duty (listfile-key prefix AND output dir); + // the FDID/listfile use must always see the raw value, never a + // filesystem-resolved path. + TargetDirectory string `json:"TargetDirectory"` + GameTablesOutDirectory string `json:"GameTablesOutDirectory"` + GameTables []string `json:"GameTables"` + Tables []string `json:"Tables"` +} + +func Load(path string) (*File, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var f File + if err := json.Unmarshal(raw, &f); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + if f.TargetDirectory == "" { + f.TargetDirectory = "dbfilesclient" + } + if f.GameTablesOutDirectory == "" { + f.GameTablesOutDirectory = "GameTables" + } + return &f, nil +} diff --git a/tools/db2tool/dbd/dbd.go b/tools/db2tool/dbd/dbd.go new file mode 100644 index 0000000000..d85a4e4bee --- /dev/null +++ b/tools/db2tool/dbd/dbd.go @@ -0,0 +1,521 @@ +// Go translation of DBDefsLib's DBDReader +// (https://github.com/wowdev/WoWDBDefs, code/C#/DBDefsLib). +// Copyright 2022 WoWDBDefs Contributors. Licensed under BSD-3-Clause; this +// file remains BSD-3-Clause (full text, including the non-endorsement clause, +// in tools/db2tool/NOTICES.md). Upstream commit 9002c532853a96d631c76dda50cb20189c27a173. + +// Package dbd parses WoWDBDefs .dbd definition files and selects the version +// block matching an exact build number. +package dbd + +import ( + "fmt" + "io" + "os" + "slices" + "strconv" + "strings" +) + +type ColumnDefinition struct { + Type string + ForeignTable string + ForeignColumn string + Verified bool + Comment string +} + +type Definition struct { + Size int + ArrLength int + Name string + IsID bool + IsRelation bool + IsNonInline bool + IsSigned bool + Comment string +} + +type Build struct { + Expansion int16 + Major int16 + Minor int16 + Build uint32 +} + +func ParseBuild(s string) (Build, error) { + split := strings.Split(s, ".") + if len(split) != 4 { + return Build{}, fmt.Errorf("invalid build string %q", s) + } + expansion, err := strconv.ParseInt(split[0], 10, 16) + if err != nil { + return Build{}, fmt.Errorf("invalid build string %q: %w", s, err) + } + major, err := strconv.ParseInt(split[1], 10, 16) + if err != nil { + return Build{}, fmt.Errorf("invalid build string %q: %w", s, err) + } + minor, err := strconv.ParseInt(split[2], 10, 16) + if err != nil { + return Build{}, fmt.Errorf("invalid build string %q: %w", s, err) + } + build, err := strconv.ParseUint(split[3], 10, 32) + if err != nil { + return Build{}, fmt.Errorf("invalid build string %q: %w", s, err) + } + return Build{Expansion: int16(expansion), Major: int16(major), Minor: int16(minor), Build: uint32(build)}, nil +} + +func (b Build) String() string { + return fmt.Sprintf("%d.%d.%d.%d", b.Expansion, b.Major, b.Minor, b.Build) +} + +func (b Build) Compare(o Build) int { + if b.Expansion != o.Expansion { + return int(b.Expansion) - int(o.Expansion) + } + if b.Major != o.Major { + return int(b.Major) - int(o.Major) + } + if b.Minor != o.Minor { + return int(b.Minor) - int(o.Minor) + } + if b.Build != o.Build { + if b.Build < o.Build { + return -1 + } + return 1 + } + return 0 +} + +type BuildRange struct { + MinBuild Build + MaxBuild Build +} + +func (r BuildRange) Contains(b Build) bool { + return b.Compare(r.MinBuild) >= 0 && b.Compare(r.MaxBuild) <= 0 +} + +func (r BuildRange) String() string { + return r.MinBuild.String() + "-" + r.MaxBuild.String() +} + +type VersionDefinitions struct { + Builds []Build + BuildRanges []BuildRange + LayoutHashes []string + Comment string + Definitions []Definition +} + +type DBDefinition struct { + ColumnDefinitions map[string]ColumnDefinition + VersionDefinitions []VersionDefinitions +} + +// ReadFile parses a .dbd definition file from disk. +func ReadFile(path string, validate bool) (DBDefinition, error) { + f, err := os.Open(path) + if err != nil { + return DBDefinition{}, err + } + defer f.Close() + def, err := Read(f, validate) + if err != nil { + return DBDefinition{}, fmt.Errorf("%s: %w", path, err) + } + return def, nil +} + +// Read parses a .dbd definition stream. It is a line-for-line transcription +// of the upstream reader (deliberately faithful even where behavior is +// quirky). +func Read(r io.Reader, validate bool) (DBDefinition, error) { + raw, err := io.ReadAll(r) + if err != nil { + return DBDefinition{}, err + } + lines := readLines(raw) + if len(lines) == 0 { + return DBDefinition{}, fmt.Errorf("empty .dbd file") + } + + columnDefinitions := make(map[string]ColumnDefinition) + lineNumber := 0 + + if !strings.HasPrefix(lines[0], "COLUMNS") { + return DBDefinition{}, fmt.Errorf("file does not start with column definitions") + } + + lineNumber++ + for lineNumber < len(lines) { + line := lines[lineNumber] + lineNumber++ + + // Column definitions are done after encountering a blank line. + if isBlank(line) { + break + } + + var colDef ColumnDefinition + + if !strings.Contains(line, " ") { + return DBDefinition{}, fmt.Errorf("line %q does not contain a space between type and column name", line) + } + + // Read line up to space (end of type) or < (foreign key). + typeEnd := strings.IndexAny(line, " <") + colType := line[:typeEnd] + switch colType { + case "uint", "int", "float", "string", "locstring": + colDef.Type = colType + default: + return DBDefinition{}, fmt.Errorf("invalid type %q on line %d", colType, lineNumber) + } + + // Only read foreign key if the identifier is right after the type. + if strings.HasPrefix(line, colType+"<") { + lt := strings.Index(line, "<") + gt := strings.Index(line, ">") + if gt < lt { + return DBDefinition{}, fmt.Errorf("malformed foreign key on line %d", lineNumber) + } + foreignKey := strings.Split(line[lt+1:gt], "::") + if len(foreignKey) != 2 { + return DBDefinition{}, fmt.Errorf("invalid foreign key length: %d", len(foreignKey)) + } + colDef.ForeignTable = foreignKey[0] + colDef.ForeignColumn = foreignKey[1] + } + + var name string + if strings.LastIndex(line, " ") == strings.Index(line, " ") { + // Simple line like "uint ID". + name = line[strings.Index(line, " ")+1:] + } else { + start := strings.Index(line, " ") + second := indexFrom(line, ' ', start+1) + name = line[start+1 : second] + } + + if strings.HasSuffix(name, "?") { + colDef.Verified = false + name = name[:len(name)-1] + } else { + colDef.Verified = true + } + + if idx := strings.Index(line, "//"); idx >= 0 { + colDef.Comment = strings.TrimSpace(line[idx+2:]) + } + + if _, exists := columnDefinitions[name]; exists { + fmt.Fprintf(os.Stderr, "dbd: collision with existing column name %q, skipping\n", name) + } else { + columnDefinitions[name] = colDef + } + } + + var versionDefinitions []VersionDefinitions + + var definitions []Definition + var layoutHashes []string + comment := "" + var builds []Build + var buildRanges []BuildRange + + flush := func() error { + if len(builds) != 0 || len(buildRanges) != 0 || len(layoutHashes) != 0 { + versionDefinitions = append(versionDefinitions, VersionDefinitions{ + Builds: append([]Build(nil), builds...), + BuildRanges: append([]BuildRange(nil), buildRanges...), + LayoutHashes: append([]string(nil), layoutHashes...), + Comment: comment, + Definitions: append([]Definition(nil), definitions...), + }) + } else if len(definitions) != 0 || !isBlank(comment) { + return fmt.Errorf("no BUILD or LAYOUT, but non-empty lines/definitions") + } + return nil + } + + for i := lineNumber; i < len(lines); i++ { + line := lines[i] + + if isBlank(line) { + if err := flush(); err != nil { + return DBDefinition{}, err + } + definitions = nil + layoutHashes = nil + comment = "" + builds = nil + buildRanges = nil + } + + if strings.HasPrefix(line, "LAYOUT") { + layoutHashes = append(layoutHashes, strings.Split(line[7:], ", ")...) + } + + if strings.HasPrefix(line, "BUILD") { + for splitBuild := range strings.SplitSeq(line[6:], ", ") { + if strings.Contains(splitBuild, "-") { + splitRange := strings.Split(splitBuild, "-") + minBuild, err := ParseBuild(splitRange[0]) + if err != nil { + return DBDefinition{}, err + } + maxBuild, err := ParseBuild(splitRange[1]) + if err != nil { + return DBDefinition{}, err + } + buildRanges = append(buildRanges, BuildRange{MinBuild: minBuild, MaxBuild: maxBuild}) + } else { + build, err := ParseBuild(splitBuild) + if err != nil { + return DBDefinition{}, err + } + builds = append(builds, build) + } + } + } + + if strings.HasPrefix(line, "COMMENT") { + comment = strings.TrimSpace(line[7:]) + } + + if !strings.HasPrefix(line, "LAYOUT") && !strings.HasPrefix(line, "BUILD") && + !strings.HasPrefix(line, "COMMENT") && !isBlank(line) { + definition := Definition{IsNonInline: false} + + if strings.Contains(line, "$") { + annotationStart := strings.Index(line, "$") + annotationEnd := indexFrom(line, '$', 1) + if annotationEnd < 0 { + return DBDefinition{}, fmt.Errorf("unterminated annotation on line %q", line) + } + for a := range strings.SplitSeq(line[annotationStart+1:annotationEnd], ",") { + switch a { + case "id": + definition.IsID = true + case "noninline": + definition.IsNonInline = true + case "relation": + definition.IsRelation = true + } + } + // Upstream removes annotationEnd+1 chars from annotationStart + // (not the annotation's span); replicated faithfully. + line = line[:annotationStart] + line[annotationStart+annotationEnd+1:] + } + + if strings.Contains(line, "<") { + lt := strings.Index(line, "<") + gt := strings.Index(line, ">") + if gt < lt { + return DBDefinition{}, fmt.Errorf("malformed size on line %q", line) + } + size := line[lt+1 : gt] + if size == "" { + return DBDefinition{}, fmt.Errorf("empty size on line %q", line) + } + if size[0] == 'u' { + definition.IsSigned = false + n, err := strconv.Atoi(strings.ReplaceAll(size, "u", "")) + if err != nil { + return DBDefinition{}, fmt.Errorf("invalid size %q: %w", size, err) + } + definition.Size = n + } else { + definition.IsSigned = true + n, err := strconv.Atoi(size) + if err != nil { + return DBDefinition{}, fmt.Errorf("invalid size %q: %w", size, err) + } + definition.Size = n + } + line = line[:lt] + line[gt+1:] + } + + if strings.Contains(line, "[") { + lb := strings.Index(line, "[") + rb := strings.Index(line, "]") + if rb < lb { + return DBDefinition{}, fmt.Errorf("invalid array length format") + } + n, err := strconv.Atoi(line[lb+1 : rb]) + if err != nil { + return DBDefinition{}, fmt.Errorf("invalid array length format") + } + definition.ArrLength = n + line = line[:lb] + line[rb+1:] + } + + if idx := strings.Index(line, "//"); idx >= 0 { + definition.Comment = strings.TrimSpace(line[idx+2:]) + line = strings.TrimSpace(line[:idx]) + } + + definition.Name = line + + colDef, ok := columnDefinitions[definition.Name] + if !ok { + return DBDefinition{}, fmt.Errorf("unable to find %q in column definitions", definition.Name) + } + // Temporary unsigned format update conversion code (upstream). + if colDef.Type == "uint" { + definition.IsSigned = false + } + + definitions = append(definitions, definition) + } + + if len(lines) == i+1 { + if err := flush(); err != nil { + return DBDefinition{}, err + } + } + } + + if validate { + if err := runValidation(columnDefinitions, versionDefinitions); err != nil { + return DBDefinition{}, err + } + } + + return DBDefinition{ + ColumnDefinitions: columnDefinitions, + VersionDefinitions: versionDefinitions, + }, nil +} + +// runValidation is the optional validation pass: warnings go to stderr, hard +// violations become errors. It also removes column definitions never used by +// any version block, as upstream does. +func runValidation(columnDefinitions map[string]ColumnDefinition, versionDefinitions []VersionDefinitions) error { + for name := range columnDefinitions { + found := false + for _, version := range versionDefinitions { + for _, definition := range version.Definitions { + if name == definition.Name { + found = true + break + } + } + if found { + break + } + } + if !found { + fmt.Fprintf(os.Stderr, "dbd: column definition %q is never used in version definitions\n", name) + delete(columnDefinitions, name) + } + } + + seenBuilds := make(map[Build]bool) + seenLayoutHashes := make(map[string]bool) + + for _, version := range versionDefinitions { + for _, build := range version.Builds { + if seenBuilds[build] { + return fmt.Errorf("build %s is already defined", build) + } + seenBuilds[build] = true + } + + for _, layoutHash := range version.LayoutHashes { + if seenLayoutHashes[layoutHash] { + return fmt.Errorf("layout hash %s is already defined", layoutHash) + } + seenLayoutHashes[layoutHash] = true + if len(layoutHash) != 8 { + return fmt.Errorf("layout hash %q is wrong length", layoutHash) + } + } + + for _, definition := range version.Definitions { + colType := columnDefinitions[definition.Name].Type + if (colType == "int" || colType == "uint") && definition.Size == 0 { + return fmt.Errorf("version definition %s is an int/uint but is missing size", definition.Name) + } + if colType != "int" && colType != "uint" && definition.Size != 0 { + return fmt.Errorf("version definition %s is NOT an int/uint but has size", definition.Name) + } + } + + names := make(map[string]bool) + for _, definition := range version.Definitions { + if names[definition.Name] { + return fmt.Errorf("version definitions contains multiple columns of the same name") + } + names[definition.Name] = true + } + } + + for i := range versionDefinitions { + for j := range versionDefinitions { + if i == j { + continue + } + for _, r := range versionDefinitions[i].BuildRanges { + for _, b := range versionDefinitions[j].Builds { + if r.Contains(b) { + return fmt.Errorf("build %s conflicts with %s", b, r) + } + } + for _, or := range versionDefinitions[j].BuildRanges { + if r.Contains(or.MinBuild) || r.Contains(or.MaxBuild) { + return fmt.Errorf("build %s conflicts with %s", or, r) + } + } + } + + if slices.Equal(versionDefinitions[i].Definitions, versionDefinitions[j].Definitions) { + if len(versionDefinitions[i].LayoutHashes) > 0 && len(versionDefinitions[j].LayoutHashes) > 0 && + !slices.Equal(versionDefinitions[i].LayoutHashes, versionDefinitions[j].LayoutHashes) { + // Upstream ignores this case (identical definitions, different layout hashes). + } else { + return fmt.Errorf("dbd file has 2 identical version definitions (%d and %d)", i+1, j+1) + } + } + } + } + + return nil +} + +// readLines splits raw file bytes into lines: \r\n, \r, and \n all terminate +// a line, a terminator at EOF does not produce a trailing empty line, and a +// leading UTF-8 BOM is stripped. +func readLines(raw []byte) []string { + s := string(raw) + s = strings.TrimPrefix(s, "\ufeff") + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + lines := strings.Split(s, "\n") + if len(lines) > 0 && lines[len(lines)-1] == "" && strings.HasSuffix(s, "\n") { + lines = lines[:len(lines)-1] + } + if len(lines) == 1 && lines[0] == "" { + return nil + } + return lines +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func indexFrom(s string, c byte, from int) int { + if from >= len(s) { + return -1 + } + idx := strings.IndexByte(s[from:], c) + if idx < 0 { + return -1 + } + return from + idx +} diff --git a/tools/db2tool/dbd/dbd_test.go b/tools/db2tool/dbd/dbd_test.go new file mode 100644 index 0000000000..ca5d8edc3e --- /dev/null +++ b/tools/db2tool/dbd/dbd_test.go @@ -0,0 +1,247 @@ +package dbd + +import ( + "strings" + "testing" +) + +// A .dbd covering every construct the 72 configured tables actually use: +// plain/foreign-key/unverified columns, all five types, $id$ / $noninline,id$ / +// $relation$ annotations, signed and unsigned suffixes, [n] arrays, +// LAYOUT + single builds + a build range, and a COMMENT. +const sampleDBD = `COLUMNS +int ID +int ItemID +uint Flags +float Coefficient +string Path +locstring Name +int Unverified? +int Legacy + +LAYOUT 0A1B2C3D +BUILD 5.5.0.60000 +COMMENT older layout +$id$ID<32> +Legacy<16> +Coefficient +Name + +LAYOUT 4E5F6071, 8899AABB +BUILD 5.5.4.68571, 5.5.4.68806 +BUILD 5.4.0.10000-5.4.9.19999 +$noninline,id$ID<32> +$relation$ItemID<32> +Flags[3] +Coefficient +Path +Name +Unverified<32> +` + +func parseSample(t *testing.T) DBDefinition { + t.Helper() + def, err := Read(strings.NewReader(sampleDBD), true) + if err != nil { + t.Fatal(err) + } + return def +} + +func TestReadColumnDefinitions(t *testing.T) { + def := parseSample(t) + + if got := def.ColumnDefinitions["ID"].Type; got != "int" { + t.Errorf("ID type = %q, want int", got) + } + if got := def.ColumnDefinitions["Flags"].Type; got != "uint" { + t.Errorf("Flags type = %q, want uint", got) + } + if got := def.ColumnDefinitions["Name"].Type; got != "locstring" { + t.Errorf("Name type = %q, want locstring", got) + } + + // Foreign keys drive both the NULL-ability and the IX_ index in the schema. + item := def.ColumnDefinitions["ItemID"] + if item.ForeignTable != "Item" || item.ForeignColumn != "ID" { + t.Errorf("ItemID foreign key = %q::%q, want Item::ID", item.ForeignTable, item.ForeignColumn) + } + if def.ColumnDefinitions["ID"].ForeignTable != "" { + t.Error("ID must not have a foreign key") + } + + // A trailing ? marks the column unverified and must not survive in the name. + if !def.ColumnDefinitions["ID"].Verified { + t.Error("ID should be verified") + } + if u, ok := def.ColumnDefinitions["Unverified"]; !ok || u.Verified { + t.Errorf("Unverified column = %+v, ok=%v; want present and unverified", u, ok) + } +} + +func TestReadVersionDefinitions(t *testing.T) { + def := parseSample(t) + + if len(def.VersionDefinitions) != 2 { + t.Fatalf("parsed %d version blocks, want 2", len(def.VersionDefinitions)) + } + old, cur := def.VersionDefinitions[0], def.VersionDefinitions[1] + + if len(old.Builds) != 1 || old.Builds[0].Build != 60000 { + t.Errorf("first block builds = %v, want one 60000", old.Builds) + } + if old.Comment != "older layout" { + t.Errorf("first block comment = %q", old.Comment) + } + if len(old.LayoutHashes) != 1 || old.LayoutHashes[0] != "0A1B2C3D" { + t.Errorf("first block layout hashes = %v", old.LayoutHashes) + } + + if len(cur.Builds) != 2 || cur.Builds[0].Build != 68571 || cur.Builds[1].Build != 68806 { + t.Errorf("second block builds = %v, want 68571 and 68806", cur.Builds) + } + if len(cur.BuildRanges) != 1 { + t.Fatalf("second block build ranges = %v, want 1", cur.BuildRanges) + } + if got := cur.BuildRanges[0].String(); got != "5.4.0.10000-5.4.9.19999" { + t.Errorf("build range = %q", got) + } + if len(cur.LayoutHashes) != 2 { + t.Errorf("second block layout hashes = %v, want 2", cur.LayoutHashes) + } +} + +func TestReadDefinitionAnnotations(t *testing.T) { + def := parseSample(t) + byName := map[string]Definition{} + for _, d := range def.VersionDefinitions[1].Definitions { + byName[d.Name] = d + } + + // $noninline,id$ — the id lives in the index block, not the record. + id := byName["ID"] + if !id.IsID || !id.IsNonInline || id.Size != 32 || !id.IsSigned { + t.Errorf("ID = %+v, want id + noninline + signed size 32", id) + } + // An inline $id$ must NOT be flagged non-inline. + oldID := def.VersionDefinitions[0].Definitions[0] + if !oldID.IsID || oldID.IsNonInline { + t.Errorf("first-block ID = %+v, want id and inline", oldID) + } + + rel := byName["ItemID"] + if !rel.IsRelation || rel.IsNonInline { + t.Errorf("ItemID = %+v, want relation and inline", rel) + } + + // [3]: unsigned, 8-bit, three elements. + flags := byName["Flags"] + if flags.Size != 8 || flags.IsSigned || flags.ArrLength != 3 { + t.Errorf("Flags = %+v, want unsigned size 8 arrLength 3", flags) + } + + // float/string/locstring carry no size, and non-arrays report 0. + for _, name := range []string{"Coefficient", "Path", "Name"} { + if d := byName[name]; d.Size != 0 || d.ArrLength != 0 { + t.Errorf("%s = %+v, want size 0 arrLength 0", name, d) + } + } +} + +func TestSelectVersionExactBuildOnly(t *testing.T) { + def := parseSample(t) + + for _, build := range []uint32{68571, 68806} { + v, err := SelectVersion(def, build) + if err != nil { + t.Fatalf("build %d: %v", build, err) + } + if len(v.Definitions) != 7 { + t.Errorf("build %d selected %d definitions, want the 7-column block", build, len(v.Definitions)) + } + } + + v, err := SelectVersion(def, 60000) + if err != nil { + t.Fatal(err) + } + if len(v.Definitions) != 4 { + t.Errorf("build 60000 selected %d definitions, want the 4-column block", len(v.Definitions)) + } + + // A build covered only by a buildRange is deliberately NOT matched: an + // unlisted live build must fail loud rather than decode with a near-miss + // layout. + if _, err := SelectVersion(def, 15000); err == nil { + t.Error("a build inside a buildRange must not be selected") + } + if _, err := SelectVersion(def, 99999); err == nil { + t.Error("an unknown build must be rejected") + } +} + +func TestReadRejectsMalformed(t *testing.T) { + tests := map[string]string{ + "no COLUMNS header": "BUILD 1.2.3.4\nID\n", + "unknown type": "COLUMNS\nblob Data\n\nBUILD 1.2.3.4\nData\n", + "missing space": "COLUMNS\nintID\n\nBUILD 1.2.3.4\nintID\n", + "undeclared column": "COLUMNS\nint ID\n\nBUILD 1.2.3.4\nMissing<32>\n", + "int without size": "COLUMNS\nint ID\n\nBUILD 1.2.3.4\nID\n", + "size on a string": "COLUMNS\nstring S\n\nBUILD 1.2.3.4\nS<32>\n", + "bad build string": "COLUMNS\nint ID\n\nBUILD notabuild\nID<32>\n", + "empty file": "", + } + for name, src := range tests { + if _, err := Read(strings.NewReader(src), true); err == nil { + t.Errorf("%s: expected an error, got nil", name) + } + } +} + +func TestParseBuild(t *testing.T) { + b, err := ParseBuild("5.5.4.68571") + if err != nil { + t.Fatal(err) + } + if b.Expansion != 5 || b.Major != 5 || b.Minor != 4 || b.Build != 68571 { + t.Errorf("ParseBuild = %+v", b) + } + if got := b.String(); got != "5.5.4.68571" { + t.Errorf("String() = %q", got) + } + for _, bad := range []string{"5.5.4", "5.5.4.68571.1", "", "a.b.c.d", "5.5.4.x"} { + if _, err := ParseBuild(bad); err == nil { + t.Errorf("ParseBuild(%q) should have failed", bad) + } + } +} + +func TestBuildCompareAndRange(t *testing.T) { + mustParse := func(s string) Build { + t.Helper() + b, err := ParseBuild(s) + if err != nil { + t.Fatal(err) + } + return b + } + lo, hi := mustParse("5.4.0.10000"), mustParse("5.4.9.19999") + r := BuildRange{MinBuild: lo, MaxBuild: hi} + + for _, in := range []string{"5.4.0.10000", "5.4.5.15000", "5.4.9.19999"} { + if !r.Contains(mustParse(in)) { + t.Errorf("%s should be inside %s", in, r) + } + } + for _, out := range []string{"5.3.9.9999", "5.5.0.10001", "5.4.0.9999"} { + if r.Contains(mustParse(out)) { + t.Errorf("%s should be outside %s", out, r) + } + } + if mustParse("5.5.4.68571").Compare(mustParse("5.5.4.68806")) >= 0 { + t.Error("68571 should compare less than 68806") + } + if mustParse("5.5.4.68571").Compare(mustParse("5.5.4.68571")) != 0 { + t.Error("identical builds should compare equal") + } +} diff --git a/tools/db2tool/dbd/fetch.go b/tools/db2tool/dbd/fetch.go new file mode 100644 index 0000000000..309e7cf686 --- /dev/null +++ b/tools/db2tool/dbd/fetch.go @@ -0,0 +1,71 @@ +// Fetch-and-cache for .dbd definitions: fetch from WoWDBDefs master into a +// gitignored cache directory with a 24h-mtime freshness rule. The .dbd files +// themselves are CC BY-SA 4.0 DATA and are deliberately cached, never +// vendored. + +package dbd + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" +) + +const dbdURLFormat = "https://raw.githubusercontent.com/wowdev/WoWDBDefs/master/definitions/%s.dbd" + +// httpClient bounds the fetch so a stalled connection cannot hang make db +// indefinitely; the fallback to a cached copy handles the timeout. +var httpClient = &http.Client{Timeout: 60 * time.Second} + +// FetchCached returns the path to a cached .dbd for tableName under cacheDir, +// fetching from WoWDBDefs when the cached copy is absent or older than 24h. +// On a failed refresh of an existing copy, the stale copy is used; a missing +// copy that cannot be fetched is fatal. +func FetchCached(cacheDir, tableName string) (string, error) { + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", err + } + path := filepath.Join(cacheDir, tableName+".dbd") + info, statErr := os.Stat(path) + if statErr == nil && time.Since(info.ModTime()) < 24*time.Hour { + return path, nil + } + url := fmt.Sprintf(dbdURLFormat, tableName) + if err := download(url, path); err != nil { + if statErr == nil { + fmt.Fprintf(os.Stderr, "db2tool: refresh of %s failed (%v), using cached copy\n", tableName+".dbd", err) + return path, nil + } + return "", fmt.Errorf("fetching %s: %w", url, err) + } + return path, nil +} + +func download(url, path string) error { + resp, err := httpClient.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP %s", resp.Status) + } + tmp := path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return err + } + if _, err := io.Copy(f, resp.Body); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return err + } + return os.Rename(tmp, path) +} diff --git a/tools/db2tool/dbd/select.go b/tools/db2tool/dbd/select.go new file mode 100644 index 0000000000..7a3a9875e5 --- /dev/null +++ b/tools/db2tool/dbd/select.go @@ -0,0 +1,26 @@ +// Version selection for .dbd definitions. +// Derived from DBDefsLib types (https://github.com/wowdev/WoWDBDefs). +// Copyright 2022 WoWDBDefs Contributors. BSD-3-Clause — see tools/db2tool/NOTICES.md. + +package dbd + +import "fmt" + +// SelectVersion returns the LAST versionDefinition (in file order) whose +// Builds list contains an entry with trailing build number == buildNumber. +// +// Exact equality on the trailing build number only — buildRanges and +// layoutHashes are deliberately NOT consulted: WoWDBDefs lists the live +// builds explicitly for every configured table, and failing loud here beats +// silently decoding with a near-miss layout. +func SelectVersion(def DBDefinition, buildNumber uint32) (VersionDefinitions, error) { + for i := len(def.VersionDefinitions) - 1; i >= 0; i-- { + for _, b := range def.VersionDefinitions[i].Builds { + if b.Build == buildNumber { + return def.VersionDefinitions[i], nil + } + } + } + return VersionDefinitions{}, fmt.Errorf( + "build %d not found in the .dbd definition — WoWDBDefs may not contain this build yet; wait for upstream or refresh DBDCache", buildNumber) +} diff --git a/tools/db2tool/main.go b/tools/db2tool/main.go new file mode 100644 index 0000000000..81a814aeab --- /dev/null +++ b/tools/db2tool/main.go @@ -0,0 +1,305 @@ +// db2tool extracts World of Warcraft client data into tools/database/wowsims.db. +// +// The default mode reads the local install named by the settings' BaseDir: +// .build.info picks the build, files come from local CASC +// (root → encoding → .idx → data.NNN → BLTE), .dbd definitions and the +// community listfile are fetched/cached over plain HTTPS. The client's +// DBCache.bin hotfixes for the extracted build are applied to the decoded +// rows; --dbcache pins specific cache files instead of the default +// scan and --no-hotfixes disables the overlay. +// +// With --build (and optionally --db2dir/--dbddir), the offline mode decodes +// pre-extracted .db2 files instead — no install required and no hotfixes +// unless --dbcache is given. +package main + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/wowsims/mop/tools/db2tool/config" + "github.com/wowsims/mop/tools/db2tool/dbd" + "github.com/wowsims/mop/tools/db2tool/sqlite" + "github.com/wowsims/mop/tools/db2tool/tact" + "github.com/wowsims/mop/tools/db2tool/wdc" + _ "modernc.org/sqlite" +) + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "db2tool:", err) + os.Exit(1) + } +} + +type options struct { + settingsFile string + databaseFile string + db2Dir string // offline mode only + dbdDir string // offline mode override + buildNumber uint32 // nonzero → offline mode + dbCaches []string // explicit DBCache files, overriding the default scan + noHotfixes bool // skip hotfix application entirely +} + +// parseArgs scans the args pairwise. --settings/-s and --output/-output/-o +// are aliases, plus the offline-mode and hotfix flags. +func parseArgs(args []string) (options, error) { + opts := options{ + settingsFile: "appsettings.json", + databaseFile: "wowsims.db", + } + for i := 0; i < len(args); i++ { + next := func() (string, error) { + if i+1 < len(args) { + i++ + return args[i], nil + } + return "", fmt.Errorf("flag %s needs a value", args[i]) + } + var err error + switch args[i] { + case "--settings", "-s": + opts.settingsFile, err = next() + case "--output", "-output", "-o": + opts.databaseFile, err = next() + case "--db2dir": + opts.db2Dir, err = next() + case "--dbddir": + opts.dbdDir, err = next() + case "--dbcache": + var f string + if f, err = next(); err == nil { + opts.dbCaches = append(opts.dbCaches, f) + } + case "--no-hotfixes": + opts.noHotfixes = true + case "--build": + var v string + if v, err = next(); err == nil { + // Accept either a bare build number (68571) or a full version + // string (5.5.4.68571), of which only the trailing component + // identifies the build. + n := v + if dot := strings.LastIndexByte(n, '.'); dot >= 0 { + n = n[dot+1:] + } + b, perr := strconv.ParseUint(n, 10, 32) + if perr != nil { + return opts, fmt.Errorf("invalid --build %q: want a build number like 68571 or a version like 5.5.4.68571", v) + } + opts.buildNumber = uint32(b) + } + default: + return opts, fmt.Errorf("unknown argument %q", args[i]) + } + if err != nil { + return opts, err + } + } + return opts, nil +} + +// resolvePath resolves a possibly-relative settings path against the tool +// home directory tools/db2tool — relative settings values like +// "../../assets/db_inputs/basestats" are anchored there. +func resolvePath(toolHome, value string) string { + if filepath.IsAbs(value) { + return value + } + return filepath.Join(toolHome, value) +} + +func run(args []string) error { + opts, err := parseArgs(args) + if err != nil { + return err + } + + settingsAbs, err := filepath.Abs(opts.settingsFile) + if err != nil { + return err + } + settings, err := config.Load(settingsAbs) + if err != nil { + return fmt.Errorf("loading settings: %w", err) + } + if len(settings.Tables) == 0 { + return fmt.Errorf("settings file lists no Tables") + } + + // The tool home is resolved from the working directory, which must be the + // repo root — the same invariant gen_db already has (its ./tools/... and + // ./assets literals). Fail loud otherwise. + toolHome, err := filepath.Abs(filepath.Join("tools", "db2tool")) + if err != nil { + return err + } + if _, err := os.Stat(toolHome); err != nil { + return fmt.Errorf("tools/db2tool not found — run from the repository root (CWD-dependent like gen_db): %w", err) + } + dbdCacheDir := opts.dbdDir + if dbdCacheDir == "" { + dbdCacheDir = filepath.Join(toolHome, "DBDCache") + } + + var buildNumber uint32 + var openTable func(tableName string) (*wdc.Table, error) + + if opts.buildNumber != 0 { + // Offline mode: pre-extracted .db2 files. + buildNumber = opts.buildNumber + db2Dir := opts.db2Dir + if db2Dir == "" { + db2Dir = filepath.Join(toolHome, "dbfilesclient") + } + openTable = func(tableName string) (*wdc.Table, error) { + return wdc.ReadFile(filepath.Join(db2Dir, tableName+".db2")) + } + } else { + // Local-CASC mode (the default): everything from the install. + if settings.Settings.BaseDir == "" { + return fmt.Errorf("settings BaseDir is required (or pass --build for offline mode)") + } + build, err := tact.Open(settings.Settings.BaseDir, settings.Settings.Product) + if err != nil { + return err + } + buildNumber = build.BuildNumber + fmt.Printf("Extracting %s %s (build %d) from local install\n", build.Entry.Product, build.Entry.Version, buildNumber) + + listfile := &tact.Listfile{Path: filepath.Join(toolHome, "listfile.csv")} + if err := listfile.Refresh(); err != nil { + return err + } + + // GameTables: raw bytes, filename casing preserved from settings. + gameTablesOutDir := resolvePath(toolHome, settings.GameTablesOutDirectory) + if err := os.MkdirAll(gameTablesOutDir, 0o755); err != nil { + return err + } + for _, gameTable := range settings.GameTables { + fdid, err := listfile.GetFDID("gametables/" + gameTable + ".txt") + if err != nil { + return err + } + data, err := build.OpenFileByFDID(fdid) + if err != nil { + return fmt.Errorf("gametable %s: %w", gameTable, err) + } + if err := os.WriteFile(filepath.Join(gameTablesOutDir, gameTable+".txt"), data, 0o644); err != nil { + return err + } + } + + // Tables: extract each .db2 to the target directory, then parse it. + // The FDID key uses the RAW settings TargetDirectory value; only the + // on-disk output use is resolved. + targetDirOnDisk := resolvePath(toolHome, settings.TargetDirectory) + if err := os.MkdirAll(targetDirOnDisk, 0o755); err != nil { + return err + } + openTable = func(tableName string) (*wdc.Table, error) { + fdid, err := listfile.GetFDID(settings.TargetDirectory + "/" + tableName + ".db2") + if err != nil { + return nil, err + } + data, err := build.OpenFileByFDID(fdid) + if err != nil { + return nil, fmt.Errorf("table %s: %w", tableName, err) + } + path := filepath.Join(targetDirOnDisk, tableName+".db2") + if err := os.WriteFile(path, data, 0o644); err != nil { + return nil, err + } + return wdc.ReadFile(path) + } + } + + type loaded struct { + def sqlite.TableDef + table *wdc.Table + decoded *wdc.Decoded + } + tables := make([]loaded, 0, len(settings.Tables)) + tableDefs := make([]sqlite.TableDef, 0, len(settings.Tables)) + + for _, tableName := range settings.Tables { + table, err := openTable(tableName) + if err != nil { + return err + } + dbdPath, err := dbd.FetchCached(dbdCacheDir, tableName) + if err != nil { + return err + } + def, err := dbd.ReadFile(dbdPath, true) + if err != nil { + return err + } + version, err := dbd.SelectVersion(def, buildNumber) + if err != nil { + return fmt.Errorf("table %s: %w", tableName, err) + } + decoded, err := table.DecodeRows(def, version, buildNumber) + if err != nil { + return fmt.Errorf("table %s: %w", tableName, err) + } + td := sqlite.TableDef{Name: tableName, Def: def, Version: version} + tables = append(tables, loaded{def: td, table: table, decoded: decoded}) + tableDefs = append(tableDefs, td) + } + + db, err := sqlite.Open(opts.databaseFile) + if err != nil { + return err + } + defer db.Close() + + if err := sqlite.CreateTables(db, tableDefs); err != nil { + return err + } + + // Hotfixes: overlay the client's DBCache.bin records before the inserts. + // Only a cache for this exact build applies; having none is not an error. + // --dbcache pins specific cache files (deterministic runs); with no + // override, local-CASC mode scans tools/db2tool/caches plus + // /**/DBCache.bin, while the offline --build mode stays + // hotfix-free. + var hotfixReader *wdc.HotfixReader + if !opts.noHotfixes { + var readers map[uint32]*wdc.HotfixReader + if len(opts.dbCaches) > 0 { + if readers, err = wdc.CombineHotfixFiles(opts.dbCaches); err != nil { + return err + } + } else if opts.buildNumber == 0 { + if readers, err = wdc.LoadHotfixCaches(filepath.Join(toolHome, "caches"), settings.Settings.BaseDir); err != nil { + return err + } + } + hotfixReader = readers[buildNumber] + if hotfixReader == nil && len(opts.dbCaches) > 0 { + // Pinned caches that hold no records for the extracted build would + // otherwise silently produce a hotfix-free run. + fmt.Fprintf(os.Stderr, "db2tool: warning: none of the given --dbcache files hold hotfixes for build %d; continuing without the overlay\n", buildNumber) + } + } + + for _, t := range tables { + if hotfixReader != nil { + if err := hotfixReader.ApplyHotfixes(t.table, t.def.Def, t.def.Version, buildNumber, t.decoded); err != nil { + return fmt.Errorf("table %s: applying hotfixes: %w", t.def.Name, err) + } + } + if err := sqlite.InsertRows(db, t.def, t.decoded); err != nil { + return err + } + } + + fmt.Println("Processing completed.") + return nil +} diff --git a/tools/db2tool/sqlite/insert.go b/tools/db2tool/sqlite/insert.go new file mode 100644 index 0000000000..2b8fd8fa8f --- /dev/null +++ b/tools/db2tool/sqlite/insert.go @@ -0,0 +1,133 @@ +// Row insertion for the extracted tables. + +package sqlite + +import ( + "database/sql" + "encoding/json" + "fmt" + "math" + "strings" + + "github.com/wowsims/mop/tools/db2tool/wdc" +) + +// InsertRows upserts every decoded row of one table inside one transaction. +// +// Contract notes (the wowsims.db output format): +// - definition order = column order = bind order; +// - relation-column idx_ indexes use table-less names (idx_) +// with IF NOT EXISTS, so only the first table processed with a given +// relation-column name gets the index — tables MUST be processed in +// settings order; +// - relation values of 0 stay 0, never NULLed; +// - arrays serialize as JSON text via encoding/json over plain numeric +// slices (u8 arrays emit [0,0,0], never base64); +// - float scalars bind as the double-widened float32. +func InsertRows(db *sql.DB, t TableDef, decoded *wdc.Decoded) error { + defs := t.Version.Definitions + + columnNames := make([]string, len(defs)) + for i, d := range defs { + columnNames[i] = d.Name + } + + pkColumn := "" + for _, d := range defs { + if d.IsID { + pkColumn = d.Name + break + } + } + if pkColumn == "" { + return fmt.Errorf("table %s has no id column", t.Name) + } + + var cols, vals []string + for _, c := range columnNames { + cols = append(cols, "["+c+"]") + vals = append(vals, "@"+c) + } + + var updates []string + for _, c := range columnNames { + if c != pkColumn { + updates = append(updates, fmt.Sprintf("[%s] = excluded.[%s]", c, c)) + } + } + updateClause := "DO NOTHING" + if len(updates) > 0 { + updateClause = "DO UPDATE SET " + strings.Join(updates, ", ") + } + + upsertSql := fmt.Sprintf("INSERT INTO [%s] (%s) VALUES (%s) ON CONFLICT([%s]) %s;", + t.Name, strings.Join(cols, ", "), strings.Join(vals, ", "), pkColumn, updateClause) + + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + // Relation-column indexes, created before the inserts. + for _, d := range defs { + if d.IsRelation { + stmt := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_%s ON %s (%s);", strings.ToLower(d.Name), t.Name, d.Name) + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("creating relation index on %s.%s: %w", t.Name, d.Name, err) + } + } + } + + stmt, err := tx.Prepare(upsertSql) + if err != nil { + return fmt.Errorf("preparing upsert for %s: %w", t.Name, err) + } + defer stmt.Close() + + args := make([]any, len(defs)) + for _, row := range decoded.Rows { + if len(row.Values) != len(defs) { + return fmt.Errorf("table %s row %d: %d values for %d definitions", t.Name, row.ID, len(row.Values), len(defs)) + } + for i, value := range row.Values { + bound, err := bindValue(value) + if err != nil { + return fmt.Errorf("table %s row %d column %s: %w", t.Name, row.ID, defs[i].Name, err) + } + args[i] = sql.Named(defs[i].Name, bound) + } + if _, err := stmt.Exec(args...); err != nil { + return fmt.Errorf("inserting row %d into %s: %w", row.ID, t.Name, err) + } + } + + return tx.Commit() +} + +// bindValue converts a decoded value into a driver-bindable one. +func bindValue(value any) (any, error) { + switch v := value.(type) { + case nil: + return nil, nil + case int64: + return v, nil + case uint64: + if v > math.MaxInt64 { + return nil, fmt.Errorf("uint64 value %d overflows INTEGER", v) + } + return int64(v), nil + case float32: + return float64(v), nil // REAL binds as the double-widened float32 + case string: + return v, nil + case []int64, []uint64, []float32, []string: + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + return string(b), nil + default: + return nil, fmt.Errorf("unsupported value type %T", value) + } +} diff --git a/tools/db2tool/sqlite/schema.go b/tools/db2tool/sqlite/schema.go new file mode 100644 index 0000000000..ba6719b8a1 --- /dev/null +++ b/tools/db2tool/sqlite/schema.go @@ -0,0 +1,126 @@ +// Package sqlite writes the extracted tables to wowsims.db: the schema (one +// table per .dbd definition, arrays as JSON text plus generated per-element +// columns) and the row inserts. This file is the schema half. +package sqlite + +import ( + "database/sql" + "fmt" + "os" + "strings" + + "github.com/wowsims/mop/tools/db2tool/dbd" +) + +// TableDef pairs a table name with its parsed definition and the version +// block selected for the current build (the caller selects once and both +// schema and inserts use the same block). +type TableDef struct { + Name string + Def dbd.DBDefinition + Version dbd.VersionDefinitions +} + +// Open deletes any pre-existing database file — every run starts from an +// empty file, which is what makes post-patch re-runs and db/ptrdb +// alternation correct — and opens a fresh connection with +// PRAGMA foreign_keys = ON. +func Open(path string) (*sql.DB, error) { + if _, err := os.Stat(path); err == nil { + if err := os.Remove(path); err != nil { + return nil, fmt.Errorf("deleting existing database: %w", err) + } + } + db, err := sql.Open("sqlite", path) + if err != nil { + return nil, err + } + // The writer is single-threaded; a single connection keeps every + // statement on one session. + db.SetMaxOpenConns(1) + if _, err := db.Exec("PRAGMA foreign_keys = ON;"); err != nil { + db.Close() + return nil, err + } + return db, nil +} + +// CreateTables emits the schema for every table, in order, inside one +// transaction. +func CreateTables(db *sql.DB, tables []TableDef) error { + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + for _, t := range tables { + var columnDefinitionsSql []string + var indexSql []string + + for _, def := range t.Version.Definitions { + colDef, ok := t.Def.ColumnDefinitions[def.Name] + if !ok { + return fmt.Errorf("column definition for %s not found in table %s", def.Name, t.Name) + } + + if def.ArrLength == 0 { + sqliteType, err := mapToSQLiteType(colDef.Type) + if err != nil { + return fmt.Errorf("table %s column %s: %w", t.Name, def.Name, err) + } + nullability := "" + if colDef.ForeignTable != "" && colDef.ForeignColumn != "" && !def.IsID { + nullability = " NULL" + } + columnSql := fmt.Sprintf("[%s] %s%s", def.Name, sqliteType, nullability) + if def.IsID { + columnSql += " PRIMARY KEY" + } + columnDefinitionsSql = append(columnDefinitionsSql, columnSql) + + if colDef.ForeignTable != "" && colDef.ForeignColumn != "" { + indexSql = append(indexSql, fmt.Sprintf( + "CREATE INDEX IF NOT EXISTS IX_%s_%s ON [%s] ([%s])", t.Name, def.Name, t.Name, def.Name)) + } + } else { + columnDefinitionsSql = append(columnDefinitionsSql, fmt.Sprintf("[%s] TEXT", def.Name)) + + elementType, err := mapToSQLiteType(colDef.Type) + if err != nil { + return fmt.Errorf("table %s column %s: %w", t.Name, def.Name, err) + } + for i := 0; i < def.ArrLength; i++ { + columnDefinitionsSql = append(columnDefinitionsSql, fmt.Sprintf( + "[%s_%d] %s GENERATED ALWAYS AS (json_extract([%s], '$[%d]')) VIRTUAL", + def.Name, i, elementType, def.Name, i)) + } + } + } + + createTableSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS [%s] (%s);", t.Name, strings.Join(columnDefinitionsSql, ", ")) + if _, err := tx.Exec(createTableSql); err != nil { + return fmt.Errorf("creating table %s: %w", t.Name, err) + } + for _, stmt := range indexSql { + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("creating index on %s: %w", t.Name, err) + } + } + } + + return tx.Commit() +} + +func mapToSQLiteType(colType string) (string, error) { + switch colType { + case "int", "uint": + return "INTEGER", nil + case "float": + return "REAL", nil + case "string", "locstring": + return "TEXT", nil + default: + return "", fmt.Errorf("unsupported type: %s", colType) + } +} diff --git a/tools/db2tool/sqlite/sqlite_test.go b/tools/db2tool/sqlite/sqlite_test.go new file mode 100644 index 0000000000..1833dbacfa --- /dev/null +++ b/tools/db2tool/sqlite/sqlite_test.go @@ -0,0 +1,147 @@ +package sqlite + +import ( + "database/sql" + "path/filepath" + "testing" + + "github.com/wowsims/mop/tools/db2tool/dbd" + "github.com/wowsims/mop/tools/db2tool/wdc" + _ "modernc.org/sqlite" +) + +// modernc driver-marshaling smoke test: creates the extractor's schema +// shapes, upserts via named params, and reads back json_extract virtual +// columns, NULL scans, and REAL vs INTEGER marshaling — a permanent +// regression gate for driver behavior, independent of any game-data +// snapshot. +func TestModerncMarshalingContract(t *testing.T) { + path := filepath.Join(t.TempDir(), "smoke.db") + + def := dbd.DBDefinition{ + ColumnDefinitions: map[string]dbd.ColumnDefinition{ + "ID": {Type: "int"}, + "Name": {Type: "locstring"}, + "Rate": {Type: "float"}, + "Stats": {Type: "int"}, + "Scales": {Type: "float"}, + "ParentID": {Type: "int", ForeignTable: "Other", ForeignColumn: "ID"}, + }, + } + version := dbd.VersionDefinitions{ + Definitions: []dbd.Definition{ + {Name: "ID", Size: 32, IsID: true, IsSigned: true}, + {Name: "Name"}, + {Name: "Rate"}, + {Name: "Stats", Size: 32, ArrLength: 3, IsSigned: true}, + {Name: "Scales", ArrLength: 2}, + {Name: "ParentID", Size: 32, IsSigned: true, IsRelation: true}, + }, + } + td := TableDef{Name: "Smoke", Def: def, Version: version} + + db, err := Open(path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if err := CreateTables(db, []TableDef{td}); err != nil { + t.Fatal(err) + } + + decoded := &wdc.Decoded{ + Rows: []wdc.Row{ + {ID: 1, Values: []any{int64(1), "first", float32(0.581), []int64{1, -2, 3}, []float32{0.1, 0}, int64(0)}}, + {ID: 2, Values: []any{int64(2), "", float32(0), []int64{0, 0, 0}, []float32{0, 0}, int64(7)}}, + // Never upserted below, so it keeps its all-zero arrays. + {ID: 3, Values: []any{int64(3), "", float32(0), []int64{0, 0, 0}, []float32{0, 0}, int64(0)}}, + }, + } + if err := InsertRows(db, td, decoded); err != nil { + t.Fatal(err) + } + // Upsert (same PK) must update, not duplicate. + if err := InsertRows(db, td, &wdc.Decoded{Rows: []wdc.Row{ + {ID: 2, Values: []any{int64(2), "second", float32(1.5), []int64{9, 9, 9}, []float32{2.5, 0}, int64(7)}}, + }}); err != nil { + t.Fatal(err) + } + + var n int + if err := db.QueryRow("SELECT count(*) FROM Smoke").Scan(&n); err != nil { + t.Fatal(err) + } + if n != 3 { + t.Fatalf("expected 3 rows after upsert, got %d", n) + } + + // float32 scalar must store the double-widened value. + var rate float64 + if err := db.QueryRow("SELECT Rate FROM Smoke WHERE ID=1").Scan(&rate); err != nil { + t.Fatal(err) + } + if rate != float64(float32(0.581)) { + t.Errorf("Rate = %v, want double-widened float32 %v", rate, float64(float32(0.581))) + } + + // Arrays: exact JSON text and virtual-column extraction (int and float). + var statsText string + var stats1 int + var scales0 float64 + if err := db.QueryRow("SELECT Stats, Stats_1, Scales_0 FROM Smoke WHERE ID=1").Scan(&statsText, &stats1, &scales0); err != nil { + t.Fatal(err) + } + if statsText != "[1,-2,3]" { + t.Errorf("Stats text = %q, want [1,-2,3]", statsText) + } + if stats1 != -2 { + t.Errorf("Stats_1 = %d, want -2", stats1) + } + // json_extract parses the stored float32 shortest-round-trip TEXT ("0.1") + // as a double — so virtual float columns yield 0.1, NOT the widened + // float32 0.10000000149011612. + if scales0 != 0.1 { + t.Errorf("Scales_0 = %v, want 0.1", scales0) + } + + // The upsert must have replaced row 2's arrays wholesale. + var upsertedStats, upsertedScales string + if err := db.QueryRow("SELECT Stats, Scales FROM Smoke WHERE ID=2").Scan(&upsertedStats, &upsertedScales); err != nil { + t.Fatal(err) + } + if upsertedStats != "[9,9,9]" || upsertedScales != "[2.5,0]" { + t.Errorf("upserted arrays = %q / %q, want [9,9,9] / [2.5,0]", upsertedStats, upsertedScales) + } + + // All-zero arrays serialize as [0,...], never NULL/[]/"" — checked on the + // row that was never upserted. + var zeroStats, zeroScales string + if err := db.QueryRow("SELECT Stats, Scales FROM Smoke WHERE ID=3").Scan(&zeroStats, &zeroScales); err != nil { + t.Fatal(err) + } + if zeroStats != "[0,0,0]" || zeroScales != "[0,0]" { + t.Errorf("all-zero arrays = %q / %q, want [0,0,0] / [0,0]", zeroStats, zeroScales) + } + + // Relation value 0 stays 0 — never converted to NULL. The C# original's + // relation-0-to-NULL branch was dead code (a boxed reference compare). + var parent sql.NullInt64 + if err := db.QueryRow("SELECT ParentID FROM Smoke WHERE ID=1").Scan(&parent); err != nil { + t.Fatal(err) + } + if !parent.Valid || parent.Int64 != 0 { + t.Errorf("ParentID = %+v, want valid 0", parent) + } + + // Schema shape: FK index + relation index + PK + generated columns exist. + for _, wantIdx := range []string{"IX_Smoke_ParentID", "idx_parentid"} { + var cnt int + if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='index' AND name=?", wantIdx).Scan(&cnt); err != nil { + t.Fatal(err) + } + if cnt != 1 { + t.Errorf("index %s missing", wantIdx) + } + } +} diff --git a/tools/db2tool/tact/blte.go b/tools/db2tool/tact/blte.go new file mode 100644 index 0000000000..3d73eaba14 --- /dev/null +++ b/tools/db2tool/tact/blte.go @@ -0,0 +1,103 @@ +// Go translation of TACTSharp's BLTE decoder (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. +// +// Keyless: no TACT keys are loaded, so 'E' (encrypted) chunks are left +// zero-filled in the output — exactly what the WDC layer's encrypted-section +// skip expects. 'F' never occurs. + +package tact + +import ( + "bytes" + "compress/zlib" + "fmt" + "io" +) + +// blteDecode decodes a BLTE blob. totalDecompSize may be 0 (computed from +// chunk infos for multi-chunk files; required for single-chunk non-'N'). +func blteDecode(data []byte, totalDecompSize uint64) ([]byte, error) { + if len(data) < 8 || data[0] != 'B' || data[1] != 'L' || data[2] != 'T' || data[3] != 'E' { + return nil, fmt.Errorf("invalid BLTE header") + } + const fixedHeaderSize = 8 + headerSize := int(be32(data[4:])) + + if headerSize == 0 { + mode := data[fixedHeaderSize] + if mode != 'N' && totalDecompSize == 0 { + return nil, fmt.Errorf("totalDecompSize must be set for single non-normal BLTE block") + } + if mode == 'N' && totalDecompSize == 0 { + totalDecompSize = uint64(len(data) - fixedHeaderSize - 1) + } + out := make([]byte, totalDecompSize) + if err := handleDataBlock(mode, data[fixedHeaderSize+1:], out); err != nil { + return nil, err + } + return out, nil + } + + if data[fixedHeaderSize] != 0xF { + return nil, fmt.Errorf("unexpected BLTE table format 0x%x", data[fixedHeaderSize]) + } + const blockInfoSize = 24 + chunkCount := int(data[fixedHeaderSize+1])<<16 | int(data[fixedHeaderSize+2])<<8 | int(data[fixedHeaderSize+3]) + infoStart := fixedHeaderSize + 4 + + if totalDecompSize == 0 { + o := infoStart + 4 + for range chunkCount { + totalDecompSize += uint64(be32(data[o:])) + o += blockInfoSize + } + } + + out := make([]byte, totalDecompSize) + infoOffset := infoStart + compOffset := headerSize + decompOffset := 0 + + for chunk := range chunkCount { + compSize := int(be32(data[infoOffset:])) + decompSize := int(be32(data[infoOffset+4:])) + if compOffset+compSize > len(data) || decompOffset+decompSize > len(out) { + return nil, fmt.Errorf("BLTE chunk %d out of bounds", chunk) + } + if err := handleDataBlock(data[compOffset], data[compOffset+1:compOffset+compSize], out[decompOffset:decompOffset+decompSize]); err != nil { + return nil, fmt.Errorf("BLTE chunk %d: %w", chunk, err) + } + infoOffset += blockInfoSize + compOffset += compSize + decompOffset += decompSize + } + return out, nil +} + +func handleDataBlock(mode byte, compData, out []byte) error { + switch mode { + case 'N': + copy(out, compData) + return nil + case 'Z': + zr, err := zlib.NewReader(bytes.NewReader(compData)) + if err != nil { + return err + } + defer zr.Close() + _, err = io.ReadFull(zr, out) + return err + case 'E': + // Keyless: leave the output range zero-filled. + return nil + case 'F': + return fmt.Errorf("BLTE frame ('F') decompression not implemented (never occurs in this data)") + default: + return fmt.Errorf("invalid BLTE chunk mode %q", mode) + } +} + +func be32(b []byte) uint32 { + return uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3]) +} diff --git a/tools/db2tool/tact/build.go b/tools/db2tool/tact/build.go new file mode 100644 index 0000000000..e9dd429707 --- /dev/null +++ b/tools/db2tool/tact/build.go @@ -0,0 +1,133 @@ +// Local-CASC build orchestration — the pure-local equivalent of TACTSharp's +// BuildInstance (https://github.com/wowdev/TACTSharp, v0.0.13-alpha, commit +// d0ab516eb98b5db35682467b6e4977d88955046d): FDID → root CKey → encoding EKey +// → local .idx → data.NNN → BLTE. No CDN, no group/file indices (upstream +// consults them but the local .idx always wins for resident files). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +// Package tact reads files out of a local World of Warcraft CASC install: +// .build.info picks the build, then root → encoding → .idx → data.NNN → BLTE +// resolves a file data id to its bytes. There is no CDN fallback. +package tact + +import ( + "encoding/hex" + "fmt" + "path/filepath" +) + +type Build struct { + Entry AvailableBuild + BuildNumber uint32 + + store *cascStore + encoding *encodingTable + root *rootTable +} + +// Open loads everything needed to serve OpenFileByFDID from a local install. +func Open(baseDir, product string) (*Build, error) { + entries, err := ParseBuildInfo(filepath.Join(baseDir, ".build.info")) + if err != nil { + return nil, err + } + entry, err := SelectBuild(entries, product) + if err != nil { + return nil, err + } + buildNumber, err := BuildNumber(entry.Version) + if err != nil { + return nil, err + } + + // Only the build config is needed: it names the encoding and root files. + // The CDN config describes remote archives this local-only reader never + // touches. + buildConfig, err := LoadConfig(baseDir, entry.BuildConfig) + if err != nil { + return nil, fmt.Errorf("loading build config: %w", err) + } + + store, err := openCascStore(baseDir) + if err != nil { + return nil, fmt.Errorf("opening local CASC store: %w", err) + } + + b := &Build{ + Entry: entry, + BuildNumber: buildNumber, + store: store, + } + + // Encoding: the build config's `encoding` line is `ckey ekey`; open by the + // EKey directly (BuildInstance.Load uses encoding[1]). + encodingKeys, ok := buildConfig["encoding"] + if !ok || len(encodingKeys) < 2 { + return nil, fmt.Errorf("no encoding key pair in build config") + } + encodingRaw, err := b.openEKeyHex(encodingKeys[1], 0) + if err != nil { + return nil, fmt.Errorf("opening encoding file: %w", err) + } + if b.encoding, err = parseEncoding(encodingRaw); err != nil { + return nil, err + } + + // Root: config gives the CKey; resolve via encoding. + rootKey, ok := buildConfig["root"] + if !ok || len(rootKey) < 1 { + return nil, fmt.Errorf("no root key in build config") + } + rootCKey, err := hex.DecodeString(rootKey[0]) + if err != nil { + return nil, fmt.Errorf("invalid root ckey: %w", err) + } + rootRaw, err := b.OpenFileByCKey(rootCKey) + if err != nil { + return nil, fmt.Errorf("opening root file: %w", err) + } + if b.root, err = parseRoot(rootRaw); err != nil { + return nil, err + } + + return b, nil +} + +func (b *Build) openEKeyHex(eKeyHex string, decodedSize uint64) ([]byte, error) { + eKey, err := hex.DecodeString(eKeyHex) + if err != nil { + return nil, fmt.Errorf("invalid ekey %q: %w", eKeyHex, err) + } + return b.openEKey(eKey, decodedSize) +} + +func (b *Build) openEKey(eKey []byte, decodedSize uint64) ([]byte, error) { + raw, err := b.store.readEKey(eKey) + if err != nil { + return nil, err + } + return blteDecode(raw, decodedSize) +} + +// OpenFileByCKey resolves a content key through encoding and opens the first +// encoding key locally. +func (b *Build) OpenFileByCKey(cKey []byte) ([]byte, error) { + eKey, decodedSize, ok := b.encoding.findContentKey(cKey) + if !ok { + return nil, fmt.Errorf("ckey %x not found in encoding", cKey) + } + return b.openEKey(eKey, decodedSize) +} + +// OpenFileByFDID opens a file by its file data id via the WoW root. +func (b *Build) OpenFileByFDID(fdid uint32) ([]byte, error) { + cKey, ok := b.root.byFDID[fdid] + if !ok { + return nil, fmt.Errorf("fdid %d not found in root", fdid) + } + data, err := b.OpenFileByCKey(cKey[:]) + if err != nil { + return nil, fmt.Errorf("fdid %d: %w", fdid, err) + } + return data, nil +} diff --git a/tools/db2tool/tact/buildinfo.go b/tools/db2tool/tact/buildinfo.go new file mode 100644 index 0000000000..11bdebeeb1 --- /dev/null +++ b/tools/db2tool/tact/buildinfo.go @@ -0,0 +1,83 @@ +// Go translation of TACTSharp's BuildInfo (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// AvailableBuild is one .build.info product entry, reduced to the fields this +// local-only reader consumes. +type AvailableBuild struct { + BuildConfig string + Version string + Product string +} + +// ParseBuildInfo reads /.build.info (typed pipe format: the header +// line names columns as "Name!TYPE:len") and returns all product entries. +func ParseBuildInfo(path string) ([]AvailableBuild, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var entries []AvailableBuild + headerMap := map[string]int{} + for line := range strings.SplitSeq(strings.ReplaceAll(string(raw), "\r\n", "\n"), "\n") { + if line == "" { + continue + } + split := strings.Split(line, "|") + if strings.HasPrefix(split[0], "Branch!") { + for i, header := range split { + headerMap[strings.Split(header, "!")[0]] = i + } + continue + } + col := func(name string) string { + idx, ok := headerMap[name] + if !ok || idx >= len(split) { + return "" + } + return split[idx] + } + entries = append(entries, AvailableBuild{ + BuildConfig: col("Build Key"), + Version: col("Version"), + Product: col("Product"), + }) + } + if len(entries) == 0 { + return nil, fmt.Errorf("%s: no build entries found", path) + } + return entries, nil +} + +// SelectBuild returns the first entry for the given product. +func SelectBuild(entries []AvailableBuild, product string) (AvailableBuild, error) { + for _, e := range entries { + if e.Product == product { + return e, nil + } + } + return AvailableBuild{}, fmt.Errorf("product %q not found in .build.info", product) +} + +// BuildNumber extracts the trailing build number from a 4-part version +// string. +func BuildNumber(version string) (uint32, error) { + split := strings.Split(version, ".") + if len(split) != 4 { + return 0, fmt.Errorf("invalid build %q", version) + } + n, err := strconv.ParseUint(split[3], 10, 32) + if err != nil { + return 0, fmt.Errorf("invalid build %q: %w", version, err) + } + return uint32(n), nil +} diff --git a/tools/db2tool/tact/cascidx.go b/tools/db2tool/tact/cascidx.go new file mode 100644 index 0000000000..78cc0f801d --- /dev/null +++ b/tools/db2tool/tact/cascidx.go @@ -0,0 +1,176 @@ +// Go translation of TACTSharp's CASCIndexInstance + the local-archive read +// from CDN.TryGetLocalFile (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "bytes" + "encoding/binary" + "fmt" + "os" + "path/filepath" + "strings" +) + +// cascIndex is one parsed .idx bucket file (v7: 9-byte key prefixes, 5-byte +// packed archive/offset, 4-byte size). +type cascIndex struct { + entrySizeBytes int + entryOffsetBytes int + entryKeyBytes int + entries []byte // raw entry block + entrySize int +} + +const cascIdxHeaderSize = 40 + +func loadCascIndex(path string) (*cascIndex, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if len(raw) < cascIdxHeaderSize { + return nil, fmt.Errorf("%s: too small for .idx header", path) + } + // IndexHeader layout: + // u32 headerHashSize, u32 headerHash, u16 version, u8 bucketIndex, + // u8 extraBytes, u8 entrySizeBytes, u8 entryOffsetBytes, u8 entryKeyBytes, + // u8 entryOffsetBits, u64 maxArchiveSize @16, 8 pad, u32 entriesSize @32. + version := binary.LittleEndian.Uint16(raw[8:10]) + if version != 7 { + return nil, fmt.Errorf("%s: unsupported .idx version %d (want 7)", path, version) + } + idx := &cascIndex{ + entrySizeBytes: int(raw[12]), + entryOffsetBytes: int(raw[13]), + entryKeyBytes: int(raw[14]), + } + idx.entrySize = idx.entrySizeBytes + idx.entryOffsetBytes + idx.entryKeyBytes + entriesSize := int(binary.LittleEndian.Uint32(raw[32:36])) + if cascIdxHeaderSize+entriesSize > len(raw) { + return nil, fmt.Errorf("%s: entries block exceeds file size", path) + } + idx.entries = raw[cascIdxHeaderSize : cascIdxHeaderSize+entriesSize] + return idx, nil +} + +// getIndexInfo returns (archiveOffset, size, archiveIndex) for an eKey, with +// the 30-byte per-entry storage frame already skipped (offset+30, size-30), +// or (-1,-1,-1) when absent. +// +// Deviation from upstream, in the safe direction: TACTSharp reports a miss +// whenever lower_bound lands on entry 0 even if it matches; this port accepts +// a genuine entry-0 match (upstream silently falls back to the CDN there — +// this pure-local port has no fallback to hide behind). +func (idx *cascIndex) getIndexInfo(eKey []byte) (int64, int, int) { + needle := eKey[:idx.entryKeyBytes] + n := len(idx.entries) / idx.entrySize + lo, hi := 0, n + for lo < hi { + mid := (lo + hi) / 2 + key := idx.entries[mid*idx.entrySize : mid*idx.entrySize+idx.entryKeyBytes] + if bytes.Compare(key, needle) < 0 { + lo = mid + 1 + } else { + hi = mid + } + } + if lo >= n { + return -1, -1, -1 + } + entry := idx.entries[lo*idx.entrySize : (lo+1)*idx.entrySize] + if !bytes.Equal(entry[:idx.entryKeyBytes], needle) { + return -1, -1, -1 + } + k := idx.entryKeyBytes + indexHigh := int(entry[k]) + indexLow := int(binary.BigEndian.Uint32(entry[k+1 : k+5])) + size := int(binary.LittleEndian.Uint32(entry[k+5:k+5+idx.entrySizeBytes])) - 30 + archiveIndex := indexHigh<<2 | (indexLow&0xC0000000)>>30 + archiveOffset := int64(indexLow&0x3FFFFFFF) + 30 + return archiveOffset, size, archiveIndex +} + +// cascStore is the set of .idx buckets plus the data.NNN archive directory. +type cascStore struct { + dataDir string + buckets map[byte]*cascIndex +} + +// openCascStore loads the highest-version .idx per bucket from +// /Data/data (CDN.LoadCASCIndices). +func openCascStore(baseDir string) (*cascStore, error) { + dataDir := filepath.Join(baseDir, "Data", "data") + entries, err := os.ReadDir(dataDir) + if err != nil { + return nil, err + } + highest := map[byte]int64{} + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".idx") || strings.Contains(name, "tempfile") { + continue + } + stem := strings.TrimSuffix(name, ".idx") + if len(stem) != 10 { + continue + } + var bucket byte + if _, err := fmt.Sscanf(stem[0:2], "%02x", &bucket); err != nil { + continue + } + var version int64 + if _, err := fmt.Sscanf(stem[2:], "%08x", &version); err != nil { + continue + } + if v, ok := highest[bucket]; !ok || version > v { + highest[bucket] = version + } + } + if len(highest) == 0 { + return nil, fmt.Errorf("no .idx files found in %s", dataDir) + } + store := &cascStore{dataDir: dataDir, buckets: map[byte]*cascIndex{}} + for bucket, version := range highest { + path := filepath.Join(dataDir, fmt.Sprintf("%02x%08x.idx", bucket, version)) + idx, err := loadCascIndex(path) + if err != nil { + return nil, err + } + store.buckets[bucket] = idx + } + return store, nil +} + +// bucketForEKey ports CDN.TryGetLocalFile's bucket selection: XOR of the +// first 9 eKey bytes, then fold nibbles. +func bucketForEKey(eKey []byte) byte { + i := eKey[0] ^ eKey[1] ^ eKey[2] ^ eKey[3] ^ eKey[4] ^ eKey[5] ^ eKey[6] ^ eKey[7] ^ eKey[8] + return (i & 0xf) ^ (i >> 4) +} + +// readEKey returns the raw (BLTE-encoded) bytes for an eKey from the local +// archives, or an error when the key is not locally resident. +func (s *cascStore) readEKey(eKey []byte) ([]byte, error) { + idx, ok := s.buckets[bucketForEKey(eKey)] + if !ok { + return nil, fmt.Errorf("no .idx bucket %02x", bucketForEKey(eKey)) + } + offset, size, archiveIndex := idx.getIndexInfo(eKey) + if offset == -1 { + return nil, fmt.Errorf("eKey %x not found in local CASC indices", eKey) + } + archivePath := filepath.Join(s.dataDir, fmt.Sprintf("data.%03d", archiveIndex)) + f, err := os.Open(archivePath) + if err != nil { + return nil, err + } + defer f.Close() + buf := make([]byte, size) + if _, err := f.ReadAt(buf, offset); err != nil { + return nil, fmt.Errorf("reading %s @%d+%d: %w", archivePath, offset, size, err) + } + return buf, nil +} diff --git a/tools/db2tool/tact/config.go b/tools/db2tool/tact/config.go new file mode 100644 index 0000000000..42d91d34ed --- /dev/null +++ b/tools/db2tool/tact/config.go @@ -0,0 +1,38 @@ +// Go translation of TACTSharp's Config (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// LoadConfig reads a build/CDN config from the local install's +// Data/config/// layout. Values are space-separated (typically +// `ckey [ekey]`). All keys are kept, including the ~318 unused `vfs-*` TVFS +// lines — they parse fine and are simply never consulted. +func LoadConfig(baseDir, hash string) (map[string][]string, error) { + if len(hash) != 32 { + return nil, fmt.Errorf("invalid config hash %q", hash) + } + path := filepath.Join(baseDir, "Data", "config", hash[0:2], hash[2:4], hash) + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if len(raw) == 0 || raw[0] != '#' { + return nil, fmt.Errorf("%s: config file is unreadable", path) + } + values := map[string][]string{} + for line := range strings.SplitSeq(string(raw), "\n") { + splitLine := strings.SplitN(line, "=", 2) + if len(splitLine) > 1 { + values[strings.TrimSpace(splitLine[0])] = strings.Split(strings.TrimSpace(splitLine[1]), " ") + } + } + return values, nil +} diff --git a/tools/db2tool/tact/encoding.go b/tools/db2tool/tact/encoding.go new file mode 100644 index 0000000000..08794c329c --- /dev/null +++ b/tools/db2tool/tact/encoding.go @@ -0,0 +1,90 @@ +// Go translation of TACTSharp's EncodingInstance (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "bytes" + "fmt" +) + +// encodingTable supports CKey→EKey resolution over the decoded encoding file +// (the "EN" table: paged, big-endian, 40-bit decoded sizes). +type encodingTable struct { + data []byte + ckeySize int + ekeySize int + pageSize int + pageCount int + headerOff int // ckey page-header block offset + pagesOff int // ckey pages block offset +} + +func parseEncoding(data []byte) (*encodingTable, error) { + if len(data) < 22 || data[0] != 'E' || data[1] != 'N' { + return nil, fmt.Errorf("invalid encoding file magic") + } + if data[2] != 1 { + return nil, fmt.Errorf("unsupported encoding version %d", data[2]) + } + e := &encodingTable{data: data} + e.ckeySize = int(data[3]) + e.ekeySize = int(data[4]) + ckeyPageSize := int(uint16(data[5])<<8|uint16(data[6])) * 1024 + ckeyPageCount := int(be32(data[9:])) + especBlockSize := int(be32(data[0x12:])) + + e.pageSize = ckeyPageSize + e.pageCount = ckeyPageCount + e.headerOff = 22 + especBlockSize + e.pagesOff = e.headerOff + ckeyPageCount*(e.ckeySize+0x10) + if e.pagesOff+ckeyPageCount*ckeyPageSize > len(data) { + return nil, fmt.Errorf("encoding ckey pages exceed file size") + } + return e, nil +} + +// findContentKey returns the first eKey and decoded file size for a cKey, or +// ok=false when absent. Page selection: last page header whose first key <= +// target; then a linear record scan within the page. +func (e *encodingTable) findContentKey(cKey []byte) (eKey []byte, decodedSize uint64, ok bool) { + entrySize := e.ckeySize + 0x10 + n := e.pageCount + // upper_bound on page first-keys, then step back one. + lo, hi := 0, n + for lo < hi { + mid := (lo + hi) / 2 + key := e.data[e.headerOff+mid*entrySize : e.headerOff+mid*entrySize+e.ckeySize] + if bytes.Compare(key, cKey) <= 0 { + lo = mid + 1 + } else { + hi = mid + } + } + pageIndex := lo - 1 + if pageIndex < 0 { + return nil, 0, false + } + + page := e.data[e.pagesOff+pageIndex*e.pageSize : e.pagesOff+(pageIndex+1)*e.pageSize] + for len(page) >= 1+5+e.ckeySize { + keyCount := int(page[0]) + recLen := 5 + e.ckeySize + e.ekeySize*keyCount + if 1+recLen > len(page) { + break + } + rec := page[1 : 1+recLen] + if keyCount == 0 { + page = page[1+recLen:] + continue + } + recCKey := rec[5 : 5+e.ckeySize] + if bytes.Equal(recCKey, cKey) { + size := uint64(rec[0])<<32 | uint64(rec[1])<<24 | uint64(rec[2])<<16 | uint64(rec[3])<<8 | uint64(rec[4]) + return rec[5+e.ckeySize : 5+e.ckeySize+e.ekeySize], size, true + } + page = page[1+recLen:] + } + return nil, 0, false +} diff --git a/tools/db2tool/tact/fdid.go b/tools/db2tool/tact/fdid.go new file mode 100644 index 0000000000..950bc2e2cf --- /dev/null +++ b/tools/db2tool/tact/fdid.go @@ -0,0 +1,144 @@ +// FDID resolution: a static path→FDID map for the configured tables/gametables +// (primary; FDIDs are stable per path), with the community listfile.csv as the +// fallback for paths not in the map. Lookups use plain case-normalized paths. + +package tact + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +// staticFDIDs was generated from the community listfile for the 80 paths the +// generator settings reference (72 dbfilesclient/*.db2 + 8 gametables/*.txt), +// build 5.5.4.68571 snapshot. Keys are lowercase game paths. +var staticFDIDs = map[string]uint32{ + "dbfilesclient/areatable.db2": 1353545, + "dbfilesclient/armorlocation.db2": 1284818, + "dbfilesclient/curve.db2": 892585, + "dbfilesclient/curvepoint.db2": 892586, + "dbfilesclient/difficulty.db2": 1352127, + "dbfilesclient/faction.db2": 1361972, + "dbfilesclient/gemproperties.db2": 1343604, + "dbfilesclient/glyphproperties.db2": 1345274, + "dbfilesclient/item.db2": 841626, + "dbfilesclient/itemarmorquality.db2": 1283021, + "dbfilesclient/itemarmorshield.db2": 1277741, + "dbfilesclient/itemarmortotal.db2": 1283022, + "dbfilesclient/itembonus.db2": 959070, + "dbfilesclient/itemclass.db2": 1140189, + "dbfilesclient/itemdamageammo.db2": 1277740, + "dbfilesclient/itemdamageonehand.db2": 1277743, + "dbfilesclient/itemdamageonehandcaster.db2": 1277739, + "dbfilesclient/itemdamageranged.db2": 6156256, + "dbfilesclient/itemdamagethrown.db2": 6156257, + "dbfilesclient/itemdamagetwohand.db2": 1277738, + "dbfilesclient/itemdamagetwohandcaster.db2": 1277742, + "dbfilesclient/itemdamagewand.db2": 6156258, + "dbfilesclient/itemeffect.db2": 969941, + "dbfilesclient/itemextendedcost.db2": 801681, + "dbfilesclient/itemnamedescription.db2": 1332559, + "dbfilesclient/itemrandomproperties.db2": 1237441, + "dbfilesclient/itemrandomsuffix.db2": 1237592, + "dbfilesclient/itemreforge.db2": 5633983, + "dbfilesclient/itemset.db2": 1343609, + "dbfilesclient/itemsetspell.db2": 1314689, + "dbfilesclient/itemsparse.db2": 1572924, + "dbfilesclient/itemsubclass.db2": 1261604, + "dbfilesclient/itemsubclassmask.db2": 1302852, + "dbfilesclient/itemupgrade.db2": 801687, + "dbfilesclient/journalencounter.db2": 1240336, + "dbfilesclient/journalencounteritem.db2": 1344467, + "dbfilesclient/journalinstance.db2": 1237438, + "dbfilesclient/map.db2": 1349477, + "dbfilesclient/randproppoints.db2": 1310245, + "dbfilesclient/rulesetitemupgrade.db2": 801749, + "dbfilesclient/scalingstatdistribution.db2": 1141728, + "dbfilesclient/skillline.db2": 1240935, + "dbfilesclient/skilllineability.db2": 1266278, + "dbfilesclient/spell.db2": 1140089, + "dbfilesclient/spellauraoptions.db2": 1139952, + "dbfilesclient/spellcategories.db2": 1139939, + "dbfilesclient/spellcategory.db2": 1280619, + "dbfilesclient/spellclassoptions.db2": 979663, + "dbfilesclient/spellcooldowns.db2": 1139924, + "dbfilesclient/spelldescriptionvariables.db2": 1140004, + "dbfilesclient/spellduration.db2": 1137828, + "dbfilesclient/spelleffect.db2": 1140088, + "dbfilesclient/spellequippeditems.db2": 1140011, + "dbfilesclient/spellinterrupts.db2": 1139906, + "dbfilesclient/spellitemenchantment.db2": 1362771, + "dbfilesclient/spelllabel.db2": 1347275, + "dbfilesclient/spelllevels.db2": 1140079, + "dbfilesclient/spellmechanic.db2": 1014438, + "dbfilesclient/spellmisc.db2": 1003144, + "dbfilesclient/spellname.db2": 1990283, + "dbfilesclient/spellpower.db2": 982806, + "dbfilesclient/spellprocsperminute.db2": 1133526, + "dbfilesclient/spellprocsperminutemod.db2": 1133525, + "dbfilesclient/spellradius.db2": 1134584, + "dbfilesclient/spellrange.db2": 1146820, + "dbfilesclient/spellreagents.db2": 841946, + "dbfilesclient/spellscaling.db2": 1139940, + "dbfilesclient/spellshapeshift.db2": 1139929, + "dbfilesclient/spelltargetrestrictions.db2": 1139993, + "dbfilesclient/spellxdescriptionvariables.db2": 1724949, + "dbfilesclient/talent.db2": 1369062, + "dbfilesclient/talenttab.db2": 2178102, + "gametables/chancetomeleecrit.txt": 3999262, + "gametables/chancetomeleecritbase.txt": 3999263, + "gametables/chancetospellcrit.txt": 3999265, + "gametables/chancetospellcritbase.txt": 3999264, + "gametables/combatratings.txt": 1391669, + "gametables/octbasehpbyclass.txt": 5464960, + "gametables/octbasempbyclass.txt": 4049853, + "gametables/spellscaling.txt": 1391660, +} + +// GetFDID resolves a game path (e.g. "dbfilesclient/Spell.db2") to its file +// data id: static map first, then the listfile (loaded lazily). The lookup +// key is the raw game path, lowercased — never a filesystem-resolved path. +func (l *Listfile) GetFDID(path string) (uint32, error) { + key := strings.ToLower(path) + if fdid, ok := staticFDIDs[key]; ok { + return fdid, nil + } + if err := l.load(); err != nil { + return 0, fmt.Errorf("resolving %q: %w", path, err) + } + if fdid, ok := l.byPath[key]; ok { + return fdid, nil + } + return 0, fmt.Errorf("path %q not found in static FDID map or listfile", path) +} + +// load parses the listfile lazily (FDID;path per line). +func (l *Listfile) load() error { + if l.byPath != nil { + return nil + } + f, err := os.Open(l.Path) + if err != nil { + return err + } + defer f.Close() + l.byPath = make(map[string]uint32, 4<<20) + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 1<<20), 1<<20) + for scanner.Scan() { + line := scanner.Text() + sep := strings.IndexByte(line, ';') + if sep < 0 { + continue + } + fdid, err := strconv.ParseUint(line[:sep], 10, 32) + if err != nil { + continue + } + l.byPath[strings.ToLower(line[sep+1:])] = uint32(fdid) + } + return scanner.Err() +} diff --git a/tools/db2tool/tact/listfile.go b/tools/db2tool/tact/listfile.go new file mode 100644 index 0000000000..dacca897f3 --- /dev/null +++ b/tools/db2tool/tact/listfile.go @@ -0,0 +1,98 @@ +// Go translation of TACTSharp's Listfile download/freshness logic +// (https://github.com/wowdev/TACTSharp, v0.0.13-alpha, commit +// d0ab516eb98b5db35682467b6e4977d88955046d). +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "fmt" + "io" + "net" + "net/http" + "os" + "time" +) + +const defaultListfileURL = "https://github.com/wowdev/wow-listfile/releases/latest/download/community-listfile.csv" + +// headClient bounds the freshness probe; downloadClient bounds connect and +// response-header time but NOT the transfer, since the listfile is ~150 MB and +// a slow-but-progressing download must not be cut off. Without these a stalled +// connection would hang make db indefinitely. +var ( + headClient = &http.Client{Timeout: 30 * time.Second} + downloadClient = &http.Client{Transport: &http.Transport{ + DialContext: (&net.Dialer{Timeout: 30 * time.Second}).DialContext, + TLSHandshakeTimeout: 30 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + }} +) + +// Listfile manages the community listfile.csv with download-if-stale +// semantics: HEAD + Last-Modified vs local mtime; a failed freshness check +// triggers a re-download, and a failed download falls back to the existing +// file when one exists. +type Listfile struct { + Path string + URL string + byPath map[string]uint32 +} + +// Refresh ensures the listfile exists and is current. It never deletes a +// usable existing file on network failure — the extractor (via the static +// FDID map) and gen_db's icon map can still run offline. +func (l *Listfile) Refresh() error { + url := l.URL + if url == "" { + url = defaultListfileURL + } + info, statErr := os.Stat(l.Path) + if statErr == nil { + resp, err := headClient.Head(url) + if err == nil { + lastModified, perr := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) + resp.Body.Close() + if perr == nil && !lastModified.After(info.ModTime().UTC()) { + return nil // up to date + } + } + // Stale or check failed: attempt a re-download, but keep the existing + // file if that fails. + if err := l.download(url); err != nil { + fmt.Fprintf(os.Stderr, "db2tool: listfile refresh failed (%v), using existing %s\n", err, l.Path) + } + return nil + } + // No local file: the download must succeed. + if err := l.download(url); err != nil { + return fmt.Errorf("downloading listfile: %w", err) + } + return nil +} + +func (l *Listfile) download(url string) error { + resp, err := downloadClient.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("HTTP %s", resp.Status) + } + tmp := l.Path + ".tmp" + f, err := os.Create(tmp) + if err != nil { + return err + } + if _, err := io.Copy(f, resp.Body); err != nil { + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return err + } + return os.Rename(tmp, l.Path) +} diff --git a/tools/db2tool/tact/root.go b/tools/db2tool/tact/root.go new file mode 100644 index 0000000000..a041f93932 --- /dev/null +++ b/tools/db2tool/tact/root.go @@ -0,0 +1,113 @@ +// Go translation of TACTSharp's RootInstance (https://github.com/wowdev/TACTSharp, +// v0.0.13-alpha, commit d0ab516eb98b5db35682467b6e4977d88955046d) — Normal +// load mode, enUS locale, FDID→CKey only. +// Copyright (c) 2024 Martin Benjamins. MIT License — see tools/db2tool/NOTICES.md. + +package tact + +import ( + "encoding/binary" + "fmt" +) + +const ( + localeEnUS = 0x2 + localeAllWoW = 0x2 | 0x4 | 0x10 | 0x20 | 0x40 | 0x80 | 0x100 | 0x200 | 0x1000 | 0x2000 | 0x4000 | 0x8000 | 0x10000 + contentLowViolence = 0x80 + contentNoNames = 0x10000000 + tsfmMagic = 1296454484 // "TSFM" +) + +// rootTable maps FDID → CKey (first entry wins, like entriesFDID.TryAdd). +type rootTable struct { + byFDID map[uint32][16]byte +} + +func parseRoot(data []byte) (*rootTable, error) { + r := &rootTable{byFDID: map[uint32][16]byte{}} + if len(data) < 12 { + return nil, fmt.Errorf("root file too small") + } + + newRoot := false + dfVersion := uint32(0) + offset := 0 + + if binary.LittleEndian.Uint32(data) == tsfmMagic { + newRoot = true + offset = 12 + totalFiles := binary.LittleEndian.Uint32(data[4:]) + namedFiles := binary.LittleEndian.Uint32(data[8:]) + if namedFiles == 1 || namedFiles == 2 { + // Post-10.1.7 header: [magic, headerSize, dfVersion, ...] + dfHeaderSize := totalFiles + dfVersion = namedFiles + offset = int(dfHeaderSize) + } + _ = totalFiles + } + + for offset < len(data) { + if offset+4 > len(data) { + return nil, fmt.Errorf("truncated root block header at %d", offset) + } + count := int(binary.LittleEndian.Uint32(data[offset:])) + offset += 4 + + var contentFlags, localeFlags uint32 + if dfVersion == 2 { + localeFlags = binary.LittleEndian.Uint32(data[offset:]) + unkFlags := binary.LittleEndian.Uint32(data[offset+4:]) + unkFlags2 := binary.LittleEndian.Uint32(data[offset+8:]) + unkByte := uint32(data[offset+12]) + offset += 13 + contentFlags = unkFlags | unkFlags2 | unkByte<<17 + } else { + contentFlags = binary.LittleEndian.Uint32(data[offset:]) + localeFlags = binary.LittleEndian.Uint32(data[offset+4:]) + offset += 8 + } + + localeSkip := localeFlags&localeAllWoW != localeAllWoW && localeFlags&localeEnUS == 0 + contentSkip := contentFlags&contentLowViolence != 0 + skipChunk := localeSkip || contentSkip + + separateLookup := newRoot + doLookup := !newRoot || contentFlags&contentNoNames == 0 + const sizeFdid, sizeCHash, sizeLookup = 4, 16, 8 + strideCHash := sizeCHash + sizeLookup + if separateLookup { + strideCHash = sizeCHash + } + offsetFdid := offset + offsetCHash := offsetFdid + count*sizeFdid + blockSize := count * (sizeFdid + sizeCHash) + if doLookup { + blockSize += count * sizeLookup + } + if offset+blockSize > len(data) { + return nil, fmt.Errorf("truncated root block at %d (need %d bytes)", offset, blockSize) + } + + if !skipChunk { + fileDataIndex := uint32(0) + for range count { + fdidOffset := binary.LittleEndian.Uint32(data[offsetFdid:]) + offsetFdid += sizeFdid + fdid := fileDataIndex + fdidOffset + fileDataIndex = fdid + 1 + + var md5 [16]byte + copy(md5[:], data[offsetCHash:offsetCHash+sizeCHash]) + offsetCHash += strideCHash + + if _, exists := r.byFDID[fdid]; !exists { + r.byFDID[fdid] = md5 + } + } + } + + offset += blockSize + } + return r, nil +} diff --git a/tools/db2tool/wdc/bitreader.go b/tools/db2tool/wdc/bitreader.go new file mode 100644 index 0000000000..1551f4ff93 --- /dev/null +++ b/tools/db2tool/wdc/bitreader.go @@ -0,0 +1,80 @@ +// Go translation of DBCD.IO's BitReader (https://github.com/wowdev/DBCD, +// v2.1.2, commit 2180edb4d08b3822b3cfa964293ba8ccd4236ac0). +// Copyright (c) 2020 wowdev. MIT License — see tools/db2tool/NOTICES.md. + +package wdc + +import ( + "encoding/binary" +) + +// bitReader reads unaligned little-endian bit windows: a raw 4/8-byte load +// at the current byte, shifted left then right to isolate numBits. Loads can +// extend past the last meaningful byte, so record buffers must carry 8 zero +// bytes of padding (see padRecordData) to keep slice bounds safe. Shift +// counts are masked (&31 / &63) so degenerate widths (0 or full-width) +// behave consistently rather than panicking. +type bitReader struct { + data []byte + Position int // in bits, relative to Offset + Offset int // in bytes +} + +// newBitReader wraps data that MUST already include 8 bytes of zero padding +// beyond the last meaningful byte (see padRecordData). +func newBitReader(data []byte) *bitReader { + return &bitReader{data: data} +} + +// padRecordData appends 8 zero bytes, making unaligned loads at the tail +// safe. The extra bytes are always masked out of results. +func padRecordData(data []byte) []byte { + // Must copy: data may alias the file buffer, and appending in place would + // overwrite the bytes that follow the record block. + out := make([]byte, len(data)+8) + copy(out, data) + return out +} + +func (r *bitReader) ReadUInt32(numBits int) uint32 { + v := binary.LittleEndian.Uint32(r.data[r.Offset+(r.Position>>3):]) + result := v << ((32 - numBits - (r.Position & 7)) & 31) >> ((32 - numBits) & 31) + r.Position += numBits + return result +} + +func (r *bitReader) ReadUInt64(numBits int) uint64 { + v := binary.LittleEndian.Uint64(r.data[r.Offset+(r.Position>>3):]) + result := v << ((64 - numBits - (r.Position & 7)) & 63) >> ((64 - numBits) & 63) + r.Position += numBits + return result +} + +// ReadValue64 returns the raw (zero-extended) bits; the caller reinterprets +// them per the DBD-declared field type. +func (r *bitReader) ReadValue64(numBits int) uint64 { + return r.ReadUInt64(numBits) +} + +// ReadValue64Signed sign-extends a numBits-wide value to 64 bits. +func (r *bitReader) ReadValue64Signed(numBits int) uint64 { + result := r.ReadUInt64(numBits) + signedShift := uint64(1) << ((numBits - 1) & 63) + return (signedShift ^ result) - signedShift +} + +func (r *bitReader) ReadCString() string { + var bytes []byte + for { + num := r.ReadUInt32(8) + if num == 0 { + break + } + bytes = append(bytes, byte(num)) + } + return string(bytes) +} + +// value32 is 4 raw bytes from the pallet/common blocks, reinterpreted by the +// caller per the DBD-declared field type. +type value32 uint32 diff --git a/tools/db2tool/wdc/bitreader_test.go b/tools/db2tool/wdc/bitreader_test.go new file mode 100644 index 0000000000..0d26f1dc24 --- /dev/null +++ b/tools/db2tool/wdc/bitreader_test.go @@ -0,0 +1,166 @@ +package wdc + +import ( + "math" + "testing" +) + +// bitAt returns bit k of the little-endian bit stream: byte i supplies bits +// [8i, 8i+8), least-significant first. This is the definition the WDC5 record +// layout uses and is deliberately independent of bitReader's shift arithmetic. +func bitAt(data []byte, k int) uint64 { + return uint64(data[k/8]>>(k%8)) & 1 +} + +// refRead is the reference extraction: numBits starting at bit position pos. +func refRead(data []byte, pos, numBits int) uint64 { + var v uint64 + for j := range numBits { + v |= bitAt(data, pos+j) << j + } + return v +} + +func testData() []byte { + // Fixed pseudo-random bytes; no Math/rand so failures are reproducible. + data := make([]byte, 32) + x := byte(0x9d) + for i := range data { + data[i] = x + x = x*31 + 17 + } + return padRecordData(data) +} + +// ReadUInt32's shift math requires 32-numBits-(pos&7) >= 0, i.e. numBits <= 25 +// for an arbitrary bit offset. The decoder only ever calls it with small widths +// (pallet indices of cm.B bits, and 8 for ReadCString), so that is the range +// worth pinning. +func TestReadUInt32AllOffsetsAndWidths(t *testing.T) { + data := testData() + for pos := range 64 { + for numBits := 1; numBits <= 25; numBits++ { + r := &bitReader{data: data, Position: pos} + got := uint64(r.ReadUInt32(numBits)) + want := refRead(data, pos, numBits) + if got != want { + t.Fatalf("ReadUInt32(pos=%d, bits=%d) = %#x, want %#x", pos, numBits, got, want) + } + if r.Position != pos+numBits { + t.Fatalf("ReadUInt32(pos=%d, bits=%d) left Position=%d, want %d", pos, numBits, r.Position, pos+numBits) + } + } + } +} + +// ReadUInt64 backs every field read (ReadValue64); its constraint is +// numBits <= 57 for an arbitrary bit offset. +func TestReadUInt64AllOffsetsAndWidths(t *testing.T) { + data := testData() + for pos := range 64 { + for numBits := 1; numBits <= 57; numBits++ { + r := &bitReader{data: data, Position: pos} + got := r.ReadValue64(numBits) + want := refRead(data, pos, numBits) + if got != want { + t.Fatalf("ReadValue64(pos=%d, bits=%d) = %#x, want %#x", pos, numBits, got, want) + } + } + } +} + +// Offset is a byte-granular base that must compose with the bit Position. +func TestReadHonoursByteOffset(t *testing.T) { + data := testData() + for _, offset := range []int{0, 1, 7, 16} { + for _, numBits := range []int{1, 8, 13, 32} { + r := &bitReader{data: data, Offset: offset, Position: 3} + got := r.ReadValue64(numBits) + want := refRead(data, offset*8+3, numBits) + if got != want { + t.Fatalf("Offset=%d bits=%d: got %#x, want %#x", offset, numBits, got, want) + } + } + } +} + +func TestReadValue64Signed(t *testing.T) { + cases := []struct { + bits int + raw uint64 + want int64 + }{ + {8, 0x7f, 127}, + {8, 0x80, -128}, + {8, 0xff, -1}, + {16, 0x7fff, 32767}, + {16, 0x8000, -32768}, + {4, 0x7, 7}, + {4, 0x8, -8}, + {32, 0xffffffff, -1}, + {32, 0x80000000, math.MinInt32}, + } + for _, c := range cases { + // Lay the raw value down at bit 0 of a fresh buffer. + data := make([]byte, 16) + for j := range c.bits { + if c.raw>>j&1 == 1 { + data[j/8] |= 1 << (j % 8) + } + } + r := newBitReader(padRecordData(data)) + if got := int64(r.ReadValue64Signed(c.bits)); got != c.want { + t.Errorf("ReadValue64Signed(%d bits, raw %#x) = %d, want %d", c.bits, c.raw, got, c.want) + } + } +} + +func TestReadCString(t *testing.T) { + data := append([]byte("abc\x00de\x00"), 0) + r := newBitReader(padRecordData(data)) + if got := r.ReadCString(); got != "abc" { + t.Errorf("first ReadCString = %q, want \"abc\"", got) + } + if got := r.ReadCString(); got != "de" { + t.Errorf("second ReadCString = %q, want \"de\"", got) + } + // An immediately-terminated string is empty, not a read past the end. + if got := r.ReadCString(); got != "" { + t.Errorf("third ReadCString = %q, want \"\"", got) + } +} + +// padRecordData must COPY: the records block aliases the mapped file buffer, so +// appending in place would overwrite the bytes that follow it. +func TestPadRecordDataCopies(t *testing.T) { + file := []byte{1, 2, 3, 4, 5, 6, 7, 8} + records := file[:4] + padded := padRecordData(records) + + if len(padded) != len(records)+8 { + t.Fatalf("padded length = %d, want %d", len(padded), len(records)+8) + } + for i, b := range padded[len(records):] { + if b != 0 { + t.Errorf("pad byte %d = %d, want 0", i, b) + } + } + padded[5] = 0xff + if file[5] != 6 { + t.Errorf("padRecordData wrote through to the backing buffer: file[5] = %d, want 6", file[5]) + } +} + +// A read at the very last meaningful byte still loads 8 bytes, which is exactly +// what the padding exists for. +func TestReadAtTailIsInBounds(t *testing.T) { + data := padRecordData([]byte{0xaa}) + r := newBitReader(data) + if got := r.ReadValue64(8); got != 0xaa { + t.Errorf("tail ReadValue64(8) = %#x, want 0xaa", got) + } + r2 := &bitReader{data: data, Offset: 1} + if got := r2.ReadValue64(32); got != 0 { + t.Errorf("read into padding = %#x, want 0", got) + } +} diff --git a/tools/db2tool/wdc/hotfix.go b/tools/db2tool/wdc/hotfix.go new file mode 100644 index 0000000000..76487d7476 --- /dev/null +++ b/tools/db2tool/wdc/hotfix.go @@ -0,0 +1,397 @@ +// Go translation of DBCD.IO's hotfix support (https://github.com/wowdev/DBCD, +// v2.1.2, commit 2180edb4d08b3822b3cfa964293ba8ccd4236ac0), plus the DBCache +// scanning and SStrHash table-name hash from wow.tools.local +// (https://github.com/Marlamin/wow.tools.local). +// Copyright (c) 2020 wowdev; Copyright (c) 2022 Martin Benjamins. +// MIT License — see tools/db2tool/NOTICES.md. + +package wdc + +import ( + "encoding/binary" + "fmt" + "io/fs" + "math" + "os" + "path/filepath" + "slices" + "sort" + "strings" + + "github.com/wowsims/mop/tools/db2tool/dbd" +) + +const hotfixMagic = "XFTH" + +// HotfixEntry is one XFTH v9 record: the HotfixEntryV9 header plus its data +// blob. DataSize always equals len(Data) after parsing; it is kept explicit +// because it is part of the Combine dedup identity. +type HotfixEntry struct { + RegionID int32 + PushID int32 + UniqueID int32 + TableHash uint32 + RecordID int32 + DataSize int32 + IsValid bool // status/op byte == 1 + Data []byte +} + +// hotfixIdentity is the identity Combine dedups on: the 5-tuple plus the +// record's data bytes, so records that differ only in data are both kept — +// full-record identity, not the 5-tuple alone. RegionID/UniqueID never +// participate. +type hotfixIdentity struct { + pushID int32 + tableHash uint32 + recordID int32 + isValid bool + dataSize int32 + data string +} + +func (e *HotfixEntry) identity() hotfixIdentity { + return hotfixIdentity{ + pushID: e.PushID, + tableHash: e.TableHash, + recordID: e.RecordID, + isValid: e.IsValid, + dataSize: e.DataSize, + data: string(e.Data), + } +} + +// HotfixReader holds every hotfix record of one DBCache build, in +// file/combine insertion order. +type HotfixReader struct { + Version int32 + BuildID int32 + records []HotfixEntry +} + +// ReadHotfixFile parses one DBCache-format file. Only XFTH version 9 (the +// current live-client format) is supported; older versions (v1–v8, written +// only by long-obsolete clients) fail loud. +func ReadHotfixFile(path string) (*HotfixReader, error) { + buf, err := os.ReadFile(path) + if err != nil { + return nil, err + } + h, err := parseHotfix(buf) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return h, nil +} + +func parseHotfix(buf []byte) (*HotfixReader, error) { + if len(buf) < 12 { + return nil, fmt.Errorf("hotfix file is corrupted (shorter than the 12-byte header)") + } + if string(buf[0:4]) != hotfixMagic { + return nil, fmt.Errorf("hotfix file is corrupted (bad magic %q)", string(buf[0:4])) + } + h := &HotfixReader{ + Version: int32(binary.LittleEndian.Uint32(buf[4:8])), + BuildID: int32(binary.LittleEndian.Uint32(buf[8:12])), + } + if h.Version != 9 { + return nil, fmt.Errorf("unsupported DBCache version %d (only XFTH v9 is supported)", h.Version) + } + // Version >= 5 extended header: a 32-byte SHA hash, skipped. + if len(buf) < 44 { + return nil, fmt.Errorf("hotfix file is corrupted (shorter than the 44-byte extended header)") + } + pos := 44 + + for pos < len(buf) { + if pos+4 > len(buf) || string(buf[pos:pos+4]) != hotfixMagic { + return nil, fmt.Errorf("hotfix file is corrupted (bad entry magic at offset %d)", pos) + } + pos += 4 + if pos+28 > len(buf) { + return nil, fmt.Errorf("hotfix file is corrupted (truncated entry header at offset %d)", pos) + } + e := HotfixEntry{ + RegionID: int32(binary.LittleEndian.Uint32(buf[pos:])), + PushID: int32(binary.LittleEndian.Uint32(buf[pos+4:])), + UniqueID: int32(binary.LittleEndian.Uint32(buf[pos+8:])), + TableHash: binary.LittleEndian.Uint32(buf[pos+12:]), + RecordID: int32(binary.LittleEndian.Uint32(buf[pos+16:])), + DataSize: int32(binary.LittleEndian.Uint32(buf[pos+20:])), + IsValid: buf[pos+24] == 1, + // buf[pos+25:pos+28] is padding. + } + pos += 28 + if e.DataSize < 0 || pos+int(e.DataSize) > len(buf) { + return nil, fmt.Errorf("hotfix file is corrupted (truncated entry data at offset %d, size %d)", pos, e.DataSize) + } + e.Data = buf[pos : pos+int(e.DataSize) : pos+int(e.DataSize)] + pos += int(e.DataSize) + h.records = append(h.records, e) + } + return h, nil +} + +// Combine merges another reader into this one: other's records are appended +// in order unless an identical record is already present. Readers for a +// different build are ignored. +func (h *HotfixReader) Combine(other *HotfixReader) { + if other.BuildID != h.BuildID { + return + } + lookup := make(map[hotfixIdentity]struct{}, len(h.records)) + for i := range h.records { + lookup[h.records[i].identity()] = struct{}{} + } + for i := range other.records { + id := other.records[i].identity() + if _, ok := lookup[id]; !ok { + h.records = append(h.records, other.records[i]) + lookup[id] = struct{}{} + } + } +} + +// CombineHotfixFiles parses each file in order into readers keyed by BuildId: +// the first file seen for a build becomes its base reader and later files +// Combine into it. +func CombineHotfixFiles(files []string) (map[uint32]*HotfixReader, error) { + readers := make(map[uint32]*HotfixReader) + for _, f := range files { + r, err := ReadHotfixFile(f) + if err != nil { + return nil, err + } + if base, ok := readers[uint32(r.BuildID)]; ok { + base.Combine(r) + } else { + readers[uint32(r.BuildID)] = r + } + fmt.Printf("Loaded hotfixes from %s for build %d\n", f, r.BuildID) + } + return readers, nil +} + +// LoadHotfixCaches scans for cache files: if cachesDir exists, every *.bin +// under it (recursively) loads first, then every file named DBCache.bin +// anywhere under baseDir. Finding no cache file is not an error; a malformed +// or unsupported-version file fails loud. Files are visited in WalkDir's +// deterministic lexical order. +func LoadHotfixCaches(cachesDir, baseDir string) (map[uint32]*HotfixReader, error) { + var files []string + if st, err := os.Stat(cachesDir); err == nil && st.IsDir() { + err := filepath.WalkDir(cachesDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && strings.HasSuffix(d.Name(), ".bin") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + } + err := filepath.WalkDir(baseDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && d.Name() == "DBCache.bin" { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + return CombineHotfixFiles(files) +} + +// SStrHash is the Blizzard SStrHash variant DBCache table hashes use. +// Callers hash the UPPERCASED table name; the result equals the table's WDC5 +// header TableHash (which ApplyHotfixes actually keys on). +func SStrHash(s string) uint32 { + sHashtable := [16]uint32{ + 0x486E26EE, 0xDCAA16B3, 0xE1918EEF, 0x202DAFDB, + 0x341C7DC7, 0x1C365303, 0x40EF2D37, 0x65FD5E49, + 0xD6057177, 0x904ECE93, 0x1C38024F, 0x98FD323B, + 0xE3061AE7, 0xA39B0FA1, 0x9797F25F, 0xE4444563, + } + v := uint32(0x7fed7fed) + x := uint32(0xeeeeeeee) + for i := 0; i < len(s); i++ { + c := uint32(s[i]) + v += x + v ^= sHashtable[(c>>4)&0xf] - sHashtable[c&0xf] + x = x*33 + v + c + 3 + } + return v +} + +// ApplyHotfixes overlays this reader's records for table t onto decoded in +// place. Records apply in a stable ascending-PushId sort (file/combine +// insertion order preserved within a PushId). An Add +// (IsValid && DataSize > 0) replaces or inserts the whole row decoded from +// the blob; otherwise the row is deleted when shouldDelete. Rows come back +// out in ascending-ID order, the order the sqlite inserts expect. +func (h *HotfixReader) ApplyHotfixes(t *Table, def dbd.DBDefinition, version dbd.VersionDefinitions, buildNumber uint32, decoded *Decoded) error { + var recs []*HotfixEntry + for i := range h.records { + if h.records[i].TableHash == t.TableHash { + recs = append(recs, &h.records[i]) + } + } + if len(recs) == 0 { + return nil + } + + plans, err := buildFieldPlans(def, version, buildNumber) + if err != nil { + return err + } + + sort.SliceStable(recs, func(i, j int) bool { return recs[i].PushID < recs[j].PushID }) + + // The shouldDelete carve-out only affects TactKey (0xDF2F53CF) and + // BroadcastText (0x021826BB), neither of which is in Tables[]. + anyValidCached := false + for _, r := range recs { + if r.IsValid && r.PushID == -1 && r.DataSize > 0 { + anyValidCached = true + break + } + } + shouldDelete := (t.TableHash != 0xDF2F53CF && t.TableHash != 0x021826BB) || !anyValidCached + + byID := make(map[int32][]any, len(decoded.Rows)) + for _, row := range decoded.Rows { + byID[row.ID] = row.Values + } + + for _, rec := range recs { + switch { + case rec.IsValid && rec.DataSize > 0: // RowOp.Add + values, err := decodeHotfixRow(t, plans, rec) + if err != nil { + return fmt.Errorf("hotfix record %d (push %d): %w", rec.RecordID, rec.PushID, err) + } + byID[rec.RecordID] = values + case shouldDelete: // RowOp.Delete + delete(byID, rec.RecordID) + } + // else RowOp.Ignore + } + + ids := make([]int32, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + slices.Sort(ids) + + rows := make([]Row, len(ids)) + for i, id := range ids { + rows[i] = Row{ID: id, Values: byID[id]} + } + decoded.Rows = rows + return nil +} + +// decodeHotfixRow decodes one hotfix data blob. Hotfix blobs are NOT +// bitpacked: fields are byte-aligned little-endian values in definition +// order, at their DBD-declared widths, with strings inline null-terminated. +// The non-inline ID is absent from the blob and comes from RecordId; a +// non-inline relation IS in the blob at its DBD-declared type, then +// converted to int. +func decodeHotfixRow(t *Table, plans []fieldPlan, rec *HotfixEntry) ([]any, error) { + r := newBitReader(padRecordData(rec.Data)) + values := make([]any, len(plans)) + + for i := range plans { + p := &plans[i] + + // The record ID replaces the field for a non-inline DBD id, and for + // the id field when the table's Index flag is set. + if p.isNonInlineID || (t.Flags&flagIndex != 0 && i == int(t.IdFieldIndex)) { + values[i] = int64(rec.RecordID) + continue + } + + if p.isNonInlineRel { + if p.arrLength != 0 { + return nil, fmt.Errorf("field %s: non-inline relation arrays are not supported", p.name) + } + if p.hfKind != kindInt { + return nil, fmt.Errorf("field %s: non-integer non-inline relation is not supported", p.name) + } + v, err := toInt32Checked(rawToInt(r.ReadValue64(p.hfSize), p.hfSize, p.hfSigned)) + if err != nil { + return nil, fmt.Errorf("field %s: %w", p.name, err) + } + values[i] = v + continue + } + + if p.arrLength != 0 { + switch p.kind { + case kindString: + out := make([]string, p.arrLength) + for j := range out { + out[j] = r.ReadCString() + } + values[i] = out + case kindFloat: + out := make([]float32, p.arrLength) + for j := range out { + out[j] = math.Float32frombits(uint32(r.ReadValue64(32))) + } + values[i] = out + default: + if p.size == 64 && !p.signed { + out := make([]uint64, p.arrLength) + for j := range out { + out[j] = r.ReadValue64(64) + } + values[i] = out + } else { + out := make([]int64, p.arrLength) + for j := range out { + out[j] = rawToInt(r.ReadValue64(p.size), p.size, p.signed).(int64) + } + values[i] = out + } + } + continue + } + + switch p.kind { + case kindString: + values[i] = r.ReadCString() + case kindFloat: + values[i] = math.Float32frombits(uint32(r.ReadValue64(32))) + default: + values[i] = rawToInt(r.ReadValue64(p.size), p.size, p.signed) + } + } + // The blob is deliberately not validated to be fully consumed. + return values, nil +} + +// toInt32Checked converts to the int32 range: value-preserving, +// overflow-checked. +func toInt32Checked(v any) (int64, error) { + switch x := v.(type) { + case int64: + if x < math.MinInt32 || x > math.MaxInt32 { + return 0, fmt.Errorf("relation value %d overflows int32", x) + } + return x, nil + case uint64: + if x > math.MaxInt32 { + return 0, fmt.Errorf("relation value %d overflows int32", x) + } + return int64(x), nil + } + return 0, fmt.Errorf("relation value has unexpected type %T", v) +} diff --git a/tools/db2tool/wdc/hotfix_test.go b/tools/db2tool/wdc/hotfix_test.go new file mode 100644 index 0000000000..43385a5f0b --- /dev/null +++ b/tools/db2tool/wdc/hotfix_test.go @@ -0,0 +1,170 @@ +package wdc + +import ( + "encoding/binary" + "testing" +) + +// The two table hashes ApplyHotfixes special-cases are known constants, so they +// pin the SStrHash port without needing any client data. A DBCache record is +// matched to a table by comparing this hash against the WDC5 header's +// TableHash, so a wrong hash silently drops every hotfix for a table. +func TestSStrHashKnownTableHashes(t *testing.T) { + cases := map[string]uint32{ + "TACTKEY": 0xDF2F53CF, + "BROADCASTTEXT": 0x021826BB, + } + for name, want := range cases { + if got := SStrHash(name); got != want { + t.Errorf("SStrHash(%q) = 0x%08X, want 0x%08X", name, got, want) + } + } + // Callers must uppercase; the hash is case-sensitive and the lowercase form + // is a different value. + if SStrHash("tactkey") == SStrHash("TACTKEY") { + t.Error("SStrHash is unexpectedly case-insensitive") + } + if SStrHash("") != 0x7fed7fed { + t.Errorf("SStrHash(\"\") = 0x%08X, want the 0x7fed7fed seed", SStrHash("")) + } +} + +// xfthEntry builds one DBCache record: per-entry magic, the 28-byte header, then +// the payload. +func xfthEntry(pushID int32, tableHash uint32, recordID int32, status byte, data []byte) []byte { + b := make([]byte, 0, 32+len(data)) + b = append(b, []byte(hotfixMagic)...) + var hdr [28]byte + binary.LittleEndian.PutUint32(hdr[0:], 1) // RegionID + binary.LittleEndian.PutUint32(hdr[4:], uint32(pushID)) + binary.LittleEndian.PutUint32(hdr[8:], 42) // UniqueID + binary.LittleEndian.PutUint32(hdr[12:], tableHash) + binary.LittleEndian.PutUint32(hdr[16:], uint32(recordID)) + binary.LittleEndian.PutUint32(hdr[20:], uint32(len(data))) + hdr[24] = status + b = append(b, hdr[:]...) + return append(b, data...) +} + +func xfthFile(version, build int32, entries ...[]byte) []byte { + buf := make([]byte, 44) + copy(buf, hotfixMagic) + binary.LittleEndian.PutUint32(buf[4:], uint32(version)) + binary.LittleEndian.PutUint32(buf[8:], uint32(build)) + // buf[12:44] is the v>=5 32-byte hash, skipped by the parser. + for _, e := range entries { + buf = append(buf, e...) + } + return buf +} + +func TestParseHotfixV9(t *testing.T) { + const build = 68571 + raw := xfthFile(9, build, + xfthEntry(100, 0xAABBCCDD, 7, 1, []byte{1, 2, 3, 4}), + xfthEntry(101, 0xAABBCCDD, 8, 0, nil), // delete: status 0, no payload + ) + + h, err := parseHotfix(raw) + if err != nil { + t.Fatal(err) + } + if h.Version != 9 || h.BuildID != build { + t.Fatalf("header = version %d build %d, want 9 / %d", h.Version, h.BuildID, build) + } + if len(h.records) != 2 { + t.Fatalf("parsed %d records, want 2", len(h.records)) + } + + first := h.records[0] + if first.PushID != 100 || first.TableHash != 0xAABBCCDD || first.RecordID != 7 { + t.Errorf("first record = %+v", first) + } + if !first.IsValid { + t.Error("status byte 1 must parse as valid") + } + if first.DataSize != 4 || string(first.Data) != "\x01\x02\x03\x04" { + t.Errorf("first payload = %v (size %d), want 4 bytes 1..4", first.Data, first.DataSize) + } + if h.records[1].IsValid { + t.Error("status byte 0 must parse as invalid") + } + if h.records[1].DataSize != 0 { + t.Errorf("second DataSize = %d, want 0", h.records[1].DataSize) + } +} + +// The payload must be capped so a later append cannot reach into the following +// record's bytes. +func TestParseHotfixPayloadIsCapped(t *testing.T) { + raw := xfthFile(9, 1, + xfthEntry(1, 0x1, 1, 1, []byte{9, 9}), + xfthEntry(2, 0x1, 2, 1, []byte{7, 7}), + ) + h, err := parseHotfix(raw) + if err != nil { + t.Fatal(err) + } + _ = append(h.records[0].Data, 0xff) + if h.records[1].Data[0] != 7 { + t.Errorf("appending to record 0's payload corrupted record 1: %v", h.records[1].Data) + } +} + +func TestParseHotfixRejectsBadInput(t *testing.T) { + badMagic := xfthFile(9, 1) + copy(badMagic, "ZZZZ") + + tests := map[string][]byte{ + "short header": []byte("XFT"), + "bad magic": badMagic, + "truncated ext": append([]byte(hotfixMagic), 0x09, 0, 0, 0, 0x01, 0, 0, 0), + "bad entry magic": append(xfthFile(9, 1), []byte("NOPE????")...), + "truncated data": append(xfthFile(9, 1), xfthEntry(1, 0x1, 1, 1, []byte{1, 2, 3, 4})[:34]...), + "unsupported ver8": xfthFile(8, 1), + } + + for name, raw := range tests { + if _, err := parseHotfix(raw); err == nil { + t.Errorf("%s: expected an error, got nil", name) + } + } +} + +// Records with an unsupported DataSize must not be silently accepted: a +// negative size would otherwise index backwards. +func TestParseHotfixRejectsNegativeDataSize(t *testing.T) { + e := xfthEntry(1, 0x1, 1, 1, nil) + binary.LittleEndian.PutUint32(e[4+20:], 0xFFFFFFFF) // DataSize = -1 + if _, err := parseHotfix(xfthFile(9, 1, e)); err == nil { + t.Error("expected an error for a negative DataSize") + } +} + +func TestCombineDedupsOnFullRecordIdentity(t *testing.T) { + rec := func(push int32, data []byte) []byte { return xfthEntry(push, 0x1, 5, 1, data) } + + base, err := parseHotfix(xfthFile(9, 1, rec(1, []byte{1}))) + if err != nil { + t.Fatal(err) + } + // Identical record: dropped. + same, _ := parseHotfix(xfthFile(9, 1, rec(1, []byte{1}))) + base.Combine(same) + if len(base.records) != 1 { + t.Fatalf("identical record was not deduped: %d records", len(base.records)) + } + // Same 5-tuple but different payload bytes: kept, because identity includes + // the data. + other, _ := parseHotfix(xfthFile(9, 1, rec(1, []byte{2}))) + base.Combine(other) + if len(base.records) != 2 { + t.Fatalf("record differing only in payload was dropped: %d records", len(base.records)) + } + // A different build is ignored entirely. + otherBuild, _ := parseHotfix(xfthFile(9, 2, rec(9, []byte{3}))) + base.Combine(otherBuild) + if len(base.records) != 2 { + t.Fatalf("record from another build was merged in: %d records", len(base.records)) + } +} diff --git a/tools/db2tool/wdc/row.go b/tools/db2tool/wdc/row.go new file mode 100644 index 0000000000..13d310f163 --- /dev/null +++ b/tools/db2tool/wdc/row.go @@ -0,0 +1,425 @@ +// Go translation of DBCD.IO's row decoding, DBD-to-field-type mapping and +// copy-row semantics (https://github.com/wowdev/DBCD, v2.1.2, commit +// 2180edb4d08b3822b3cfa964293ba8ccd4236ac0). +// Copyright (c) 2020 wowdev. MIT License — see tools/db2tool/NOTICES.md. + +package wdc + +import ( + "fmt" + "math" + "slices" + + "github.com/wowsims/mop/tools/db2tool/dbd" +) + +type colKind int + +const ( + kindInt colKind = iota + kindFloat + kindString // string, or locstring when locStringSize == 1 (always true for MoP builds) +) + +// fieldPlan is the precomputed per-definition decode plan. +type fieldPlan struct { + name string + kind colKind + size int // int bit width from the DBD (8/16/32/64); 0 for float/string + signed bool + arrLength int + isNonInlineRel bool + isNonInlineID bool + isID bool + + // The DBD-declared view, used only by the hotfix decoder: a non-inline + // relation is read from a hotfix blob at its DBD-declared type (then + // converted to int), while kind/size/signed above carry the int32 + // override. Identical to kind/size/signed for every other field. + hfKind colKind + hfSize int + hfSigned bool +} + +// Row is one decoded record; Values align 1:1 with the version block's +// Definitions, which is also the column order the sqlite writer binds. +// Value dynamic types are limited to int64/uint64 scalars, float32, string, +// []int64, []uint64, []float32 and []string — never []byte, so encoding/json +// writes every array element-by-element as numbers (no base64), the +// wowsims.db array-text format. +type Row struct { + ID int32 + Values []any +} + +type Decoded struct { + Rows []Row // ascending ID +} + +func buildFieldPlans(def dbd.DBDefinition, version dbd.VersionDefinitions, buildNumber uint32) ([]fieldPlan, error) { + // The locstring size is 1 for post-wotlk builds (build > 12340) — always + // the case for the builds this tool targets. A locstring therefore maps + // to a single string field with no _mask column. + if buildNumber <= 12340 { + return nil, fmt.Errorf("build %d predates single-locale locstrings; this port only supports locStringSize == 1", buildNumber) + } + + plans := make([]fieldPlan, len(version.Definitions)) + for i, d := range version.Definitions { + col, ok := def.ColumnDefinitions[d.Name] + if !ok { + return nil, fmt.Errorf("column definition for %q not found", d.Name) + } + p := fieldPlan{ + name: d.Name, + arrLength: d.ArrLength, + isNonInlineRel: d.IsRelation && d.IsNonInline, + isNonInlineID: d.IsID && d.IsNonInline, + isID: d.IsID, + } + switch col.Type { + case "int", "uint": + p.kind = kindInt + p.size = d.Size + p.signed = d.IsSigned + switch d.Size { + case 8, 16, 32, 64: + default: + return nil, fmt.Errorf("column %q: unsupported int size %d", d.Name, d.Size) + } + case "float": + p.kind = kindFloat + case "string", "locstring": + p.kind = kindString + default: + return nil, fmt.Errorf("column %q: unable to construct field type from %q", d.Name, col.Type) + } + // Capture the DBD-declared mapping before the non-inline-relation + // override — the hotfix decoder reads that type. + p.hfKind, p.hfSize, p.hfSigned = p.kind, p.size, p.signed + // A non-inline relation always decodes as a signed 32-bit int, + // regardless of the DBD-declared type. + if p.isNonInlineRel { + p.kind = kindInt + p.size = 32 + p.signed = true + } + plans[i] = p + } + return plans, nil +} + +// DecodeRows decodes every record (including copy-table duplicates) into +// values aligned with version.Definitions, returned in ascending-ID order. +func (t *Table) DecodeRows(def dbd.DBDefinition, version dbd.VersionDefinitions, buildNumber uint32) (*Decoded, error) { + plans, err := buildFieldPlans(def, version, buildNumber) + if err != nil { + return nil, err + } + + byID := make(map[int32][]any, len(t.rows)) + + hadInlineID := false + for _, row := range t.rows { + id, values, err := t.decodeRow(row, plans) + if err != nil { + return nil, fmt.Errorf("record %d: %w", row.recordIndex, err) + } + if row.id == -1 { + hadInlineID = true + } + if _, dup := byID[id]; dup { + return nil, fmt.Errorf("duplicate row id %d", id) + } + byID[id] = values + } + + // Copy-table rows: clone the source row's decoded values and rewrite the + // id field. + if len(t.copyData) > 0 { + if hadInlineID { + // Never occurs on real data, and the id-field mapping would be + // ambiguous; refuse rather than guess. + return nil, fmt.Errorf("copy table present on a table with inline ids — unsupported") + } + idFieldIndex := int(t.IdFieldIndex) + if idFieldIndex >= len(plans) { + return nil, fmt.Errorf("IdFieldIndex %d out of range for %d definitions", idFieldIndex, len(plans)) + } + // Cloning the source row's decoded values is only equivalent to + // upstream's re-decode-with-the-destination-id while no column is + // COMMON-compressed: a common value is looked up BY ROW ID + // (getFieldRaw), so a copy row would wrongly inherit the source id's + // value. No table has both today; fail loud if a future build changes + // that rather than emit silently wrong rows. + for i := range t.ColumnMeta { + if t.ColumnMeta[i].CompressionType == compressionCommon { + return nil, fmt.Errorf("table has both a copy table and a COMMON-compressed column (field %d) — copy rows would resolve common data by the source id; decode copy rows per destination id instead", i) + } + } + for _, ce := range t.copyData { + src, ok := byID[ce.Src] + if !ok { + return nil, fmt.Errorf("copy-table source row %d not found (dest %d)", ce.Src, ce.Dest) + } + if _, dup := byID[ce.Dest]; dup { + return nil, fmt.Errorf("duplicate row id %d from copy table", ce.Dest) + } + values := make([]any, len(src)) + copy(values, src) + values[idFieldIndex] = int64(ce.Dest) + byID[ce.Dest] = values + } + } + + ids := make([]int32, 0, len(byID)) + for id := range byID { + ids = append(ids, id) + } + slices.Sort(ids) + + decoded := &Decoded{Rows: make([]Row, len(ids))} + for i, id := range ids { + decoded.Rows[i] = Row{ID: id, Values: byID[id]} + } + return decoded, nil +} + +// decodeRow decodes one raw record into values aligned with plans. +func (t *Table) decodeRow(row rawRow, plans []fieldPlan) (int32, []any, error) { + r := row.data + r.Position = row.dataPos + r.Offset = row.dataOffset + + id := row.id + values := make([]any, len(plans)) + indexFieldOffset := 0 + + for i, p := range plans { + if i == int(t.IdFieldIndex) { + if id != -1 { + indexFieldOffset++ + } else { + raw, err := t.getFieldRaw(0, r, i) + if err != nil { + return 0, nil, fmt.Errorf("field %s: %w", p.name, err) + } + id = int32(uint32(raw)) + } + values[i] = int64(id) + continue + } + + fieldIndex := i - indexFieldOffset + + if fieldIndex >= len(t.Meta) { + // Trailing non-inline relation: the parent-lookup refID. + values[i] = int64(row.refID) + continue + } + + var err error + if p.arrLength != 0 { + values[i], err = t.readArrayField(r, fieldIndex, p, row) + } else { + values[i], err = t.readScalarField(id, r, fieldIndex, p, row) + } + if err != nil { + return 0, nil, fmt.Errorf("field %s: %w", p.name, err) + } + } + + return id, values, nil +} + +func (t *Table) readScalarField(id int32, r *bitReader, fieldIndex int, p fieldPlan, row rawRow) (any, error) { + if p.kind == kindString { + if t.Flags&flagSparse != 0 { + return r.ReadCString(), nil + } + // The byte position is captured BEFORE the relative offset is read — + // the offset is relative to the field's own position. + recordOffset := (int(row.recordIndex) * int(t.RecordSize)) - (int(t.RecordsCount) * int(t.RecordSize)) + bytePos := r.Position >> 3 + raw, err := t.getFieldRaw(id, r, fieldIndex) + if err != nil { + return nil, err + } + index := max(recordOffset+bytePos+int(int32(uint32(raw))), 0) + s, ok := t.StringTable[int64(index)] + if !ok { + return nil, fmt.Errorf("string table miss at offset %d", index) + } + return s, nil + } + + raw, err := t.getFieldRaw(id, r, fieldIndex) + if err != nil { + return nil, err + } + if p.kind == kindFloat { + return math.Float32frombits(uint32(raw)), nil + } + return rawToInt(raw, p.size, p.signed), nil +} + +// rawToInt reinterprets the low bits of the 64-bit read per the DBD-declared +// width and signedness. Unsigned 64-bit stays uint64; every other case fits +// int64. +func rawToInt(raw uint64, size int, signed bool) any { + switch size { + case 8: + if signed { + return int64(int8(raw)) + } + return int64(uint8(raw)) + case 16: + if signed { + return int64(int16(raw)) + } + return int64(uint16(raw)) + case 32: + if signed { + return int64(int32(raw)) + } + return int64(uint32(raw)) + case 64: + if signed { + return int64(raw) + } + return raw + } + panic(fmt.Sprintf("unsupported int size %d", size)) // guarded in buildFieldPlans +} + +func (t *Table) readArrayField(r *bitReader, fieldIndex int, p fieldPlan, row rawRow) (any, error) { + fm := t.Meta[fieldIndex] + cm := t.ColumnMeta[fieldIndex] + + if p.kind == kindString { + if t.Flags&flagSparse != 0 { + // No configured table has string arrays in a sparse table. + return nil, fmt.Errorf("string arrays in sparse tables are not supported") + } + if cm.CompressionType != compressionNone { + return nil, fmt.Errorf("unexpected compression type %d for string array", cm.CompressionType) + } + bitSize := 32 - int(fm.Bits) + if bitSize <= 0 { + bitSize = int(cm.B) + } + count := int(cm.Size) / 32 + recordOffset := (int(row.recordIndex) * int(t.RecordSize)) - (int(t.RecordsCount) * int(t.RecordSize)) + out := make([]string, count) + for i := range out { + bytePos := r.Position >> 3 + raw := r.ReadValue64(bitSize) + index := max(bytePos+recordOffset+int(int32(uint32(raw))), 0) + s, ok := t.StringTable[int64(index)] + if !ok { + return nil, fmt.Errorf("string table miss at offset %d", index) + } + out[i] = s + } + return out, nil + } + + elemBits := 32 + if p.kind == kindInt { + elemBits = p.size + } + + var raws []uint64 + switch cm.CompressionType { + case compressionNone: + bitSize := 32 - int(fm.Bits) + if bitSize <= 0 { + bitSize = int(cm.B) + } + count := int(cm.Size) / elemBits + raws = make([]uint64, count) + for i := range raws { + raws[i] = r.ReadValue64(bitSize) + } + case compressionPalletArray: + cardinality := int(cm.C) + idx := int(r.ReadUInt32(int(cm.B))) + pallet := t.PalletData[fieldIndex] + raws = make([]uint64, cardinality) + for i := range raws { + pi := i + cardinality*idx + if pi < 0 || pi >= len(pallet) { + return nil, fmt.Errorf("pallet-array index %d out of range (%d entries)", pi, len(pallet)) + } + raws[i] = uint64(uint32(t.PalletData[fieldIndex][pi])) + } + default: + return nil, fmt.Errorf("unexpected compression type %d for array field", cm.CompressionType) + } + + if p.kind == kindFloat { + out := make([]float32, len(raws)) + for i, raw := range raws { + out[i] = math.Float32frombits(uint32(raw)) + } + return out, nil + } + + // Byte-sized arrays must stay plain numeric slices (never []byte): the + // wowsims.db array-text format is element-by-element numbers, e.g. + // [0,0,0], NOT base64. Width/sign truncation still follows the + // DBD-declared element type. + if p.size == 64 && !p.signed { + out := make([]uint64, len(raws)) + copy(out, raws) + return out, nil + } + out := make([]int64, len(raws)) + for i, raw := range raws { + out[i] = rawToInt(raw, p.size, p.signed).(int64) + } + return out, nil +} + +// getFieldRaw dispatches on the column's compression type, returning the +// raw 64-bit value before type reinterpretation. +func (t *Table) getFieldRaw(id int32, r *bitReader, fieldIndex int) (uint64, error) { + fm := t.Meta[fieldIndex] + cm := t.ColumnMeta[fieldIndex] + + switch cm.CompressionType { + case compressionNone: + bitSize := 32 - int(fm.Bits) + if bitSize <= 0 { + bitSize = int(cm.B) // Immediate.BitWidth + } + return r.ReadValue64(bitSize), nil + case compressionSignedImmediate: + return r.ReadValue64Signed(int(cm.B)), nil + case compressionImmediate: + return r.ReadValue64(int(cm.B)), nil + case compressionCommon: + if v, ok := t.CommonData[fieldIndex][id]; ok { + return uint64(uint32(v)), nil + } + return uint64(uint32(cm.A)), nil // Common.DefaultValue raw bytes + case compressionPallet: + idx := int(r.ReadUInt32(int(cm.B))) + pallet := t.PalletData[fieldIndex] + if idx < 0 || idx >= len(pallet) { + return 0, fmt.Errorf("pallet index %d out of range (%d entries)", idx, len(pallet)) + } + return uint64(uint32(pallet[idx])), nil + case compressionPalletArray: + if cm.C != 1 { // Pallet.Cardinality + return 0, fmt.Errorf("unexpected compression type %d (pallet-array cardinality %d on scalar field)", cm.CompressionType, cm.C) + } + idx := int(r.ReadUInt32(int(cm.B))) + pallet := t.PalletData[fieldIndex] + if idx < 0 || idx >= len(pallet) { + return 0, fmt.Errorf("pallet-array index %d out of range (%d entries)", idx, len(pallet)) + } + return uint64(uint32(pallet[idx])), nil + } + return 0, fmt.Errorf("unexpected compression type %d", cm.CompressionType) +} diff --git a/tools/db2tool/wdc/wdc5.go b/tools/db2tool/wdc/wdc5.go new file mode 100644 index 0000000000..2ae2010f7b --- /dev/null +++ b/tools/db2tool/wdc/wdc5.go @@ -0,0 +1,578 @@ +// Go translation of DBCD.IO's WDC5Reader (https://github.com/wowdev/DBCD, +// v2.1.2, commit 2180edb4d08b3822b3cfa964293ba8ccd4236ac0), including its +// encrypted-section skip path (no TACT keys). +// Copyright (c) 2020 wowdev. MIT License — see tools/db2tool/NOTICES.md. + +// Package wdc decodes WDC5 .db2 client tables into rows shaped by a .dbd +// definition, and overlays the client's XFTH DBCache hotfix records onto them. +package wdc + +import ( + "encoding/binary" + "fmt" + "os" + "strings" +) + +const wdc5Magic = "WDC5" +const headerSize = 200 + +type db2Flags uint16 + +const ( + flagSparse db2Flags = 0x1 + flagSecondaryKey db2Flags = 0x2 + flagIndex db2Flags = 0x4 +) + +const ( + compressionNone = 0 + compressionImmediate = 1 + compressionCommon = 2 + compressionPallet = 3 + compressionPalletArray = 4 + compressionSignedImmediate = 5 +) + +type fieldMeta struct { + Bits int16 + Offset int16 +} + +// columnMeta's A/B/C are the 12-byte union interpreted per CompressionType: +// Immediate{BitOffset,BitWidth,Flags} / Pallet{BitOffset,BitWidth,Cardinality} / +// Common{DefaultValue,B,C}. +type columnMeta struct { + RecordOffset uint16 + Size uint16 + AdditionalDataSize uint32 + CompressionType uint32 + A, B, C int32 +} + +type sectionHeader struct { + TactKeyLookup uint64 + FileOffset int32 + NumRecords int32 + StringTableSize int32 + OffsetRecordsEndOffset int32 + IndexDataSize int32 + ParentLookupDataSize int32 + OffsetMapIDCount int32 + CopyTableCount int32 +} + +type sparseEntry struct { + Offset uint32 + Size uint16 +} + +// rawRow is a not-yet-decoded record: a bit reader positioned at its data, +// plus the row's identity captured at construction. +type rawRow struct { + data *bitReader + dataOffset int + dataPos int + id int32 // -1 when the id is inline in record data + refID int32 + recordIndex int32 +} + +type copyEntry struct { + Dest int32 + Src int32 +} + +// Table is the parsed (but not field-decoded) WDC5 file. Header fields the +// decoder does not consume (schema version/string, layout hash, max index, +// locale) are read past rather than kept. +type Table struct { + RecordsCount int32 + FieldsCount int32 + RecordSize int32 + StringTableSize int32 + TableHash uint32 + MinIndex int32 + Flags db2Flags + IdFieldIndex uint16 + + Sections []sectionHeader + Meta []fieldMeta + ColumnMeta []columnMeta + PalletData [][]value32 + CommonData []map[int32]value32 + + StringTable map[int64]string + + rows []rawRow + copyData []copyEntry // file order; dest==src entries already dropped + + // SkippedSections counts encrypted sections whose data was zero-filled and + // therefore skipped, which is why the header record count exceeds the number + // of emitted rows. + SkippedSections int +} + +type cursor struct { + buf []byte + pos int +} + +func (c *cursor) need(n int) ([]byte, error) { + if c.pos+n > len(c.buf) { + return nil, fmt.Errorf("unexpected EOF: need %d bytes at offset %d, file is %d bytes", n, c.pos, len(c.buf)) + } + b := c.buf[c.pos : c.pos+n] + c.pos += n + return b, nil +} + +func (c *cursor) u16() (uint16, error) { + b, err := c.need(2) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint16(b), nil +} + +func (c *cursor) i32() (int32, error) { + b, err := c.need(4) + if err != nil { + return 0, err + } + return int32(binary.LittleEndian.Uint32(b)), nil +} + +func (c *cursor) u32() (uint32, error) { + b, err := c.need(4) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint32(b), nil +} + +func (c *cursor) u64() (uint64, error) { + b, err := c.need(8) + if err != nil { + return 0, err + } + return binary.LittleEndian.Uint64(b), nil +} + +// ReadFile parses a WDC5 .db2 file. Only WDC5 is supported; anything else +// (including WDC6+) fails loud. +func ReadFile(path string) (*Table, error) { + buf, err := os.ReadFile(path) + if err != nil { + return nil, err + } + t, err := read(buf) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return t, nil +} + +func read(buf []byte) (*Table, error) { + if len(buf) < headerSize { + return nil, fmt.Errorf("WDC5 file is corrupted (shorter than %d-byte header)", headerSize) + } + if string(buf[:4]) != wdc5Magic { + return nil, fmt.Errorf("unsupported DB2 format %q (only WDC5 is supported)", string(buf[:4])) + } + + c := &cursor{buf: buf, pos: 4} + t := &Table{} + + var err error + // schema version (u32) + schema string (128 bytes): unused. + if _, err = c.u32(); err != nil { + return nil, err + } + if _, err = c.need(128); err != nil { + return nil, err + } + + // record_count, field_count, record_size, string_table_size, table_hash, + // layout_hash, min_id, max_id, locale + ints := make([]int32, 9) + for i := range ints { + if ints[i], err = c.i32(); err != nil { + return nil, err + } + } + t.RecordsCount, t.FieldsCount, t.RecordSize, t.StringTableSize = ints[0], ints[1], ints[2], ints[3] + t.TableHash = uint32(ints[4]) + t.MinIndex = ints[6] + + flags, err := c.u16() + if err != nil { + return nil, err + } + t.Flags = db2Flags(flags) + if t.IdFieldIndex, err = c.u16(); err != nil { + return nil, err + } + + // totalFieldsCount, PackedDataOffset, lookupColumnCount, columnMetaDataSize, + // commonDataSize, palletDataSize, sectionsCount + tail := make([]int32, 7) + for i := range tail { + if tail[i], err = c.i32(); err != nil { + return nil, err + } + } + sectionsCount := int(tail[6]) + + t.Sections = make([]sectionHeader, sectionsCount) + for i := range t.Sections { + s := &t.Sections[i] + if s.TactKeyLookup, err = c.u64(); err != nil { + return nil, err + } + for _, dst := range []*int32{&s.FileOffset, &s.NumRecords, &s.StringTableSize, &s.OffsetRecordsEndOffset, + &s.IndexDataSize, &s.ParentLookupDataSize, &s.OffsetMapIDCount, &s.CopyTableCount} { + if *dst, err = c.i32(); err != nil { + return nil, err + } + } + } + + // The empty ItemBonus.db2 ends mid-way through the meta blocks, and the + // early return below never consumes them. Tolerate short reads only when + // the early return will be taken; otherwise a truncated file is corrupt + // and must fail loud. + emptyTable := sectionsCount == 0 || t.RecordsCount == 0 + + t.Meta = make([]fieldMeta, t.FieldsCount) + for i := range t.Meta { + b, err := c.need(4) + if err != nil { + if emptyTable { + t.Meta = t.Meta[:i] + break + } + return nil, err + } + t.Meta[i].Bits = int16(binary.LittleEndian.Uint16(b[0:2])) + t.Meta[i].Offset = int16(binary.LittleEndian.Uint16(b[2:4])) + } + + t.ColumnMeta = make([]columnMeta, t.FieldsCount) + for i := range t.ColumnMeta { + if emptyTable && c.pos+24 > len(c.buf) { + t.ColumnMeta = t.ColumnMeta[:i] + break + } + m := &t.ColumnMeta[i] + if m.RecordOffset, err = c.u16(); err != nil { + return nil, err + } + if m.Size, err = c.u16(); err != nil { + return nil, err + } + if m.AdditionalDataSize, err = c.u32(); err != nil { + return nil, err + } + if m.CompressionType, err = c.u32(); err != nil { + return nil, err + } + if m.A, err = c.i32(); err != nil { + return nil, err + } + if m.B, err = c.i32(); err != nil { + return nil, err + } + if m.C, err = c.i32(); err != nil { + return nil, err + } + } + + // ItemBonus.db2 is empty: 0 sections / 0 records is valid. + if emptyTable { + return t, nil + } + + // pallet data + t.PalletData = make([][]value32, len(t.ColumnMeta)) + for i := range t.ColumnMeta { + ct := t.ColumnMeta[i].CompressionType + if ct == compressionPallet || ct == compressionPalletArray { + n := int(t.ColumnMeta[i].AdditionalDataSize / 4) + t.PalletData[i] = make([]value32, n) + for j := range n { + v, err := c.u32() + if err != nil { + return nil, err + } + t.PalletData[i][j] = value32(v) + } + } + } + + // common data + t.CommonData = make([]map[int32]value32, len(t.ColumnMeta)) + for i := range t.ColumnMeta { + if t.ColumnMeta[i].CompressionType == compressionCommon { + n := int(t.ColumnMeta[i].AdditionalDataSize / 8) + m := make(map[int32]value32, n) + t.CommonData[i] = m + for range n { + k, err := c.i32() + if err != nil { + return nil, err + } + v, err := c.u32() + if err != nil { + return nil, err + } + m[k] = value32(v) + } + } + } + + // encrypted ID lists (read sequentially; content unused — this tool + // never consults them) + for i := range sectionsCount { + if t.Sections[i].TactKeyLookup == 0 { + continue + } + n, err := c.i32() + if err != nil { + return nil, err + } + if _, err := c.need(int(n) * 4); err != nil { + return nil, err + } + } + + t.StringTable = make(map[int64]string) + + previousStringTableSize := int32(0) + previousRecordCount := int32(0) + for si := range t.Sections { + section := t.Sections[si] + c.pos = int(section.FileOffset) + + var recordsData []byte + if t.Flags&flagSparse == 0 { + raw, err := c.need(int(section.NumRecords) * int(t.RecordSize)) + if err != nil { + return nil, err + } + recordsData = padRecordData(raw) + + stringData, err := c.need(int(section.StringTableSize)) + if err != nil { + return nil, err + } + readStringTable(t.StringTable, stringData, int64(previousStringTableSize)) + previousStringTableSize += section.StringTableSize + } else { + raw, err := c.need(int(section.OffsetRecordsEndOffset - section.FileOffset)) + if err != nil { + return nil, err + } + recordsData = padRecordData(raw) + if c.pos != int(section.OffsetRecordsEndOffset) { + return nil, fmt.Errorf("stream position != OffsetRecordsEndOffset") + } + } + + // Skip encrypted sections: TACT key lookup set + record data + // zero-filled, unless the trailing guards below find live id data. + if section.TactKeyLookup != 0 && allZero(recordsData) { + completelyZero := false + if section.IndexDataSize > 0 || section.CopyTableCount > 0 { + // Peek the first id from IndexData/CopyData without consuming. + if c.pos+4 > len(c.buf) { + return nil, fmt.Errorf("unexpected EOF peeking encrypted-section id data") + } + completelyZero = binary.LittleEndian.Uint32(c.buf[c.pos:c.pos+4]) == 0 + } else if section.OffsetMapIDCount > 0 { + // Peek the first SparseEntry's Size without consuming. + if c.pos+6 > len(c.buf) { + return nil, fmt.Errorf("unexpected EOF peeking encrypted-section sparse data") + } + completelyZero = binary.LittleEndian.Uint16(c.buf[c.pos+4:c.pos+6]) == 0 + } else { + completelyZero = true + } + if completelyZero { + previousRecordCount += section.NumRecords + t.SkippedSections++ + continue + } + } + + // index data + indexData := make([]int32, section.IndexDataSize/4) + for i := range indexData { + if indexData[i], err = c.i32(); err != nil { + return nil, err + } + } + if len(indexData) > 0 && allZeroInts(indexData) { + for i := range indexData { + indexData[i] = t.MinIndex + previousRecordCount + int32(i) + } + } + + // duplicate rows data + for i := int32(0); i < section.CopyTableCount; i++ { + dest, err := c.i32() + if err != nil { + return nil, err + } + src, err := c.i32() + if err != nil { + return nil, err + } + if dest != src { + t.copyData = append(t.copyData, copyEntry{Dest: dest, Src: src}) + } + } + + var sparseEntries []sparseEntry + if section.OffsetMapIDCount > 0 { + // HACK: upstream skips a malformed unit-test table (hash 145293629). + if t.TableHash == 145293629 { + if _, err := c.need(4 * int(section.OffsetMapIDCount)); err != nil { + return nil, err + } + } + sparseEntries = make([]sparseEntry, section.OffsetMapIDCount) + for i := range sparseEntries { + b, err := c.need(6) + if err != nil { + return nil, err + } + sparseEntries[i].Offset = binary.LittleEndian.Uint32(b[0:4]) + sparseEntries[i].Size = binary.LittleEndian.Uint16(b[4:6]) + } + } + + if section.OffsetMapIDCount > 0 && t.Flags&flagSecondaryKey != 0 { + var err error + indexData, err = readSparseIndexData(c, section, indexData) + if err != nil { + return nil, err + } + } + + // reference (parent lookup) data + refEntries := make(map[int32]int32) + if section.ParentLookupDataSize > 0 { + numRecords, err := c.i32() + if err != nil { + return nil, err + } + if _, err := c.need(8); err != nil { // minId, maxId + return nil, err + } + for range numRecords { + id, err := c.i32() + if err != nil { + return nil, err + } + index, err := c.i32() + if err != nil { + return nil, err + } + refEntries[index] = id + } + } + + if section.OffsetMapIDCount > 0 && t.Flags&flagSecondaryKey == 0 { + var err error + indexData, err = readSparseIndexData(c, section, indexData) + if err != nil { + return nil, err + } + } + + position := 0 + for i := int32(0); i < section.NumRecords; i++ { + br := newBitReader(recordsData) + if t.Flags&flagSparse != 0 { + br.Position = position + position += int(sparseEntries[i].Size) * 8 + } else { + br.Offset = int(i) * int(t.RecordSize) + } + + var refID int32 + if t.Flags&flagSecondaryKey != 0 { + refID = refEntries[indexData[i]] + } else { + refID = refEntries[i] + } + + id := int32(-1) + if section.IndexDataSize != 0 { + id = indexData[i] + } + + t.rows = append(t.rows, rawRow{ + data: br, + dataOffset: br.Offset, + dataPos: br.Position, + id: id, + refID: refID, + recordIndex: i + previousRecordCount, + }) + } + + previousRecordCount += section.NumRecords + } + + return t, nil +} + +func readSparseIndexData(c *cursor, section sectionHeader, indexData []int32) ([]int32, error) { + sparseIndexData := make([]int32, section.OffsetMapIDCount) + for i := range sparseIndexData { + var err error + if sparseIndexData[i], err = c.i32(); err != nil { + return nil, err + } + } + if section.IndexDataSize > 0 && len(indexData) != len(sparseIndexData) { + return nil, fmt.Errorf("IndexData length != sparseIndexData length") + } + return sparseIndexData, nil +} + +// readStringTable reads NUL-separated UTF-8 strings keyed by byte offset +// (baseOffset + running offset). +func readStringTable(dst map[int64]string, data []byte, baseOffset int64) { + if len(data) == 0 { + return + } + curOfs := 0 + for str := range strings.SplitSeq(string(data), "\x00") { + if curOfs == len(data) { + break + } + dst[baseOffset+int64(curOfs)] = str + curOfs += len(str) + 1 + } +} + +func allZero(b []byte) bool { + for _, v := range b { + if v != 0 { + return false + } + } + return true +} + +func allZeroInts(v []int32) bool { + for _, x := range v { + if x != 0 { + return false + } + } + return true +} diff --git a/wowsims-mop.sln b/wowsims-mop.sln deleted file mode 100644 index 8dc8ed27bb..0000000000 --- a/wowsims-mop.sln +++ /dev/null @@ -1,29 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.5.2.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{07C2787E-EAC7-C090-1BA3-A61EC2A24D84}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DB2ToSqliteTool", "tools\DB2ToSqlite\DB2ToSqliteTool.csproj", "{29589C39-13D6-27DF-3FCD-A561C9823120}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {29589C39-13D6-27DF-3FCD-A561C9823120}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {29589C39-13D6-27DF-3FCD-A561C9823120}.Debug|Any CPU.Build.0 = Debug|Any CPU - {29589C39-13D6-27DF-3FCD-A561C9823120}.Release|Any CPU.ActiveCfg = Release|Any CPU - {29589C39-13D6-27DF-3FCD-A561C9823120}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {29589C39-13D6-27DF-3FCD-A561C9823120} = {07C2787E-EAC7-C090-1BA3-A61EC2A24D84} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {3AB56975-ECBC-4A5F-B09F-FCA6D2ED4A05} - EndGlobalSection -EndGlobal