From 138c5c03babaf2f87c42da729c05ae90ffdcdfec Mon Sep 17 00:00:00 2001 From: Michael Becker Date: Sun, 17 Nov 2019 02:44:30 -0500 Subject: [PATCH] preliminary commit for UTF table writing - DOES NOT WORK - one file writes ok, the other not good at all --- .../Associations/Database/CRI/UTF.uexml | 25 + ...lEditor.Content.PlatformIndependent.csproj | 3 + .../UTF/UTFColumnDataType.cs} | 4 +- .../UTF/UTFColumnStorageType.cs} | 4 +- .../DataFormats/Database/UTF/UTFDataFormat.cs | 553 ++++++++++++++++++ .../FileSystem/CPK/CPKDataFormat.cs | 87 ++- .../FileSystem/CPK/UTF/UTFDataFormat.cs | 282 --------- .../UniversalEditor.Plugins.CRI.csproj | 9 +- 8 files changed, 676 insertions(+), 291 deletions(-) create mode 100644 CSharp/Content/UniversalEditor.Content.PlatformIndependent/Extensions/GameDeveloper/Associations/Database/CRI/UTF.uexml rename CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/{FileSystem/CPK/CPKColumnDataType.cs => Database/UTF/UTFColumnDataType.cs} (90%) rename CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/{FileSystem/CPK/CPKColumnStorageType.cs => Database/UTF/UTFColumnStorageType.cs} (89%) create mode 100644 CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFDataFormat.cs delete mode 100644 CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs diff --git a/CSharp/Content/UniversalEditor.Content.PlatformIndependent/Extensions/GameDeveloper/Associations/Database/CRI/UTF.uexml b/CSharp/Content/UniversalEditor.Content.PlatformIndependent/Extensions/GameDeveloper/Associations/Database/CRI/UTF.uexml new file mode 100644 index 00000000..64d7eded --- /dev/null +++ b/CSharp/Content/UniversalEditor.Content.PlatformIndependent/Extensions/GameDeveloper/Associations/Database/CRI/UTF.uexml @@ -0,0 +1,25 @@ + + + + + + + + *.utf + + + + @UTF + + + + + + + + + + + + + \ No newline at end of file diff --git a/CSharp/Content/UniversalEditor.Content.PlatformIndependent/UniversalEditor.Content.PlatformIndependent.csproj b/CSharp/Content/UniversalEditor.Content.PlatformIndependent/UniversalEditor.Content.PlatformIndependent.csproj index 1a362e6b..421a30ff 100644 --- a/CSharp/Content/UniversalEditor.Content.PlatformIndependent/UniversalEditor.Content.PlatformIndependent.csproj +++ b/CSharp/Content/UniversalEditor.Content.PlatformIndependent/UniversalEditor.Content.PlatformIndependent.csproj @@ -652,6 +652,7 @@ + @@ -675,6 +676,8 @@ + + diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnDataType.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnDataType.cs similarity index 90% rename from CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnDataType.cs rename to CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnDataType.cs index 996b6b62..488b3e66 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnDataType.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnDataType.cs @@ -19,9 +19,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . using System; -namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK +namespace UniversalEditor.Plugins.CRI.DataFormats.Database.UTF { - enum CPKColumnDataType : byte + public enum UTFColumnDataType : byte { Mask = 0x0f, Data = 0x0b, diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnStorageType.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnStorageType.cs similarity index 89% rename from CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnStorageType.cs rename to CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnStorageType.cs index 107501f1..d336e841 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKColumnStorageType.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFColumnStorageType.cs @@ -19,9 +19,9 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . using System; -namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK +namespace UniversalEditor.Plugins.CRI.DataFormats.Database.UTF { - enum CPKColumnStorageType : byte + public enum UTFColumnStorageType : byte { Mask = 0xf0, PerRow = 0x50, diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFDataFormat.cs new file mode 100644 index 00000000..df5001b9 --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFDataFormat.cs @@ -0,0 +1,553 @@ +// +// UTFDataFormat.cs +// +// Author: +// Mike Becker +// +// Copyright (c) 2019 Mike Becker +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +using System; +using System.Collections.Generic; +using UniversalEditor.Accessors; +using UniversalEditor.IO; +using UniversalEditor.ObjectModels.Database; + +namespace UniversalEditor.Plugins.CRI.DataFormats.Database.UTF +{ + public class UTFDataFormat : DataFormat + { + private static DataFormatReference _dfr = null; + protected override DataFormatReference MakeReferenceInternal() + { + if (_dfr == null) + { + _dfr = base.MakeReferenceInternal(); + _dfr.Capabilities.Add(typeof(DatabaseObjectModel), DataFormatCapabilities.All); + } + return _dfr; + } + + private struct UTFTABLEINFO + { + public long utfOffset; + + public int tableSize; + public int schemaOffset; + public int rowsOffset; + public int stringTableOffset; + public int dataOffset; + public uint tableNameStringOffset; + public short tableColumns; + public short rowWidth; + public int tableRows; + public int stringTableSize; + } + + private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br) + { + UTFTABLEINFO info = new UTFTABLEINFO(); + info.utfOffset = Accessor.Position; + info.tableSize = br.ReadInt32(); + info.schemaOffset = 0x20; + info.rowsOffset = br.ReadInt32(); + info.stringTableOffset = br.ReadInt32(); + info.dataOffset = br.ReadInt32(); + + // CPK Header & UTF Header are ignored, so add 8 to each offset + + info.tableNameStringOffset = br.ReadUInt32(); // 00000007 + info.tableColumns = br.ReadInt16(); // 0023 + info.rowWidth = br.ReadInt16(); // 007e + info.tableRows = br.ReadInt32(); // 00000001 + info.stringTableSize = info.dataOffset - info.stringTableOffset; + return info; + } + + protected override void LoadInternal(ref ObjectModel objectModel) + { + DatabaseObjectModel utf = (objectModel as DatabaseObjectModel); + if (utf == null) + throw new ObjectModelNotSupportedException(); + + Reader br = base.Accessor.Reader; + string utf_signature = br.ReadFixedLengthString(4); + if (utf_signature != "@UTF") + throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK + + DatabaseTable dt = new DatabaseTable(); + + br.Endianness = IO.Endianness.BigEndian; + + UTFTABLEINFO info = ReadUTFTableInfo(br); + + int[] columnNameOffsets = new int[info.tableColumns]; + long[] constantOffsets = new long[info.tableColumns]; + UTFColumnStorageType[] storageTypes = new UTFColumnStorageType[info.tableColumns]; + UTFColumnDataType[] dataTypes = new UTFColumnDataType[info.tableColumns]; + + // Read string table - remember, this is relative to UTF data WITH the "@UTF" signature + br.Accessor.SavePosition(); + br.Seek(info.stringTableOffset + 8, IO.SeekOrigin.Begin); + + byte[] stringTableData = br.ReadBytes(info.stringTableSize); + MemoryAccessor maStringTable = new MemoryAccessor(stringTableData); + + maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin); + br.Accessor.LoadPosition(); + + for (int i = 0; i < info.tableColumns; i++) + { + byte schema = br.ReadByte(); + /*// wtf is this? + if (schema == 0) + { + br.Accessor.Seek(3, SeekOrigin.Current); + column.flags = br.ReadByte(); + } + */ + columnNameOffsets[i] = br.ReadInt32(); + + storageTypes[i] = (UTFColumnStorageType)(schema & (byte)UTFColumnStorageType.Mask); + dataTypes[i] = (UTFColumnDataType)(schema & (byte)UTFColumnDataType.Mask); + + object constantValue = null; + if (storageTypes[i] == UTFColumnStorageType.Constant) + { + constantOffsets[i] = br.Accessor.Position; + switch (dataTypes[i]) + { + case UTFColumnDataType.Long: + case UTFColumnDataType.Long2: + case UTFColumnDataType.Data: + { + constantValue = br.ReadInt64(); + break; + } + case UTFColumnDataType.Float: + { + constantValue = br.ReadSingle(); + break; + } + case UTFColumnDataType.String: + { + int valueOffset = br.ReadInt32(); + maStringTable.Reader.Seek(valueOffset, IO.SeekOrigin.Begin); + constantValue = maStringTable.Reader.ReadNullTerminatedString(); + break; + } + case UTFColumnDataType.Int: + case UTFColumnDataType.Int2: + { + constantValue = br.ReadInt32(); + break; + } + case UTFColumnDataType.Short: + case UTFColumnDataType.Short2: + { + constantValue = br.ReadInt16(); + break; + } + case UTFColumnDataType.Byte: + case UTFColumnDataType.Byte2: + { + constantValue = br.ReadByte(); + break; + } + default: + { + Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString()); + break; + } + } + } + + dt.Fields.Add("Field" + i.ToString(), constantValue, DataTypeForDataType(dataTypes[i])); + } + + dt.Name = maStringTable.Reader.ReadNullTerminatedString(); + + for (int i = 0; i < info.tableColumns; i++) + { + maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin); + dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString(); + } + + for (int i = 0; i < info.tableRows; i++) + { + uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth)); + uint rowStartOffset = rowOffset; + br.Accessor.Seek(rowOffset, SeekOrigin.Begin); + + DatabaseRecord record = new DatabaseRecord(); + + for (int j = 0; j < info.tableColumns; j++) + { + UTFColumnStorageType storageType = storageTypes[j]; + UTFColumnDataType dataType = dataTypes[j]; + long constantOffset = constantOffsets[j] - 11; + + switch (storageType) + { + case UTFColumnStorageType.PerRow: + { + switch (dataType) + { + case UTFColumnDataType.String: + { + string value = null; + if (storageType == UTFColumnStorageType.Constant) + { + value = (dt.Fields[j].Value as string); + } + else + { + uint stringOffset = br.ReadUInt32(); + if (stringOffset < stringTableData.Length) + { + maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin); + value = maStringTable.Reader.ReadNullTerminatedString(); + } + } + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + case UTFColumnDataType.Data: + { + uint varDataOffset = br.ReadUInt32(); + uint varDataSize = br.ReadUInt32(); + + byte[] value = new byte[0]; + record.Fields.Add(dt.Fields[j].Name, value); + + // Is the data in another table?? + // ReadUTFTable(br); + break; + } + case UTFColumnDataType.Long: + case UTFColumnDataType.Long2: + { + ulong value = br.ReadUInt64(); + record.Fields.Add(dt.Fields[j].Name, value); + + break; + } + case UTFColumnDataType.Int: + case UTFColumnDataType.Int2: + { + uint value = br.ReadUInt32(); + record.Fields.Add(dt.Fields[j].Name, value); + + break; + } + case UTFColumnDataType.Short: + case UTFColumnDataType.Short2: + { + ushort value = br.ReadUInt16(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + case UTFColumnDataType.Float: + { + float value = br.ReadSingle(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + case UTFColumnDataType.Byte: + case UTFColumnDataType.Byte2: + { + byte value = br.ReadByte(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + } + break; + } + case UTFColumnStorageType.Constant: + { + record.Fields.Add(dt.Fields[j].Name, dt.Fields[j].Value); + continue; + } + case UTFColumnStorageType.Zero: + { + record.Fields.Add(dt.Fields[j].Name, null); + continue; + } + } + } + + dt.Records.Add(record); + } + utf.Tables.Add(dt); + } + + private Type DataTypeForDataType(UTFColumnDataType dataType) + { + switch (dataType) + { + case UTFColumnDataType.Byte: + case UTFColumnDataType.Byte2: + { + return typeof(byte); + } + case UTFColumnDataType.Data: + { + return typeof(byte[]); + } + case UTFColumnDataType.Float: + { + return typeof(float); + } + case UTFColumnDataType.Int: + case UTFColumnDataType.Int2: + { + return typeof(int); + } + case UTFColumnDataType.Long: + case UTFColumnDataType.Long2: + { + return typeof(long); + } + case UTFColumnDataType.Short: + case UTFColumnDataType.Short2: + { + return typeof(short); + } + case UTFColumnDataType.String: + { + return typeof(string); + } + } + return null; + } + + protected override void SaveInternal(ObjectModel objectModel) + { + DatabaseObjectModel utf = (objectModel as DatabaseObjectModel); + if (utf == null) + throw new ObjectModelNotSupportedException(); + + Writer bw = Accessor.Writer; + bw.WriteFixedLengthString("@UTF"); + + DatabaseTable dt = utf.Tables[0]; + + bw.Endianness = IO.Endianness.BigEndian; + + // do the hard work here to determine if a field should be recorded as zero or not + UTFColumnStorageType[] columnStorageTypes = new UTFColumnStorageType[dt.Fields.Count]; + UTFColumnDataType[] columnDataTypes = new UTFColumnDataType[dt.Fields.Count]; + for (int i = 0; i < dt.Fields.Count; i++) + { + columnStorageTypes[i] = UTFColumnStorageType.Zero; + if (dt.Fields[i].DataType == typeof(bool)) + { + columnDataTypes[i] = UTFColumnDataType.Byte; + } + else if (dt.Fields[i].DataType == typeof(byte[])) + { + columnDataTypes[i] = UTFColumnDataType.Data; + } + else if (dt.Fields[i].DataType == typeof(float)) + { + columnDataTypes[i] = UTFColumnDataType.Float; + } + else if (dt.Fields[i].DataType == typeof(int)) + { + columnDataTypes[i] = UTFColumnDataType.Int; + } + else if (dt.Fields[i].DataType == typeof(long)) + { + columnDataTypes[i] = UTFColumnDataType.Long; + } + else if (dt.Fields[i].DataType == typeof(short)) + { + columnDataTypes[i] = UTFColumnDataType.Short; + } + else if (dt.Fields[i].DataType == typeof(string)) + { + columnDataTypes[i] = UTFColumnDataType.String; + } + + if (dt.Fields[i].Value != null) + { + columnStorageTypes[i] = UTFColumnStorageType.Constant; + continue; + } + for (int j = 0; j < dt.Records.Count; j++) + { + if (dt.Records[j].Fields[i].Value != null) + { + columnStorageTypes[i] = UTFColumnStorageType.PerRow; + break; + } + } + } + + int tableSize = 32; // size of entire file + tableSize += (5 * dt.Fields.Count); + short rowWidth = 0x007e; + tableSize += (dt.Name.Length + 1); + tableSize += 7; // "\0".Length + for (int i = 0; i < dt.Fields.Count; i++) + { + tableSize += (dt.Fields[i].Name.Length + 1); + } + + int rowsOffset = 24 + (5 * dt.Fields.Count); + int stringTableOffset = rowsOffset; + for (int i = 0; i < dt.Records.Count; i++) + { + for (int j = 0; j < dt.Records[i].Fields.Count; j++) + { + if (columnStorageTypes[j] == UTFColumnStorageType.PerRow) + { + switch (columnDataTypes[j]) + { + case UTFColumnDataType.String: + { + tableSize += 4; + tableSize += ((string)dt.Records[i].Fields[j].Value).Length + 1; + stringTableOffset += 4; + break; + } + case UTFColumnDataType.Data: + case UTFColumnDataType.Long: + case UTFColumnDataType.Long2: + { + tableSize += 8; // sizeof(long) + stringTableOffset += 8; + break; + } + case UTFColumnDataType.Byte: + case UTFColumnDataType.Byte2: + { + tableSize += 1; // sizeof(byte) + stringTableOffset += 1; + break; + } + case UTFColumnDataType.Float: + case UTFColumnDataType.Int: + case UTFColumnDataType.Int2: + { + tableSize += 4; // sizeof(int) + stringTableOffset += 4; + break; + } + case UTFColumnDataType.Short: + case UTFColumnDataType.Short2: + { + tableSize += 2; // sizeof(short) + stringTableOffset += 2; + break; + } + } + } + } + } + + tableSize += ((8 - (tableSize % 8)) % 8); + tableSize -= 8; + + bw.WriteInt32(tableSize); + bw.WriteInt32(rowsOffset); + bw.WriteInt32(stringTableOffset); + bw.WriteInt32(tableSize); // data offset - same as table size? + bw.WriteUInt32(7); // "\0".Length + bw.WriteInt16((short)dt.Fields.Count); // 0023 + bw.WriteInt16(rowWidth); // 007e + bw.WriteInt32(dt.Records.Count); // 00000001 + + int columnNameOffset = 8 + dt.Name.Length; // add space for "\0" string and dt.Name + 1 + + List stringTable = new List(); + stringTable.Add(""); + stringTable.Add(dt.Name); + for (int i = 0; i < dt.Fields.Count; i++) + { + byte schema = 0; + schema |= (byte)((byte)columnStorageTypes[i] | (byte)columnDataTypes[i]); + + bw.WriteByte(schema); + bw.WriteInt32(columnNameOffset); + + columnNameOffset += dt.Fields[i].Name.Length + 1; + stringTable.Add(dt.Fields[i].Name); + } + + for (int i = 0; i < dt.Records.Count; i++) + { + for (int j = 0; j < dt.Records[i].Fields.Count; j++) + { + if (columnStorageTypes[j] == UTFColumnStorageType.PerRow) + { + switch (columnDataTypes[j]) + { + case UTFColumnDataType.String: + { + stringTable.Add((string)dt.Records[i].Fields[j].Value); + bw.WriteUInt32((uint)stringTable.GetItemOffset(stringTable.Count - 1, 1)); + break; + } + case UTFColumnDataType.Data: + { + uint varDataOffset = 0; + uint varDataSize = 0; + bw.WriteUInt32(varDataOffset); + bw.WriteUInt32(varDataSize); + break; + } + case UTFColumnDataType.Long: + case UTFColumnDataType.Long2: + { + bw.WriteUInt64((ulong)dt.Records[i].Fields[j].Value); + break; + } + case UTFColumnDataType.Int: + case UTFColumnDataType.Int2: + { + bw.WriteUInt32((uint)dt.Records[i].Fields[j].Value); + break; + } + case UTFColumnDataType.Short: + case UTFColumnDataType.Short2: + { + bw.WriteUInt16((ushort)dt.Records[i].Fields[j].Value); + break; + } + case UTFColumnDataType.Float: + { + bw.WriteSingle((float)dt.Records[i].Fields[j].Value); + break; + } + case UTFColumnDataType.Byte: + case UTFColumnDataType.Byte2: + { + bw.WriteByte((byte)dt.Records[i].Fields[j].Value); + break; + } + } + } + } + } + + for (int i = 0; i < stringTable.Count; i++) + { + bw.WriteNullTerminatedString(stringTable[i]); + } + + bw.Align(8); + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs index 783d1f83..158e3233 100755 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs @@ -21,6 +21,8 @@ using UniversalEditor.IO; using UniversalEditor.ObjectModels.Database; using UniversalEditor.ObjectModels.FileSystem; +using UniversalEditor.Plugins.CRI.DataFormats.Database.UTF; + namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK { public class CPKDataFormat : DataFormat @@ -65,7 +67,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK ma.Reader.Seek(-4, IO.SeekOrigin.Current); } - UTF.UTFDataFormat utf_df = new UTF.UTFDataFormat(); + UTFDataFormat utf_df = new UTFDataFormat(); DatabaseObjectModel utf_om = new DatabaseObjectModel(); Document.Load(utf_om, utf_df, ma); @@ -154,6 +156,89 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK e.Data = decompressedData; } + private DatabaseTable BuildHeaderUTF(FileSystemObjectModel fsom) + { + File[] files = fsom.GetAllFiles(); + + DatabaseTable dt = new DatabaseTable(); + dt.Fields.Add("UpdateDateTime"); + dt.Fields.Add("FileSize"); + dt.Fields.Add("ContentOffset"); + dt.Fields.Add("ContentSize"); + dt.Fields.Add("TocOffset"); + dt.Fields.Add("TocSize"); + dt.Fields.Add("TocCrc"); + dt.Fields.Add("EtocOffset"); + dt.Fields.Add("EtocSize"); + dt.Fields.Add("ItocOffset"); + dt.Fields.Add("ItocSize"); + dt.Fields.Add("ItocCrc"); + dt.Fields.Add("GtocOffset"); + dt.Fields.Add("GtocSize"); + dt.Fields.Add("GtocCrc"); + dt.Fields.Add("EnabledPackedSize"); + dt.Fields.Add("EnabledDataSize"); + dt.Fields.Add("TotalDataSize"); + dt.Fields.Add("Tocs"); + dt.Fields.Add("Files"); + dt.Fields.Add("Groups"); + dt.Fields.Add("Attrs"); + dt.Fields.Add("TotalFiles"); + dt.Fields.Add("Directories"); + dt.Fields.Add("Updates"); + dt.Fields.Add("Version"); + dt.Fields.Add("Revision"); + dt.Fields.Add("Align"); + dt.Fields.Add("Sorted"); + dt.Fields.Add("EID"); + dt.Fields.Add("CpkMode"); + dt.Fields.Add("Tvers"); + dt.Fields.Add("Comment"); + dt.Fields.Add("Codec"); + dt.Fields.Add("DpkItoc"); + + dt.Records.Add(new DatabaseRecord(new DatabaseField[] + { + new DatabaseField("UpdateDateTime", (ulong)1), + new DatabaseField("FileSize", null), + new DatabaseField("ContentOffset", (ulong)153600), + new DatabaseField("ContentSize", (ulong)421693440), + new DatabaseField("TocOffset", (ulong)2048), + new DatabaseField("TocSize", (ulong)117896), + new DatabaseField("TocCrc", null), + new DatabaseField("EtocOffset", (ulong)421847040), + new DatabaseField("EtocSize", (ulong)21768), + new DatabaseField("ItocOffset", (ulong)131072), + new DatabaseField("ItocSize", (ulong)21744), + new DatabaseField("ItocCrc", null), + new DatabaseField("GtocOffset", null), + new DatabaseField("GtocSize", null), + new DatabaseField("GtocCrc", null), + new DatabaseField("EnabledPackedSize", (ulong)837335552), + new DatabaseField("EnabledDataSize", (ulong)837335552), + new DatabaseField("TotalDataSize", null), + new DatabaseField("Tocs", null), + new DatabaseField("Files", (uint)files.Length), + new DatabaseField("Groups", (uint)0), + new DatabaseField("Attrs", (uint)0), + new DatabaseField("TotalFiles", null), + new DatabaseField("Directories", null), + new DatabaseField("Updates", null), + new DatabaseField("Version", (ushort)7), + new DatabaseField("Revision", (ushort)0), + new DatabaseField("Align", (ushort)2048), + new DatabaseField("Sorted", (ushort)1), + new DatabaseField("EID", (ushort)1), + new DatabaseField("CpkMode", (uint)2), + new DatabaseField("Tvers", "CPKMC2.14.00, DLL2.74.00"), + new DatabaseField("Comment", null), + new DatabaseField("Codec", (uint)0), + new DatabaseField("DpkItoc", (uint)0) + })); + + return dt; + } + private byte[] DecryptUTF(byte[] input) { diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs deleted file mode 100644 index 71f6b23d..00000000 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs +++ /dev/null @@ -1,282 +0,0 @@ -// -// UTFDataFormat.cs -// -// Author: -// Mike Becker -// -// Copyright (c) 2019 Mike Becker -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . -using System; -using UniversalEditor.Accessors; -using UniversalEditor.IO; -using UniversalEditor.ObjectModels.Database; - -namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK.UTF -{ - public class UTFDataFormat : DataFormat - { - - private struct UTFTABLEINFO - { - public long utfOffset; - public int tableSize; - public int schemaOffset; - public int rowsOffset; - public int stringTableOffset; - public int dataOffset; - public uint tableNameStringOffset; - public short tableColumns; - public short rowWidth; - public int tableRows; - public int stringTableSize; - } - - private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br) - { - UTFTABLEINFO info = new UTFTABLEINFO(); - info.utfOffset = Accessor.Position; - info.tableSize = br.ReadInt32(); - info.schemaOffset = 0x20; - info.rowsOffset = br.ReadInt32(); - info.stringTableOffset = br.ReadInt32(); - info.dataOffset = br.ReadInt32(); - - // CPK Header & UTF Header are ignored, so add 8 to each offset - - info.tableNameStringOffset = br.ReadUInt32(); // 00000007 - info.tableColumns = br.ReadInt16(); // 0023 - info.rowWidth = br.ReadInt16(); // 007e - info.tableRows = br.ReadInt32(); // 00000001 - info.stringTableSize = info.dataOffset - info.stringTableOffset; - return info; - } - - protected override void LoadInternal(ref ObjectModel objectModel) - { - Reader br = base.Accessor.Reader; - string utf_signature = br.ReadFixedLengthString(4); - if (utf_signature != "@UTF") - throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK - - DatabaseObjectModel utf = (objectModel as DatabaseObjectModel); - DatabaseTable dt = new DatabaseTable(); - - br.Endianness = IO.Endianness.BigEndian; - - UTFTABLEINFO info = ReadUTFTableInfo(br); - - int[] columnNameOffsets = new int[info.tableColumns]; - long[] constantOffsets = new long[info.tableColumns]; - CPKColumnStorageType[] storageTypes = new CPKColumnStorageType[info.tableColumns]; - CPKColumnDataType[] dataTypes = new CPKColumnDataType[info.tableColumns]; - - for (int i = 0; i < info.tableColumns; i++) - { - byte schema = br.ReadByte(); - /*// wtf is this? - if (schema == 0) - { - br.Accessor.Seek(3, SeekOrigin.Current); - column.flags = br.ReadByte(); - } - */ - columnNameOffsets[i] = br.ReadInt32(); - - storageTypes[i] = (CPKColumnStorageType)(schema & (byte)CPKColumnStorageType.Mask); - dataTypes[i] = (CPKColumnDataType)(schema & (byte)CPKColumnDataType.Mask); - - if (storageTypes[i] == CPKColumnStorageType.Constant) - { - constantOffsets[i] = br.Accessor.Position; - switch (dataTypes[i]) - { - case CPKColumnDataType.Long: - case CPKColumnDataType.Long2: - case CPKColumnDataType.Data: - { - long dummy = br.ReadInt64(); - break; - } - case CPKColumnDataType.Float: - { - float dummy = br.ReadSingle(); - break; - } - case CPKColumnDataType.String: - case CPKColumnDataType.Int: - case CPKColumnDataType.Int2: - { - int dummy = br.ReadInt32(); - break; - } - case CPKColumnDataType.Short: - case CPKColumnDataType.Short2: - { - short dummy = br.ReadInt16(); - break; - } - case CPKColumnDataType.Byte: - case CPKColumnDataType.Byte2: - { - byte dummy = br.ReadByte(); - break; - } - default: - { - Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString()); - break; - } - } - } - - dt.Fields.Add("Field" + i.ToString(), null); - } - - // Read string table - remember, this is relative to UTF data WITH the "@UTF" signature - br.Seek(info.stringTableOffset + 8, IO.SeekOrigin.Begin); - - byte[] stringTableData = br.ReadBytes(info.stringTableSize); - MemoryAccessor maStringTable = new MemoryAccessor(stringTableData); - - maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin); - dt.Name = maStringTable.Reader.ReadNullTerminatedString(); - - for (int i = 0; i < info.tableColumns; i++) - { - maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin); - dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString(); - } - - for (int i = 0; i < info.tableRows; i++) - { - uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth)); - uint rowStartOffset = rowOffset; - br.Accessor.Seek(rowOffset, SeekOrigin.Begin); - - DatabaseRecord record = new DatabaseRecord(); - - for (int j = 0; j < info.tableColumns; j++) - { - CPKColumnStorageType storageType = storageTypes[j]; - CPKColumnDataType dataType = dataTypes[j]; - long constantOffset = constantOffsets[j] - 11; - - switch (storageType) - { - case CPKColumnStorageType.PerRow: - break; - case CPKColumnStorageType.Constant: - break; - case CPKColumnStorageType.Zero: - record.Fields.Add(dt.Fields[j].Name, null); - continue; - } - - long dataOffset1 = 0; - if (storageType == CPKColumnStorageType.Constant) - { - dataOffset1 = constantOffset; - } - else - { - dataOffset1 = rowOffset; - } - - // br.Seek(dataOffset1, IO.SeekOrigin.Begin); - switch (dataType) - { - case CPKColumnDataType.String: - { - uint stringOffset = 0; - if (storageType == CPKColumnStorageType.Constant) - { - stringOffset = (uint)constantOffset; - } - else - { - stringOffset = br.ReadUInt32(); - } - string value = null; - if (stringOffset < stringTableData.Length) - { - maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin); - value = maStringTable.Reader.ReadNullTerminatedString(); - } - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Data: - { - uint varDataOffset = br.ReadUInt32(); - uint varDataSize = br.ReadUInt32(); - - byte[] value = new byte[0]; - record.Fields.Add(dt.Fields[j].Name, value); - - // Is the data in another table?? - // ReadUTFTable(br); - break; - } - case CPKColumnDataType.Long: - case CPKColumnDataType.Long2: - { - ulong value = br.ReadUInt64(); - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Int: - case CPKColumnDataType.Int2: - { - uint value = br.ReadUInt32(); - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Short: - case CPKColumnDataType.Short2: - { - ushort value = br.ReadUInt16(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - case CPKColumnDataType.Float: - { - float value = br.ReadSingle(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - case CPKColumnDataType.Byte: - case CPKColumnDataType.Byte2: - { - byte value = br.ReadByte(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - } - } - - dt.Records.Add(record); - } - utf.Tables.Add(dt); - } - - protected override void SaveInternal(ObjectModel objectModel) - { - throw new NotImplementedException(); - } - } -} diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj b/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj index 1ec51a54..a35ca132 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj @@ -33,20 +33,21 @@ - - - + + + - + +