From 027840d75f275a9dd5087b7951afeea4a8fd5ce3 Mon Sep 17 00:00:00 2001 From: Michael Becker Date: Fri, 15 Nov 2019 12:13:45 -0500 Subject: [PATCH] CRI CPK archive loading and extraction WORKS NOW!! i'm so happy :) --- .../FileSystem/CPK/CPKDataFormat.cs | 366 ++++++------------ .../FileSystem/CPK/UTF/UTFDataFormat.cs | 282 ++++++++++++++ .../UniversalEditor.Plugins.CRI.csproj | 2 + 3 files changed, 395 insertions(+), 255 deletions(-) create mode 100644 CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs index 800dd841..783d1f83 100755 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs @@ -16,7 +16,8 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. using System; - +using UniversalEditor.Accessors; +using UniversalEditor.IO; using UniversalEditor.ObjectModels.Database; using UniversalEditor.ObjectModels.FileSystem; @@ -35,245 +36,6 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK return _dfr; } - private struct UTFTABLEINFO - { - public long utfOffset; - public int tableSize; - public int schemaOffset; - public int rowsOffset; - public int stringTableOffset; - public int dataOffset; - public uint tableNameStringIndex; - public short tableColumns; - public short rowWidth; - public int tableRows; - public int stringTableSize; - } - - private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br) - { - UTFTABLEINFO info = new UTFTABLEINFO(); - info.utfOffset = Accessor.Position; - info.tableSize = br.ReadInt32(); - info.schemaOffset = 0x20; - info.rowsOffset = br.ReadInt32(); - info.stringTableOffset = br.ReadInt32(); - info.dataOffset = br.ReadInt32(); - info.tableNameStringIndex = br.ReadUInt32(); - info.tableColumns = br.ReadInt16(); - info.rowWidth = br.ReadInt16(); - info.tableRows = br.ReadInt32(); - info.stringTableSize = info.dataOffset - info.stringTableOffset; - return info; - } - - private DatabaseTable ReadUTFTable(IO.Reader br) - { - DatabaseTable dt = new DatabaseTable(); - dt.Name = "@UTF"; - - string utfSignal = br.ReadFixedLengthString(4); - - if (utfSignal != "@UTF") - { - return null; - } - - UTFTABLEINFO info = ReadUTFTableInfo(br); - - int[] columnNameIndices = new int[info.tableColumns]; - long[] constantOffsets = new long[info.tableColumns]; - CPKColumnStorageType[] storageTypes = new CPKColumnStorageType[info.tableColumns]; - CPKColumnDataType[] dataTypes = new CPKColumnDataType[info.tableColumns]; - - for (int i = 0; i < info.tableColumns; i++) - { - byte schema = br.ReadByte(); - columnNameIndices[i] = br.ReadInt32(); - - storageTypes[i] = (CPKColumnStorageType)(schema & (byte)CPKColumnStorageType.Mask); - dataTypes[i] = (CPKColumnDataType)(schema & (byte)CPKColumnDataType.Mask); - - if (storageTypes[i] == CPKColumnStorageType.Constant) - { - constantOffsets[i] = Accessor.Position; - switch (dataTypes[i]) - { - case CPKColumnDataType.Long: - case CPKColumnDataType.Long2: - case CPKColumnDataType.Data: - { - long dummy = br.ReadInt64(); - break; - } - case CPKColumnDataType.Float: - { - float dummy = br.ReadSingle(); - break; - } - case CPKColumnDataType.String: - case CPKColumnDataType.Int: - case CPKColumnDataType.Int2: - { - int dummy = br.ReadInt32(); - break; - } - case CPKColumnDataType.Short: - case CPKColumnDataType.Short2: - { - short dummy = br.ReadInt16(); - break; - } - case CPKColumnDataType.Byte: - case CPKColumnDataType.Byte2: - { - byte dummy = br.ReadByte(); - break; - } - default: - { - Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString()); - break; - } - } - } - - dt.Fields.Add("Field" + i.ToString(), null); - } - - // Read string table - - Accessor.Seek(info.stringTableOffset + 8 + 0x10, IO.SeekOrigin.Begin); - - string[] stringTable = br.ReadNullTerminatedStringArray(info.stringTableSize); - - string tableName = stringTable[info.tableNameStringIndex]; - - - // Seek to string table offset - Accessor.Seek(info.stringTableOffset + 4 + info.utfOffset, IO.SeekOrigin.Begin); - for (short i = 0; i < info.tableColumns; i++) - { - string columnName = br.ReadNullTerminatedString(); - dt.Fields[i].Name = columnName; - } - - for (int i = 0; i < info.tableRows; i++) - { - uint rowOffset = (uint)(info.utfOffset + 8 + info.rowsOffset + (i * info.rowWidth)); - uint rowStartOffset = rowOffset; - - DatabaseRecord record = new DatabaseRecord(); - - for (int j = 0; j < info.tableColumns; j++) - { - CPKColumnStorageType storageType = storageTypes[j]; - CPKColumnDataType dataType = dataTypes[j]; - long constantOffset = constantOffsets[j]; - - switch (storageType) - { - case CPKColumnStorageType.PerRow: - break; - case CPKColumnStorageType.Constant: - break; - case CPKColumnStorageType.Zero: - record.Fields.Add(dt.Fields[j].Name, null); - continue; - } - - long dataOffset1 = 0; - if (storageType == CPKColumnStorageType.Constant) - { - dataOffset1 = constantOffset; - } - else - { - dataOffset1 = rowOffset; - } - - Accessor.Seek(dataOffset1, IO.SeekOrigin.Begin); - switch (dataType) - { - case CPKColumnDataType.String: - { - uint stringOffset = br.ReadUInt32(); - string value = null; - if (stringOffset < stringTable.Length) - { - value = stringTable[stringOffset]; - } - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Data: - { - uint varDataOffset = br.ReadUInt32(); - uint varDataSize = br.ReadUInt32(); - - byte[] value = new byte[0]; - record.Fields.Add(dt.Fields[j].Name, value); - - // Is the data in another table?? - // ReadUTFTable(br); - break; - } - case CPKColumnDataType.Long: - case CPKColumnDataType.Long2: - { - ulong value = br.ReadUInt64(); - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Int: - case CPKColumnDataType.Int2: - { - uint value = br.ReadUInt32(); - record.Fields.Add(dt.Fields[j].Name, value); - - break; - } - case CPKColumnDataType.Short: - case CPKColumnDataType.Short2: - { - ushort value = br.ReadUInt16(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - case CPKColumnDataType.Float: - { - float value = br.ReadSingle(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - case CPKColumnDataType.Byte: - case CPKColumnDataType.Byte2: - { - byte value = br.ReadByte(); - record.Fields.Add(dt.Fields[j].Name, value); - break; - } - } - } - - dt.Records.Add(record); - } - - return dt; - } - - private string[] ReadUTFStringTable(IO.Reader br) - { - Accessor.Seek(12, IO.SeekOrigin.Current); - - UTFTABLEINFO info = ReadUTFTableInfo(br); - string[] value = br.ReadNullTerminatedStringArray(info.stringTableSize); - - return value; - } - protected override void LoadInternal (ref ObjectModel objectModel) { ObjectModels.FileSystem.FileSystemObjectModel fsom = (objectModel as ObjectModels.FileSystem.FileSystemObjectModel); @@ -281,27 +43,60 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK throw new ObjectModelNotSupportedException(); IO.Reader br = Accessor.Reader; - br.Endianness = IO.Endianness.BigEndian; - + // Rebuilt based on cpk_unpack - Accessor.Seek(0, IO.SeekOrigin.Begin); - + // Rebuilt AGAIN based on github.com/esperknight/CriPakTools string CPK = br.ReadFixedLengthString (4); - Accessor.Seek(0x10, IO.SeekOrigin.Begin); + int unknown1 = br.ReadInt32(); - DatabaseTable dtUTF = ReadUTFTable(br); + long utf_size = br.ReadInt64(); // size of UTF not including "@UTF" + byte[] utf_data = br.ReadBytes(utf_size + 4); - // For now, just hardcode the TOC offset - long tocOffset = 2048; // (long)dtUTF.Records[0].Fields["TocOffset"].Value; - Accessor.Seek(tocOffset, IO.SeekOrigin.Begin); + MemoryAccessor ma = new MemoryAccessor(utf_data); + string utf_signature = ma.Reader.ReadFixedLengthString(4); + if (utf_signature != "@UTF") + { + // encrypted? + utf_data = DecryptUTF(utf_data); + ma = new MemoryAccessor(utf_data); + } + else + { + ma.Reader.Seek(-4, IO.SeekOrigin.Current); + } + + UTF.UTFDataFormat utf_df = new UTF.UTFDataFormat(); + DatabaseObjectModel utf_om = new DatabaseObjectModel(); + Document.Load(utf_om, utf_df, ma); + + DatabaseTable dtUTF = utf_om.Tables[0]; + // UTF table parsing works now, so no need to hardcode toc offset - WOOHOO!!! + ulong tocOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value; + br.Seek((long)tocOffset, IO.SeekOrigin.Begin); string tocSignature = br.ReadFixedLengthString(4); - long tocEntries = (long)dtUTF.Records.Count; + unknown1 = br.ReadInt32(); - string[] tocStringTable = ReadUTFStringTable(br); + // UTF table for TOC + utf_size = br.ReadInt64(); // size of UTF not including "@UTF" + utf_data = br.ReadBytes(utf_size + 4); - Accessor.Seek(2064, IO.SeekOrigin.Begin); - DatabaseTable dtUTF2 = ReadUTFTable(br); + ma = new MemoryAccessor(utf_data); + utf_signature = ma.Reader.ReadFixedLengthString(4); + if (utf_signature != "@UTF") + { + // encrypted? + utf_data = DecryptUTF(utf_data); + ma = new MemoryAccessor(utf_data); + } + else + { + ma.Reader.Seek(-4, IO.SeekOrigin.Current); + } + + utf_om = new DatabaseObjectModel(); + Document.Load(utf_om, utf_df, ma); + DatabaseTable dtUTF2 = utf_om.Tables[0]; if (objectModel is DatabaseObjectModel) { @@ -309,12 +104,73 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK } else if (objectModel is FileSystemObjectModel) { - for (int i = 0; i < dtUTF.Fields.Count; i++) + for (int i = 0; i < dtUTF2.Records.Count; i++) { - fsom.Files.Add (dtUTF.Fields[i].Name); + string dirName = (string)dtUTF2.Records[i].Fields["DirName"].Value; + string fileTitle = (string)dtUTF2.Records[i].Fields["FileName"].Value; + string fileName = fileTitle; + if (!String.IsNullOrEmpty(dirName)) + { + fileName = dirName + '/' + fileTitle; + } + + uint decompressedLength = (uint)dtUTF2.Records[i].Fields["FileSize"].Value; + uint compressedLength = (uint)dtUTF2.Records[i].Fields["ExtractSize"].Value; + ulong offset = (ulong)dtUTF2.Records[i].Fields["FileOffset"].Value; + ulong lContentOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value; + offset += lContentOffset; + + File f = fsom.AddFile(fileName); + f.Properties.Add("DecompressedLength", decompressedLength); + f.Properties.Add("CompressedLength", compressedLength); + f.Properties.Add("Offset", offset); + f.Size = decompressedLength; + f.DataRequest += f_DataRequest; } } } + + void f_DataRequest(object sender, DataRequestEventArgs e) + { + File f = (sender as File); + uint decompressedLength = (uint)f.Properties["DecompressedLength"]; + uint compressedLength = (uint)f.Properties["CompressedLength"]; + ulong offset = (ulong)f.Properties["Offset"]; + Reader br = base.Accessor.Reader; + + br.Accessor.Position = (long)offset; + + byte[] decompressedData = null; + if (compressedLength == 0) + { + decompressedData = br.ReadBytes(decompressedLength); + } + else + { + byte[] compressedData = br.ReadBytes(compressedLength); + decompressedData = /*compress()*/ compressedData; + } + + e.Data = decompressedData; + } + + + private byte[] DecryptUTF(byte[] input) + { + byte[] result = new byte[input.Length]; + + int m = 0x0000655f, t = 0x00004115; + for (int i = 0; i < input.Length; i++) + { + byte d = input[i]; + d = (byte)(d ^ (byte)(m & 0xff)); + result[i] = d; + m *= t; + } + + return result; + } + protected override void SaveInternal (ObjectModel objectModel) { FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel); diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs new file mode 100644 index 00000000..71f6b23d --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs @@ -0,0 +1,282 @@ +// +// UTFDataFormat.cs +// +// Author: +// Mike Becker +// +// Copyright (c) 2019 Mike Becker +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +using System; +using UniversalEditor.Accessors; +using UniversalEditor.IO; +using UniversalEditor.ObjectModels.Database; + +namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK.UTF +{ + public class UTFDataFormat : DataFormat + { + + private struct UTFTABLEINFO + { + public long utfOffset; + public int tableSize; + public int schemaOffset; + public int rowsOffset; + public int stringTableOffset; + public int dataOffset; + public uint tableNameStringOffset; + public short tableColumns; + public short rowWidth; + public int tableRows; + public int stringTableSize; + } + + private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br) + { + UTFTABLEINFO info = new UTFTABLEINFO(); + info.utfOffset = Accessor.Position; + info.tableSize = br.ReadInt32(); + info.schemaOffset = 0x20; + info.rowsOffset = br.ReadInt32(); + info.stringTableOffset = br.ReadInt32(); + info.dataOffset = br.ReadInt32(); + + // CPK Header & UTF Header are ignored, so add 8 to each offset + + info.tableNameStringOffset = br.ReadUInt32(); // 00000007 + info.tableColumns = br.ReadInt16(); // 0023 + info.rowWidth = br.ReadInt16(); // 007e + info.tableRows = br.ReadInt32(); // 00000001 + info.stringTableSize = info.dataOffset - info.stringTableOffset; + return info; + } + + protected override void LoadInternal(ref ObjectModel objectModel) + { + Reader br = base.Accessor.Reader; + string utf_signature = br.ReadFixedLengthString(4); + if (utf_signature != "@UTF") + throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK + + DatabaseObjectModel utf = (objectModel as DatabaseObjectModel); + DatabaseTable dt = new DatabaseTable(); + + br.Endianness = IO.Endianness.BigEndian; + + UTFTABLEINFO info = ReadUTFTableInfo(br); + + int[] columnNameOffsets = new int[info.tableColumns]; + long[] constantOffsets = new long[info.tableColumns]; + CPKColumnStorageType[] storageTypes = new CPKColumnStorageType[info.tableColumns]; + CPKColumnDataType[] dataTypes = new CPKColumnDataType[info.tableColumns]; + + for (int i = 0; i < info.tableColumns; i++) + { + byte schema = br.ReadByte(); + /*// wtf is this? + if (schema == 0) + { + br.Accessor.Seek(3, SeekOrigin.Current); + column.flags = br.ReadByte(); + } + */ + columnNameOffsets[i] = br.ReadInt32(); + + storageTypes[i] = (CPKColumnStorageType)(schema & (byte)CPKColumnStorageType.Mask); + dataTypes[i] = (CPKColumnDataType)(schema & (byte)CPKColumnDataType.Mask); + + if (storageTypes[i] == CPKColumnStorageType.Constant) + { + constantOffsets[i] = br.Accessor.Position; + switch (dataTypes[i]) + { + case CPKColumnDataType.Long: + case CPKColumnDataType.Long2: + case CPKColumnDataType.Data: + { + long dummy = br.ReadInt64(); + break; + } + case CPKColumnDataType.Float: + { + float dummy = br.ReadSingle(); + break; + } + case CPKColumnDataType.String: + case CPKColumnDataType.Int: + case CPKColumnDataType.Int2: + { + int dummy = br.ReadInt32(); + break; + } + case CPKColumnDataType.Short: + case CPKColumnDataType.Short2: + { + short dummy = br.ReadInt16(); + break; + } + case CPKColumnDataType.Byte: + case CPKColumnDataType.Byte2: + { + byte dummy = br.ReadByte(); + break; + } + default: + { + Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString()); + break; + } + } + } + + dt.Fields.Add("Field" + i.ToString(), null); + } + + // Read string table - remember, this is relative to UTF data WITH the "@UTF" signature + br.Seek(info.stringTableOffset + 8, IO.SeekOrigin.Begin); + + byte[] stringTableData = br.ReadBytes(info.stringTableSize); + MemoryAccessor maStringTable = new MemoryAccessor(stringTableData); + + maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin); + dt.Name = maStringTable.Reader.ReadNullTerminatedString(); + + for (int i = 0; i < info.tableColumns; i++) + { + maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin); + dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString(); + } + + for (int i = 0; i < info.tableRows; i++) + { + uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth)); + uint rowStartOffset = rowOffset; + br.Accessor.Seek(rowOffset, SeekOrigin.Begin); + + DatabaseRecord record = new DatabaseRecord(); + + for (int j = 0; j < info.tableColumns; j++) + { + CPKColumnStorageType storageType = storageTypes[j]; + CPKColumnDataType dataType = dataTypes[j]; + long constantOffset = constantOffsets[j] - 11; + + switch (storageType) + { + case CPKColumnStorageType.PerRow: + break; + case CPKColumnStorageType.Constant: + break; + case CPKColumnStorageType.Zero: + record.Fields.Add(dt.Fields[j].Name, null); + continue; + } + + long dataOffset1 = 0; + if (storageType == CPKColumnStorageType.Constant) + { + dataOffset1 = constantOffset; + } + else + { + dataOffset1 = rowOffset; + } + + // br.Seek(dataOffset1, IO.SeekOrigin.Begin); + switch (dataType) + { + case CPKColumnDataType.String: + { + uint stringOffset = 0; + if (storageType == CPKColumnStorageType.Constant) + { + stringOffset = (uint)constantOffset; + } + else + { + stringOffset = br.ReadUInt32(); + } + string value = null; + if (stringOffset < stringTableData.Length) + { + maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin); + value = maStringTable.Reader.ReadNullTerminatedString(); + } + record.Fields.Add(dt.Fields[j].Name, value); + + break; + } + case CPKColumnDataType.Data: + { + uint varDataOffset = br.ReadUInt32(); + uint varDataSize = br.ReadUInt32(); + + byte[] value = new byte[0]; + record.Fields.Add(dt.Fields[j].Name, value); + + // Is the data in another table?? + // ReadUTFTable(br); + break; + } + case CPKColumnDataType.Long: + case CPKColumnDataType.Long2: + { + ulong value = br.ReadUInt64(); + record.Fields.Add(dt.Fields[j].Name, value); + + break; + } + case CPKColumnDataType.Int: + case CPKColumnDataType.Int2: + { + uint value = br.ReadUInt32(); + record.Fields.Add(dt.Fields[j].Name, value); + + break; + } + case CPKColumnDataType.Short: + case CPKColumnDataType.Short2: + { + ushort value = br.ReadUInt16(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + case CPKColumnDataType.Float: + { + float value = br.ReadSingle(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + case CPKColumnDataType.Byte: + case CPKColumnDataType.Byte2: + { + byte value = br.ReadByte(); + record.Fields.Add(dt.Fields[j].Name, value); + break; + } + } + } + + dt.Records.Add(record); + } + utf.Tables.Add(dt); + } + + protected override void SaveInternal(ObjectModel objectModel) + { + throw new NotImplementedException(); + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj b/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj index 56ebcdf3..02f4ca7b 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj +++ b/CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj @@ -36,11 +36,13 @@ + +