universaleditor/editor-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CRI CPK archive loading and extraction WORKS NOW!! i'm so happy :)

Browse Source
This commit is contained in:
Michael Becker 2019-11-15 12:13:45 -05:00
parent 033ab67b5b
commit 027840d75f
No known key found for this signature in database
GPG Key ID: 389DFF5D73781A12
3 changed files with 395 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

366
CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs
View File

@ -16,7 +16,8 @@
// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
using System;
using UniversalEditor.Accessors;
using UniversalEditor.IO;
using UniversalEditor.ObjectModels.Database;
using UniversalEditor.ObjectModels.FileSystem;
@ -35,245 +36,6 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
return _dfr;
}
private struct UTFTABLEINFO
{
public long utfOffset;
public int tableSize;
public int schemaOffset;
public int rowsOffset;
public int stringTableOffset;
public int dataOffset;
public uint tableNameStringIndex;
public short tableColumns;
public short rowWidth;
public int tableRows;
public int stringTableSize;
}
private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br)
{
UTFTABLEINFO info = new UTFTABLEINFO();
info.utfOffset = Accessor.Position;
info.tableSize = br.ReadInt32();
info.schemaOffset = 0x20;
info.rowsOffset = br.ReadInt32();
info.stringTableOffset = br.ReadInt32();
info.dataOffset = br.ReadInt32();
info.tableNameStringIndex = br.ReadUInt32();
info.tableColumns = br.ReadInt16();
info.rowWidth = br.ReadInt16();
info.tableRows = br.ReadInt32();
info.stringTableSize = info.dataOffset - info.stringTableOffset;
return info;
}
private DatabaseTable ReadUTFTable(IO.Reader br)
{
DatabaseTable dt = new DatabaseTable();
dt.Name = "@UTF";
string utfSignal = br.ReadFixedLengthString(4);
if (utfSignal != "@UTF")
{
return null;
}
UTFTABLEINFO info = ReadUTFTableInfo(br);
int[] columnNameIndices = new int[info.tableColumns];
long[] constantOffsets = new long[info.tableColumns];
CPKColumnStorageType[] storageTypes = new CPKColumnStorageType[info.tableColumns];
CPKColumnDataType[] dataTypes = new CPKColumnDataType[info.tableColumns];
for (int i = 0; i < info.tableColumns; i++)
{
byte schema = br.ReadByte();
columnNameIndices[i] = br.ReadInt32();
storageTypes[i] = (CPKColumnStorageType)(schema & (byte)CPKColumnStorageType.Mask);
dataTypes[i] = (CPKColumnDataType)(schema & (byte)CPKColumnDataType.Mask);
if (storageTypes[i] == CPKColumnStorageType.Constant)
{
constantOffsets[i] = Accessor.Position;
switch (dataTypes[i])
{
case CPKColumnDataType.Long:
case CPKColumnDataType.Long2:
case CPKColumnDataType.Data:
{
long dummy = br.ReadInt64();
break;
}
case CPKColumnDataType.Float:
{
float dummy = br.ReadSingle();
break;
}
case CPKColumnDataType.String:
case CPKColumnDataType.Int:
case CPKColumnDataType.Int2:
{
int dummy = br.ReadInt32();
break;
}
case CPKColumnDataType.Short:
case CPKColumnDataType.Short2:
{
short dummy = br.ReadInt16();
break;
}
case CPKColumnDataType.Byte:
case CPKColumnDataType.Byte2:
{
byte dummy = br.ReadByte();
break;
}
default:
{
Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString());
break;
}
}
}
dt.Fields.Add("Field" + i.ToString(), null);
}
// Read string table
Accessor.Seek(info.stringTableOffset + 8 + 0x10, IO.SeekOrigin.Begin);
string[] stringTable = br.ReadNullTerminatedStringArray(info.stringTableSize);
string tableName = stringTable[info.tableNameStringIndex];
// Seek to string table offset
Accessor.Seek(info.stringTableOffset + 4 + info.utfOffset, IO.SeekOrigin.Begin);
for (short i = 0; i < info.tableColumns; i++)
{
string columnName = br.ReadNullTerminatedString();
dt.Fields[i].Name = columnName;
}
for (int i = 0; i < info.tableRows; i++)
{
uint rowOffset = (uint)(info.utfOffset + 8 + info.rowsOffset + (i * info.rowWidth));
uint rowStartOffset = rowOffset;
DatabaseRecord record = new DatabaseRecord();
for (int j = 0; j < info.tableColumns; j++)
{
CPKColumnStorageType storageType = storageTypes[j];
CPKColumnDataType dataType = dataTypes[j];
long constantOffset = constantOffsets[j];
switch (storageType)
{
case CPKColumnStorageType.PerRow:
break;
case CPKColumnStorageType.Constant:
break;
case CPKColumnStorageType.Zero:
record.Fields.Add(dt.Fields[j].Name, null);
continue;
}
long dataOffset1 = 0;
if (storageType == CPKColumnStorageType.Constant)
{
dataOffset1 = constantOffset;
}
else
{
dataOffset1 = rowOffset;
}
Accessor.Seek(dataOffset1, IO.SeekOrigin.Begin);
switch (dataType)
{
case CPKColumnDataType.String:
{
uint stringOffset = br.ReadUInt32();
string value = null;
if (stringOffset < stringTable.Length)
{
value = stringTable[stringOffset];
}
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Data:
{
uint varDataOffset = br.ReadUInt32();
uint varDataSize = br.ReadUInt32();
byte[] value = new byte[0];
record.Fields.Add(dt.Fields[j].Name, value);
// Is the data in another table??
// ReadUTFTable(br);
break;
}
case CPKColumnDataType.Long:
case CPKColumnDataType.Long2:
{
ulong value = br.ReadUInt64();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Int:
case CPKColumnDataType.Int2:
{
uint value = br.ReadUInt32();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Short:
case CPKColumnDataType.Short2:
{
ushort value = br.ReadUInt16();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Float:
{
float value = br.ReadSingle();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Byte:
case CPKColumnDataType.Byte2:
{
byte value = br.ReadByte();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
}
}
dt.Records.Add(record);
}
return dt;
}
private string[] ReadUTFStringTable(IO.Reader br)
{
Accessor.Seek(12, IO.SeekOrigin.Current);
UTFTABLEINFO info = ReadUTFTableInfo(br);
string[] value = br.ReadNullTerminatedStringArray(info.stringTableSize);
return value;
}
protected override void LoadInternal (ref ObjectModel objectModel)
{
ObjectModels.FileSystem.FileSystemObjectModel fsom = (objectModel as ObjectModels.FileSystem.FileSystemObjectModel);
@ -281,27 +43,60 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
throw new ObjectModelNotSupportedException();
IO.Reader br = Accessor.Reader;
br.Endianness = IO.Endianness.BigEndian;
// Rebuilt based on cpk_unpack
Accessor.Seek(0, IO.SeekOrigin.Begin);
// Rebuilt AGAIN based on github.com/esperknight/CriPakTools
string CPK = br.ReadFixedLengthString (4);
Accessor.Seek(0x10, IO.SeekOrigin.Begin);
int unknown1 = br.ReadInt32();
DatabaseTable dtUTF = ReadUTFTable(br);
long utf_size = br.ReadInt64(); // size of UTF not including "@UTF"
byte[] utf_data = br.ReadBytes(utf_size + 4);
// For now, just hardcode the TOC offset
long tocOffset = 2048; // (long)dtUTF.Records[0].Fields["TocOffset"].Value;
Accessor.Seek(tocOffset, IO.SeekOrigin.Begin);
MemoryAccessor ma = new MemoryAccessor(utf_data);
string utf_signature = ma.Reader.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
{
// encrypted?
utf_data = DecryptUTF(utf_data);
ma = new MemoryAccessor(utf_data);
}
else
{
ma.Reader.Seek(-4, IO.SeekOrigin.Current);
}
UTF.UTFDataFormat utf_df = new UTF.UTFDataFormat();
DatabaseObjectModel utf_om = new DatabaseObjectModel();
Document.Load(utf_om, utf_df, ma);
DatabaseTable dtUTF = utf_om.Tables[0];
// UTF table parsing works now, so no need to hardcode toc offset - WOOHOO!!!
ulong tocOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
br.Seek((long)tocOffset, IO.SeekOrigin.Begin);
string tocSignature = br.ReadFixedLengthString(4);
long tocEntries = (long)dtUTF.Records.Count;
unknown1 = br.ReadInt32();
string[] tocStringTable = ReadUTFStringTable(br);
// UTF table for TOC
utf_size = br.ReadInt64(); // size of UTF not including "@UTF"
utf_data = br.ReadBytes(utf_size + 4);
Accessor.Seek(2064, IO.SeekOrigin.Begin);
DatabaseTable dtUTF2 = ReadUTFTable(br);
ma = new MemoryAccessor(utf_data);
utf_signature = ma.Reader.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
{
// encrypted?
utf_data = DecryptUTF(utf_data);
ma = new MemoryAccessor(utf_data);
}
else
{
ma.Reader.Seek(-4, IO.SeekOrigin.Current);
}
utf_om = new DatabaseObjectModel();
Document.Load(utf_om, utf_df, ma);
DatabaseTable dtUTF2 = utf_om.Tables[0];
if (objectModel is DatabaseObjectModel)
{
@ -309,12 +104,73 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
}
else if (objectModel is FileSystemObjectModel)
{
for (int i = 0; i < dtUTF.Fields.Count; i++)
for (int i = 0; i < dtUTF2.Records.Count; i++)
{
fsom.Files.Add (dtUTF.Fields[i].Name);
string dirName = (string)dtUTF2.Records[i].Fields["DirName"].Value;
string fileTitle = (string)dtUTF2.Records[i].Fields["FileName"].Value;
string fileName = fileTitle;
if (!String.IsNullOrEmpty(dirName))
{
fileName = dirName + '/' + fileTitle;
}
uint decompressedLength = (uint)dtUTF2.Records[i].Fields["FileSize"].Value;
uint compressedLength = (uint)dtUTF2.Records[i].Fields["ExtractSize"].Value;
ulong offset = (ulong)dtUTF2.Records[i].Fields["FileOffset"].Value;
ulong lContentOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
offset += lContentOffset;
File f = fsom.AddFile(fileName);
f.Properties.Add("DecompressedLength", decompressedLength);
f.Properties.Add("CompressedLength", compressedLength);
f.Properties.Add("Offset", offset);
f.Size = decompressedLength;
f.DataRequest += f_DataRequest;
}
}
}
void f_DataRequest(object sender, DataRequestEventArgs e)
{
File f = (sender as File);
uint decompressedLength = (uint)f.Properties["DecompressedLength"];
uint compressedLength = (uint)f.Properties["CompressedLength"];
ulong offset = (ulong)f.Properties["Offset"];
Reader br = base.Accessor.Reader;
br.Accessor.Position = (long)offset;
byte[] decompressedData = null;
if (compressedLength == 0)
{
decompressedData = br.ReadBytes(decompressedLength);
}
else
{
byte[] compressedData = br.ReadBytes(compressedLength);
decompressedData = /*compress()*/ compressedData;
}
e.Data = decompressedData;
}
private byte[] DecryptUTF(byte[] input)
{
byte[] result = new byte[input.Length];
int m = 0x0000655f, t = 0x00004115;
for (int i = 0; i < input.Length; i++)
{
byte d = input[i];
d = (byte)(d ^ (byte)(m & 0xff));
result[i] = d;
m *= t;
}
return result;
}
protected override void SaveInternal (ObjectModel objectModel)
{
FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel);

282
CSharp/Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/UTF/UTFDataFormat.cs Normal file
View File

@ -0,0 +1,282 @@
//
// UTFDataFormat.cs
//
// Author:
// Mike Becker <alcexhim@gmail.com>
//
// Copyright (c) 2019 Mike Becker
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
using System;
using UniversalEditor.Accessors;
using UniversalEditor.IO;
using UniversalEditor.ObjectModels.Database;
namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK.UTF
{
public class UTFDataFormat : DataFormat
{
private struct UTFTABLEINFO
{
public long utfOffset;
public int tableSize;
public int schemaOffset;
public int rowsOffset;
public int stringTableOffset;
public int dataOffset;
public uint tableNameStringOffset;
public short tableColumns;
public short rowWidth;
public int tableRows;
public int stringTableSize;
}
private UTFTABLEINFO ReadUTFTableInfo(IO.Reader br)
{
UTFTABLEINFO info = new UTFTABLEINFO();
info.utfOffset = Accessor.Position;
info.tableSize = br.ReadInt32();
info.schemaOffset = 0x20;
info.rowsOffset = br.ReadInt32();
info.stringTableOffset = br.ReadInt32();
info.dataOffset = br.ReadInt32();
// CPK Header & UTF Header are ignored, so add 8 to each offset
info.tableNameStringOffset = br.ReadUInt32(); // 00000007
info.tableColumns = br.ReadInt16(); // 0023
info.rowWidth = br.ReadInt16(); // 007e
info.tableRows = br.ReadInt32(); // 00000001
info.stringTableSize = info.dataOffset - info.stringTableOffset;
return info;
}
protected override void LoadInternal(ref ObjectModel objectModel)
{
Reader br = base.Accessor.Reader;
string utf_signature = br.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK
DatabaseObjectModel utf = (objectModel as DatabaseObjectModel);
DatabaseTable dt = new DatabaseTable();
br.Endianness = IO.Endianness.BigEndian;
UTFTABLEINFO info = ReadUTFTableInfo(br);
int[] columnNameOffsets = new int[info.tableColumns];
long[] constantOffsets = new long[info.tableColumns];
CPKColumnStorageType[] storageTypes = new CPKColumnStorageType[info.tableColumns];
CPKColumnDataType[] dataTypes = new CPKColumnDataType[info.tableColumns];
for (int i = 0; i < info.tableColumns; i++)
{
byte schema = br.ReadByte();
/*// wtf is this?
if (schema == 0)
{
br.Accessor.Seek(3, SeekOrigin.Current);
column.flags = br.ReadByte();
}
*/
columnNameOffsets[i] = br.ReadInt32();
storageTypes[i] = (CPKColumnStorageType)(schema & (byte)CPKColumnStorageType.Mask);
dataTypes[i] = (CPKColumnDataType)(schema & (byte)CPKColumnDataType.Mask);
if (storageTypes[i] == CPKColumnStorageType.Constant)
{
constantOffsets[i] = br.Accessor.Position;
switch (dataTypes[i])
{
case CPKColumnDataType.Long:
case CPKColumnDataType.Long2:
case CPKColumnDataType.Data:
{
long dummy = br.ReadInt64();
break;
}
case CPKColumnDataType.Float:
{
float dummy = br.ReadSingle();
break;
}
case CPKColumnDataType.String:
case CPKColumnDataType.Int:
case CPKColumnDataType.Int2:
{
int dummy = br.ReadInt32();
break;
}
case CPKColumnDataType.Short:
case CPKColumnDataType.Short2:
{
short dummy = br.ReadInt16();
break;
}
case CPKColumnDataType.Byte:
case CPKColumnDataType.Byte2:
{
byte dummy = br.ReadByte();
break;
}
default:
{
Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString());
break;
}
}
}
dt.Fields.Add("Field" + i.ToString(), null);
}
// Read string table - remember, this is relative to UTF data WITH the "@UTF" signature
br.Seek(info.stringTableOffset + 8, IO.SeekOrigin.Begin);
byte[] stringTableData = br.ReadBytes(info.stringTableSize);
MemoryAccessor maStringTable = new MemoryAccessor(stringTableData);
maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin);
dt.Name = maStringTable.Reader.ReadNullTerminatedString();
for (int i = 0; i < info.tableColumns; i++)
{
maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin);
dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString();
}
for (int i = 0; i < info.tableRows; i++)
{
uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth));
uint rowStartOffset = rowOffset;
br.Accessor.Seek(rowOffset, SeekOrigin.Begin);
DatabaseRecord record = new DatabaseRecord();
for (int j = 0; j < info.tableColumns; j++)
{
CPKColumnStorageType storageType = storageTypes[j];
CPKColumnDataType dataType = dataTypes[j];
long constantOffset = constantOffsets[j] - 11;
switch (storageType)
{
case CPKColumnStorageType.PerRow:
break;
case CPKColumnStorageType.Constant:
break;
case CPKColumnStorageType.Zero:
record.Fields.Add(dt.Fields[j].Name, null);
continue;
}
long dataOffset1 = 0;
if (storageType == CPKColumnStorageType.Constant)
{
dataOffset1 = constantOffset;
}
else
{
dataOffset1 = rowOffset;
}
// br.Seek(dataOffset1, IO.SeekOrigin.Begin);
switch (dataType)
{
case CPKColumnDataType.String:
{
uint stringOffset = 0;
if (storageType == CPKColumnStorageType.Constant)
{
stringOffset = (uint)constantOffset;
}
else
{
stringOffset = br.ReadUInt32();
}
string value = null;
if (stringOffset < stringTableData.Length)
{
maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin);
value = maStringTable.Reader.ReadNullTerminatedString();
}
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Data:
{
uint varDataOffset = br.ReadUInt32();
uint varDataSize = br.ReadUInt32();
byte[] value = new byte[0];
record.Fields.Add(dt.Fields[j].Name, value);
// Is the data in another table??
// ReadUTFTable(br);
break;
}
case CPKColumnDataType.Long:
case CPKColumnDataType.Long2:
{
ulong value = br.ReadUInt64();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Int:
case CPKColumnDataType.Int2:
{
uint value = br.ReadUInt32();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Short:
case CPKColumnDataType.Short2:
{
ushort value = br.ReadUInt16();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Float:
{
float value = br.ReadSingle();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case CPKColumnDataType.Byte:
case CPKColumnDataType.Byte2:
{
byte value = br.ReadByte();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
}
}
dt.Records.Add(record);
}
utf.Tables.Add(dt);
}
protected override void SaveInternal(ObjectModel objectModel)
{
throw new NotImplementedException();
}
}
}

2
CSharp/Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj
View File

@ -36,11 +36,13 @@
<Compile Include="DataFormats\FileSystem\CPK\CPKColumnDataType.cs" />
<Compile Include="DataFormats\FileSystem\CPK\CPKColumnStorageType.cs" />
<Compile Include="DataFormats\FileSystem\CPK\CPKDataFormat.cs" />
<Compile Include="DataFormats\FileSystem\CPK\UTF\UTFDataFormat.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="DataFormats\" />
<Folder Include="DataFormats\FileSystem\" />
<Folder Include="DataFormats\FileSystem\CPK\" />
<Folder Include="DataFormats\FileSystem\CPK\UTF\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Libraries\UniversalEditor.Core\UniversalEditor.Core.csproj">