universaleditor/editor-legacy
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

still a work in progress, but added some more features to CRI/CPK DataFormat

Browse Source
This commit is contained in:
Michael Becker 2020-08-07 01:26:33 -04:00
parent d26aebcc99
commit 796d451f32
No known key found for this signature in database
GPG Key ID: 506F54899E2BFED7
5 changed files with 562 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

405
Plugins/UniversalEditor.Plugins.CRI/DataFormats/Database/UTF/UTFDataFormat.cs
View File

@ -83,218 +83,245 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.Database.UTF
throw new ObjectModelNotSupportedException();
Reader br = base.Accessor.Reader;
string utf_signature = br.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK
DatabaseTable dt = new DatabaseTable();
br.Endianness = IO.Endianness.BigEndian;
UTFTABLEINFO info = ReadUTFTableInfo(br);
int[] columnNameOffsets = new int[info.tableColumns];
long[] constantOffsets = new long[info.tableColumns];
UTFColumnStorageType[] storageTypes = new UTFColumnStorageType[info.tableColumns];
UTFColumnDataType[] dataTypes = new UTFColumnDataType[info.tableColumns];
// Read string table - remember, this is relative to UTF data WITH the "@UTF" signature
br.Accessor.SavePosition();
br.Seek(info.stringTableOffset + 8, IO.SeekOrigin.Begin);
while (br.PeekByte() == 0)
while (!br.EndOfStream)
{
br.ReadByte();
}
string utf_signature = br.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
throw new InvalidDataFormatException(); // we are assuming passed in decrypted UTF from the CPK
byte[] stringTableData = br.ReadBytes(info.stringTableSize);
MemoryAccessor maStringTable = new MemoryAccessor(stringTableData);
DatabaseTable dt = new DatabaseTable();
maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin);
dt.Name = maStringTable.Reader.ReadNullTerminatedString();
br.Accessor.LoadPosition();
br.Endianness = IO.Endianness.BigEndian;
for (int i = 0; i < info.tableColumns; i++)
{
byte schema = br.ReadByte();
/*// wtf is this?
if (schema == 0)
UTFTABLEINFO info = ReadUTFTableInfo(br);
int[] columnNameOffsets = new int[info.tableColumns];
long[] constantOffsets = new long[info.tableColumns];
UTFColumnStorageType[] storageTypes = new UTFColumnStorageType[info.tableColumns];
UTFColumnDataType[] dataTypes = new UTFColumnDataType[info.tableColumns];
// Read string table - remember, this is relative to UTF data WITH the "@UTF" signature
br.Accessor.SavePosition();
br.Seek(info.utfOffset + info.stringTableOffset + 4, IO.SeekOrigin.Begin);
while (br.PeekByte() == 0)
{
br.Accessor.Seek(3, SeekOrigin.Current);
column.flags = br.ReadByte();
br.ReadByte();
}
*/
columnNameOffsets[i] = br.ReadInt32();
storageTypes[i] = (UTFColumnStorageType)(schema & (byte)UTFColumnStorageType.Mask);
dataTypes[i] = (UTFColumnDataType)(schema & (byte)UTFColumnDataType.Mask);
byte[] stringTableData = br.ReadBytes(info.stringTableSize);
MemoryAccessor maStringTable = new MemoryAccessor(stringTableData);
object constantValue = null;
if (storageTypes[i] == UTFColumnStorageType.Constant)
maStringTable.Reader.Seek(info.tableNameStringOffset, IO.SeekOrigin.Begin);
dt.Name = maStringTable.Reader.ReadNullTerminatedString();
br.Accessor.LoadPosition();
for (int i = 0; i < info.tableColumns; i++)
{
constantOffsets[i] = br.Accessor.Position;
switch (dataTypes[i])
byte schema = br.ReadByte();
/*// wtf is this?
if (schema == 0)
{
case UTFColumnDataType.Long:
case UTFColumnDataType.Long2:
case UTFColumnDataType.Data:
{
constantValue = br.ReadInt64();
break;
}
case UTFColumnDataType.Float:
{
constantValue = br.ReadSingle();
break;
}
case UTFColumnDataType.String:
{
int valueOffset = br.ReadInt32();
maStringTable.Reader.Seek(valueOffset, IO.SeekOrigin.Begin);
constantValue = maStringTable.Reader.ReadNullTerminatedString();
break;
}
case UTFColumnDataType.Int:
case UTFColumnDataType.Int2:
{
constantValue = br.ReadInt32();
break;
}
case UTFColumnDataType.Short:
case UTFColumnDataType.Short2:
{
constantValue = br.ReadInt16();
break;
}
case UTFColumnDataType.Byte:
case UTFColumnDataType.Byte2:
{
constantValue = br.ReadByte();
break;
}
default:
{
Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString());
break;
}
br.Accessor.Seek(3, SeekOrigin.Current);
column.flags = br.ReadByte();
}
}
*/
columnNameOffsets[i] = br.ReadInt32();
dt.Fields.Add("Field" + i.ToString(), constantValue, SystemDataTypeForUTFDataType(dataTypes[i]));
}
storageTypes[i] = (UTFColumnStorageType)(schema & (byte)UTFColumnStorageType.Mask);
dataTypes[i] = (UTFColumnDataType)(schema & (byte)UTFColumnDataType.Mask);
for (int i = 0; i < info.tableColumns; i++)
{
maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin);
dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString();
}
for (int i = 0; i < info.tableRows; i++)
{
uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth));
uint rowStartOffset = rowOffset;
br.Accessor.Seek(rowOffset, SeekOrigin.Begin);
DatabaseRecord record = new DatabaseRecord();
for (int j = 0; j < info.tableColumns; j++)
{
UTFColumnStorageType storageType = storageTypes[j];
UTFColumnDataType dataType = dataTypes[j];
long constantOffset = constantOffsets[j] - 11;
switch (storageType)
object constantValue = null;
if (storageTypes[i] == UTFColumnStorageType.Constant)
{
case UTFColumnStorageType.PerRow:
constantOffsets[i] = br.Accessor.Position;
switch (dataTypes[i])
{
switch (dataType)
case UTFColumnDataType.Long:
case UTFColumnDataType.Long2:
case UTFColumnDataType.Data:
{
case UTFColumnDataType.String:
{
string value = null;
if (storageType == UTFColumnStorageType.Constant)
{
value = (dt.Fields[j].Value as string);
}
else
{
uint stringOffset = br.ReadUInt32();
if (stringOffset < stringTableData.Length)
{
maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin);
value = maStringTable.Reader.ReadNullTerminatedString();
}
}
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Data:
{
uint varDataOffset = br.ReadUInt32();
uint varDataSize = br.ReadUInt32();
byte[] value = new byte[0];
record.Fields.Add(dt.Fields[j].Name, value);
// Is the data in another table??
// ReadUTFTable(br);
break;
}
case UTFColumnDataType.Long:
case UTFColumnDataType.Long2:
{
ulong value = br.ReadUInt64();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Int:
case UTFColumnDataType.Int2:
{
uint value = br.ReadUInt32();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Short:
case UTFColumnDataType.Short2:
{
ushort value = br.ReadUInt16();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Float:
{
float value = br.ReadSingle();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Byte:
case UTFColumnDataType.Byte2:
{
byte value = br.ReadByte();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
constantValue = br.ReadInt64();
break;
}
case UTFColumnDataType.Float:
{
constantValue = br.ReadSingle();
break;
}
case UTFColumnDataType.String:
{
int valueOffset = br.ReadInt32();
maStringTable.Reader.Seek(valueOffset, IO.SeekOrigin.Begin);
constantValue = maStringTable.Reader.ReadNullTerminatedString();
break;
}
case UTFColumnDataType.Int:
case UTFColumnDataType.Int2:
{
constantValue = br.ReadInt32();
break;
}
case UTFColumnDataType.Short:
case UTFColumnDataType.Short2:
{
constantValue = br.ReadInt16();
break;
}
case UTFColumnDataType.Byte:
case UTFColumnDataType.Byte2:
{
constantValue = br.ReadByte();
break;
}
default:
{
Console.WriteLine("cpk: ReadUTFTable: unknown data type for column " + i.ToString());
break;
}
break;
}
case UTFColumnStorageType.Constant:
{
record.Fields.Add(dt.Fields[j].Name, dt.Fields[j].Value);
continue;
}
case UTFColumnStorageType.Zero:
{
record.Fields.Add(dt.Fields[j].Name, null);
continue;
}
}
dt.Fields.Add("Field" + i.ToString(), constantValue, SystemDataTypeForUTFDataType(dataTypes[i]));
}
dt.Records.Add(record);
for (int i = 0; i < info.tableColumns; i++)
{
maStringTable.Reader.Seek(columnNameOffsets[i], IO.SeekOrigin.Begin);
dt.Fields[i].Name = maStringTable.Reader.ReadNullTerminatedString();
}
for (int i = 0; i < info.tableRows; i++)
{
uint rowOffset = (uint)(info.utfOffset + 4 + info.rowsOffset + (i * info.rowWidth));
uint rowStartOffset = rowOffset;
br.Accessor.Seek(rowOffset, SeekOrigin.Begin);
DatabaseRecord record = new DatabaseRecord();
for (int j = 0; j < info.tableColumns; j++)
{
UTFColumnStorageType storageType = storageTypes[j];
UTFColumnDataType dataType = dataTypes[j];
long constantOffset = constantOffsets[j] - 11;
switch (storageType)
{
case UTFColumnStorageType.PerRow:
{
switch (dataType)
{
case UTFColumnDataType.String:
{
string value = null;
if (storageType == UTFColumnStorageType.Constant)
{
value = (dt.Fields[j].Value as string);
}
else
{
uint stringOffset = br.ReadUInt32();
if (stringOffset < stringTableData.Length)
{
maStringTable.Reader.Seek(stringOffset, IO.SeekOrigin.Begin);
value = maStringTable.Reader.ReadNullTerminatedString();
}
}
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Data:
{
uint varDataOffset = br.ReadUInt32();
uint varDataSize = br.ReadUInt32();
/*
br.Accessor.SavePosition();
br.Seek(varDataOffset, SeekOrigin.Begin);
byte[] value = br.ReadBytes(varDataSize);
br.Accessor.LoadPosition();
*/
byte[] value = new byte[0];
record.Fields.Add(dt.Fields[j].Name, value);
// Is the data in another table??
// ReadUTFTable(br);
break;
}
case UTFColumnDataType.Long:
case UTFColumnDataType.Long2:
{
ulong value = br.ReadUInt64();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Int:
case UTFColumnDataType.Int2:
{
uint value = br.ReadUInt32();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Short:
case UTFColumnDataType.Short2:
{
ushort value = br.ReadUInt16();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Float:
{
float value = br.ReadSingle();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
case UTFColumnDataType.Byte:
case UTFColumnDataType.Byte2:
{
byte value = br.ReadByte();
record.Fields.Add(dt.Fields[j].Name, value);
break;
}
}
break;
}
case UTFColumnStorageType.Constant:
{
record.Fields.Add(dt.Fields[j].Name, dt.Fields[j].Value);
continue;
}
case UTFColumnStorageType.Zero:
{
record.Fields.Add(dt.Fields[j].Name, null);
continue;
}
}
}
dt.Records.Add(record);
}
utf.Tables.Add(dt);
br.Seek(maStringTable.Length, SeekOrigin.Current);
string sig = br.ReadFixedLengthString(4);
br.Seek(-4, SeekOrigin.Current);
if (sig == "@UTF")
{
continue;
}
if (br.Accessor.Position + 8 < br.Accessor.Length)
{
br.Seek(8, SeekOrigin.Current);
}
else
{
break;
}
}
utf.Tables.Add(dt);
}
public static Type SystemDataTypeForUTFDataType(UTFColumnDataType dataType)

360
Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKDataFormat.cs
View File

@ -46,17 +46,33 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
{
_dfr = base.MakeReferenceInternal();
_dfr.Capabilities.Add(typeof(FileSystemObjectModel), DataFormatCapabilities.All);
_dfr.ExportOptions.Add(new CustomOptionChoice(nameof(Mode), "File _mode", true, new CustomOptionFieldChoice[]
{
new CustomOptionFieldChoice("ID only", CPKFileMode.IDOnly),
new CustomOptionFieldChoice("Filename only", CPKFileMode.FilenameOnly),
new CustomOptionFieldChoice("ID + Filename", CPKFileMode.IDFilename),
new CustomOptionFieldChoice("Filename + Group (Attribute)", CPKFileMode.FilenameGroup),
new CustomOptionFieldChoice("ID + Group (Attribute)", CPKFileMode.IDGroup),
new CustomOptionFieldChoice("Filename + ID + Group (Attribute)", CPKFileMode.FilenameIDGroup)
}));
_dfr.ExportOptions.Add(new CustomOptionText(nameof(VersionString), "_Version string", "CPKMC2.14.00, DLL2.74.00"));
_dfr.ExportOptions.Add(new CustomOptionNumber(nameof(SectorAlignment), "Sector _alignment", 2048, 0, 2048));
_dfr.ExportOptions.Add(new CustomOptionBoolean(nameof(ScrambleDirectoryInformation), "_Scramble directory information"));
_dfr.ExportOptions.Add(new CustomOptionBoolean(nameof(ForceCompression), "Force _compression"));
}
return _dfr;
}
public CPKFileMode Mode { get; set; } = CPKFileMode.FilenameOnly;
/// <summary>
/// Gets or sets the version string which contains information about the library which created the archive. The default value is "CPKMC2.14.00, DLL2.74.00" which is the version string that official CRI Middleware CPK tools use.
/// </summary>
/// <value>The version string.</value>
public string VersionString { get; set; } = "CPKMC2.14.00, DLL2.74.00";
public string VersionString { get; set; } = "CPKMC2.14.00, DLL2.74.00"; // "CPKFBSTD1.49.34, DLL3.24.00"
public bool ScrambleDirectoryInformation { get; set; } = false;
public bool ForceCompression { get; set; } = false;
/// <summary>
/// Gets or sets the sector alignment, in bytes, of the CPK archive. The default value is 2048.
@ -68,10 +84,15 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
private byte[] _HeaderData = null;
/// <summary>
/// Returns the raw data from the initial "CPK " chunk of this file, or NULL if this chunk does not exist.
/// Returns the raw data from the initial "CPK " chunk of this file, or <see langword="null"/> if this chunk does not exist.
/// </summary>
/// <value>The raw data from the "CPK " chunk.</value>
public byte[] HeaderData { get { return _HeaderData; } }
/// <summary>
/// Returns the UTF table from the initial "CPK " chunk of this file, or <see langword="null"/> if this chunk does not exist.
/// </summary>
/// <value>The header table.</value>
public DatabaseTable HeaderTable { get; private set; } = null;
private byte[] _TocData = null;
/// <summary>
@ -111,6 +132,8 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
if (fsom == null)
throw new ObjectModelNotSupportedException();
fsom.AdditionalDetails.Add("CRI.CPK.FileID", "ID");
Reader br = Accessor.Reader;
// Rebuilt based on cpk_unpack
@ -118,90 +141,211 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
DatabaseObjectModel utf_om = ReadUTF("CPK ", br, out _HeaderData);
DatabaseTable dtUTF = utf_om.Tables[0];
// UTF table parsing works now, so no need to hardcode toc offset - WOOHOO!!!
ulong tocOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
br.Seek((long)tocOffset, IO.SeekOrigin.Begin);
utf_om = ReadUTF("TOC ", br, out _TocData);
DatabaseTable dtUTFTOC = utf_om.Tables[0];
ulong itocOffset = (ulong)dtUTF.Records[0].Fields["ItocOffset"].Value;
br.Seek((long)itocOffset, IO.SeekOrigin.Begin);
utf_om = ReadUTF("ITOC", br, out _ITocData);
DatabaseTable dtUTFITOC = utf_om.Tables[0];
HeaderTable = dtUTF;
if (objectModel is DatabaseObjectModel)
{
(objectModel as DatabaseObjectModel).Tables.Add (dtUTF);
}
else if (objectModel is FileSystemObjectModel)
{
for (int i = 0; i < dtUTFTOC.Records.Count; i++)
(objectModel as DatabaseObjectModel).Tables.Add(dtUTF);
}
DatabaseTable dtUTFTOC = null, dtUTFITOC = null, dtUTFITOC_L = null, dtUTFITOC_H = null, dtUTFETOC = null;
if (dtUTF.Records[0].Fields["Tvers"].Value != null)
{
VersionString = dtUTF.Records[0].Fields["Tvers"].Value.ToString();
}
// UTF table parsing works now, so no need to hardcode toc offset - WOOHOO!!!
if (dtUTF.Records[0].Fields["TocOffset"].Value != null)
{
ulong tocOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
br.Seek((long)tocOffset, IO.SeekOrigin.Begin);
utf_om = ReadUTF("TOC ", br, out _TocData);
dtUTFTOC = utf_om.Tables[0];
}
if (dtUTF.Records[0].Fields["ItocOffset"].Value != null)
{
// Index TOC
ulong itocOffset = (ulong)dtUTF.Records[0].Fields["ItocOffset"].Value;
br.Seek((long)itocOffset, IO.SeekOrigin.Begin);
utf_om = ReadUTF("ITOC", br, out _ITocData);
dtUTFITOC = utf_om.Tables[0];
dtUTFITOC_L = utf_om.Tables["CpkItocL"];
dtUTFITOC_H = utf_om.Tables["CpkItocH"];
}
if (dtUTF.Records[0].Fields["GtocOffset"].Value != null)
{
// Groups TOC
ulong gtocOffset = (ulong)dtUTF.Records[0].Fields["GtocOffset"].Value;
br.Seek((long)gtocOffset, SeekOrigin.Begin);
utf_om = ReadUTF("GTOC", br, out _GTocData);
DatabaseTable dtUTFGTOC = utf_om.Tables[0];
DatabaseTable dtCpkGtocAttr = utf_om.Tables["CpkGtocAttr"];
List<string> listAttribs = new List<string>();
if (dtCpkGtocAttr != null)
{
string dirName = (string)dtUTFTOC.Records[i].Fields["DirName"].Value;
string fileTitle = (string)dtUTFTOC.Records[i].Fields["FileName"].Value;
string fileName = fileTitle;
if (!String.IsNullOrEmpty(dirName))
for (int i = 0; i < dtCpkGtocAttr.Records.Count; i++)
{
fileName = dirName + '/' + fileTitle;
string attribName = dtCpkGtocAttr.Records[i].Fields["Aname"]?.Value?.ToString();
listAttribs.Add(attribName);
}
uint decompressedLength = (uint)dtUTFTOC.Records[i].Fields["FileSize"].Value;
uint compressedLength = (uint)dtUTFTOC.Records[i].Fields["ExtractSize"].Value;
ulong offset = (ulong)dtUTFTOC.Records[i].Fields["FileOffset"].Value;
ulong lContentOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
offset += lContentOffset;
File f = fsom.AddFile(fileName);
if (dtUTFTOC.Records[i].Fields["ID"] != null)
f.Properties.Add("ID", (uint) dtUTFTOC.Records[i].Fields["ID"].Value);
f.Properties.Add("DecompressedLength", decompressedLength);
f.Properties.Add("CompressedLength", compressedLength);
f.Properties.Add("Offset", offset);
f.Properties.Add("Reader", br);
f.Size = decompressedLength;
f.DataRequest += f_DataRequest;
}
}
ulong etocOffset = (ulong)dtUTF.Records[0].Fields["EtocOffset"].Value;
br.Seek((long)etocOffset, SeekOrigin.Begin);
utf_om = ReadUTF("ETOC", br, out _ETocData);
DatabaseTable dtUTFETOC = utf_om.Tables[0];
if (objectModel is DatabaseObjectModel)
if (dtUTFTOC != null)
{
(objectModel as DatabaseObjectModel).Tables.Add(dtUTFETOC);
}
else if (objectModel is FileSystemObjectModel)
{
for (int i = 0; i < dtUTFETOC.Records.Count; i++)
if (objectModel is DatabaseObjectModel)
{
(objectModel as DatabaseObjectModel).Tables.Add(dtUTFTOC);
}
else if (objectModel is FileSystemObjectModel)
{
ulong updateDateTime = (ulong)dtUTFETOC.Records[i].Fields["UpdateDateTime"].Value;
string localDir = (string)dtUTFETOC.Records[i].Fields["LocalDir"].Value;
for (int i = 0; i < dtUTFTOC.Records.Count; i++)
{
string dirName = (string)dtUTFTOC.Records[i].Fields["DirName"].Value;
string fileTitle = (string)dtUTFTOC.Records[i].Fields["FileName"].Value;
string fileName = fileTitle;
if (!String.IsNullOrEmpty(dirName))
{
fileName = dirName + '/' + fileTitle;
}
if (i >= fsom.Files.Count)
continue;
uint decompressedLength = (uint)dtUTFTOC.Records[i].Fields["FileSize"].Value;
uint compressedLength = (uint)dtUTFTOC.Records[i].Fields["ExtractSize"].Value;
ulong offset = (ulong)dtUTFTOC.Records[i].Fields["FileOffset"].Value;
ulong lContentOffset = (ulong)dtUTF.Records[0].Fields["TocOffset"].Value;
offset += lContentOffset;
File f = fsom.Files[i];
File f = fsom.AddFile(fileName);
byte[] updateDateTimeBytes = BitConverter.GetBytes(updateDateTime);
// remember, the CPK is big-endian, but the UTF is little-endian
ushort updateDateTimeYear = BitConverter.ToUInt16(new byte[] { updateDateTimeBytes[6], updateDateTimeBytes[7] }, 0);
byte updateDateTimeMonth = updateDateTimeBytes[5];
byte updateDateTimeDay = updateDateTimeBytes[4];
byte updateDateTimeHour = updateDateTimeBytes[3];
byte updateDateTimeMinute = updateDateTimeBytes[2];
byte updateDateTimeSecond = updateDateTimeBytes[1];
byte updateDateTimeMs = updateDateTimeBytes[0];
if (dtUTFTOC.Records[i].Fields["ID"] != null)
{
uint id = (uint)dtUTFTOC.Records[i].Fields["ID"].Value;
f.Properties.Add("ID", id);
f.SetAdditionalDetail("CRI.CPK.FileID", id.ToString());
}
f.ModificationTimestamp = new DateTime(updateDateTimeYear, updateDateTimeMonth, updateDateTimeDay, updateDateTimeHour, updateDateTimeMinute, updateDateTimeSecond, updateDateTimeMs);
f.Properties.Add("DecompressedLength", decompressedLength);
f.Properties.Add("CompressedLength", compressedLength);
f.Properties.Add("Offset", offset);
f.Properties.Add("Reader", br);
f.Size = decompressedLength;
f.DataRequest += f_DataRequest;
}
}
}
else if (dtUTFITOC_L != null || dtUTFITOC_H != null)
{
ulong lContentOffset = (ulong)dtUTF.Records[0].Fields["ContentOffset"].Value;
ulong offset = lContentOffset;
List<File> list = new List<File>();
if (dtUTFITOC_L != null)
{
for (int i = 0; i < dtUTFITOC_L.Records.Count; i++)
{
ushort decompressedLength = (ushort)dtUTFITOC_L.Records[i].Fields["FileSize"].Value;
ushort compressedLength = (ushort)dtUTFITOC_L.Records[i].Fields["ExtractSize"].Value;
File f = new File();
ushort id = (ushort)dtUTFITOC_L.Records[i].Fields["ID"].Value;
f.Name = id.ToString();
f.SetAdditionalDetail("CRI.CPK.FileID", id.ToString());
f.Properties.Add("ID", id);
f.Properties.Add("DecompressedLength", (uint)decompressedLength);
f.Properties.Add("CompressedLength", (uint)compressedLength);
f.Properties.Add("Offset", (ulong)0);
f.Properties.Add("Reader", br);
f.Size = decompressedLength;
f.DataRequest += f_DataRequest;
list.Add(f);
}
}
if (dtUTFITOC_H != null)
{
for (int i = 0; i < dtUTFITOC_H.Records.Count; i++)
{
uint decompressedLength = (uint)dtUTFITOC_H.Records[i].Fields["FileSize"].Value;
uint compressedLength = (uint)dtUTFITOC_H.Records[i].Fields["ExtractSize"].Value;
File f = new File();
ushort id = (ushort)dtUTFITOC_H.Records[i].Fields["ID"].Value;
f.Name = id.ToString();
f.SetAdditionalDetail("CRI.CPK.FileID", id.ToString());
f.Properties.Add("ID", id);
f.Properties.Add("DecompressedLength", decompressedLength);
f.Properties.Add("CompressedLength", compressedLength);
f.Properties.Add("Offset", (ulong)0);
f.Properties.Add("Reader", br);
f.Size = decompressedLength;
f.DataRequest += f_DataRequest;
list.Add(f);
}
}
// sort them by ID - this is important because the data is stored contiguously
list.Sort(new Comparison<File>((x, y) => ((ushort)x.Properties["ID"]).CompareTo((ushort)y.Properties["ID"])));
for (int i = 0; i < list.Count; i++)
{
list[i].Properties["Offset"] = offset;
offset += (uint)list[i].Properties["CompressedLength"];
offset = offset.RoundUp(SectorAlignment);
fsom.Files.Add(list[i]);
}
}
if (dtUTF.Records[0].Fields["EtocOffset"].Value != null)
{
ulong etocOffset = (ulong)dtUTF.Records[0].Fields["EtocOffset"].Value;
br.Seek((long)etocOffset, SeekOrigin.Begin);
utf_om = ReadUTF("ETOC", br, out _ETocData);
dtUTFETOC = utf_om.Tables[0];
}
if (dtUTFETOC != null)
{
if (objectModel is DatabaseObjectModel)
{
(objectModel as DatabaseObjectModel).Tables.Add(dtUTFETOC);
}
else if (objectModel is FileSystemObjectModel)
{
for (int i = 0; i < dtUTFETOC.Records.Count; i++)
{
ulong updateDateTime = (ulong)dtUTFETOC.Records[i].Fields["UpdateDateTime"].Value;
string localDir = (string)dtUTFETOC.Records[i].Fields["LocalDir"].Value;
if (i >= fsom.Files.Count)
continue;
File f = fsom.Files[i];
byte[] updateDateTimeBytes = BitConverter.GetBytes(updateDateTime);
// remember, the CPK is big-endian, but the UTF is little-endian
ushort updateDateTimeYear = BitConverter.ToUInt16(new byte[] { updateDateTimeBytes[6], updateDateTimeBytes[7] }, 0);
byte updateDateTimeMonth = updateDateTimeBytes[5];
byte updateDateTimeDay = updateDateTimeBytes[4];
byte updateDateTimeHour = updateDateTimeBytes[3];
byte updateDateTimeMinute = updateDateTimeBytes[2];
byte updateDateTimeSecond = updateDateTimeBytes[1];
byte updateDateTimeMs = updateDateTimeBytes[0];
f.ModificationTimestamp = new DateTime(updateDateTimeYear, updateDateTimeMonth, updateDateTimeDay, updateDateTimeHour, updateDateTimeMinute, updateDateTimeSecond, updateDateTimeMs);
}
}
}
}
@ -222,6 +366,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
string utf_signature = ma.Reader.ReadFixedLengthString(4);
if (utf_signature != "@UTF")
{
ScrambleDirectoryInformation = true;
// encrypted?
utf_data = DecryptUTF(utf_data);
ma = new MemoryAccessor(utf_data);
@ -263,7 +408,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
e.Data = decompressedData;
}
private DatabaseObjectModel BuildHeaderUTF(FileSystemObjectModel fsom, int tocsize, ulong contentOffset, ulong contentSize, ulong etocOffset, ulong etocLength, ulong itocOffset, ulong itocLength)
private DatabaseObjectModel BuildHeaderUTF(FileSystemObjectModel fsom, int tocsize, ulong contentOffset, ulong contentSize, ulong etocOffset, ulong etocLength, ulong? itocOffset, ulong? itocLength)
{
File[] files = fsom.GetAllFiles();
@ -276,6 +421,11 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
dt.Fields.Add("TocOffset", null, typeof(Int64));
dt.Fields.Add("TocSize", null, typeof(Int64));
dt.Fields.Add("TocCrc", null, typeof(uint));
// added in newer version CpkFileBuilder
dt.Fields.Add("HtocOffset", null, typeof(Int64));
dt.Fields.Add("HtocSize", null, typeof(Int64));
dt.Fields.Add("EtocOffset", null, typeof(Int64));
dt.Fields.Add("EtocSize", null, typeof(Int64));
dt.Fields.Add("ItocOffset", null, typeof(Int64));
@ -284,6 +434,11 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
dt.Fields.Add("GtocOffset", null, typeof(Int64));
dt.Fields.Add("GtocSize", null, typeof(Int64));
dt.Fields.Add("GtocCrc", null, typeof(uint));
// added in newer version CpkFileBuilder
dt.Fields.Add("HgtocOffset", null, typeof(Int64));
dt.Fields.Add("HgtocSize", null, typeof(Int64));
dt.Fields.Add("EnabledPackedSize", null, typeof(Int64));
dt.Fields.Add("EnabledDataSize", null, typeof(Int64));
dt.Fields.Add("TotalDataSize", null, typeof(Int64));
@ -298,6 +453,10 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
dt.Fields.Add("Revision", null, typeof(Int16));
dt.Fields.Add("Align", null, typeof(Int16));
dt.Fields.Add("Sorted", null, typeof(Int16));
// added in newer version CpkFileBuilder
dt.Fields.Add("EnableFileName", null, typeof(Int16));
dt.Fields.Add("EID", null, typeof(Int16));
dt.Fields.Add("CpkMode", null, typeof(uint));
dt.Fields.Add("Tvers", null, typeof(string));
@ -305,6 +464,12 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
dt.Fields.Add("Codec", null, typeof(uint));
dt.Fields.Add("DpkItoc", null, typeof(uint));
//added in newer version CpkFileBuilder
dt.Fields.Add("EnableTocCrc", null, typeof(Int16));
dt.Fields.Add("EnableFileCrc", null, typeof(Int16));
dt.Fields.Add("CrcMode", null, typeof(uint));
dt.Fields.Add("CrcTable", null, typeof(byte[]));
// cri, go home, you're drunk
ulong enabledPackedSize = 0;
for (uint i = 0; i < files.Length; i++)
@ -324,6 +489,11 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
new DatabaseField("TocOffset", (ulong)SectorAlignment),
new DatabaseField("TocSize", (ulong)tocsize),
new DatabaseField("TocCrc", null),
// added in newer version CpkFileBuilder
new DatabaseField("HtocOffset", 0),
new DatabaseField("HtocSize", 0),
new DatabaseField("EtocOffset", etocOffset),
new DatabaseField("EtocSize", etocLength),
new DatabaseField("ItocOffset", itocOffset),
@ -332,6 +502,11 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
new DatabaseField("GtocOffset", null),
new DatabaseField("GtocSize", null),
new DatabaseField("GtocCrc", null),
// added in newer version CpkFileBuilder
new DatabaseField("HgtocOffset", null),
new DatabaseField("HgtocSize", null),
new DatabaseField("EnabledPackedSize", enabledPackedSize), //16434944 in diva2script.cpk
new DatabaseField("EnabledDataSize", enabledDataSize),
new DatabaseField("TotalDataSize", null),
@ -346,12 +521,22 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
new DatabaseField("Revision", (ushort)0),
new DatabaseField("Align", (ushort)SectorAlignment),
new DatabaseField("Sorted", (ushort)1),
// added in newer version CpkFileBuilder
new DatabaseField("EnableFileName", (ushort)0),
new DatabaseField("EID", (ushort)1),
new DatabaseField("CpkMode", (uint)2),
new DatabaseField("CpkMode", (uint)Mode),
new DatabaseField("Tvers", VersionString),
new DatabaseField("Comment", null),
new DatabaseField("Codec", (uint)0),
new DatabaseField("DpkItoc", (uint)0)
new DatabaseField("DpkItoc", (uint)0),
// added in newer version CpkFileBuilder
new DatabaseField("EnableTocCrc", (short)0),
new DatabaseField("EnableFileCrc", (short)0),
new DatabaseField("CrcMode", (uint)0),
new DatabaseField("CrcTable", null)
}));
DatabaseObjectModel db = new DatabaseObjectModel();
@ -376,22 +561,6 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
return size;
}
private struct IDOFFSET
{
public int INDEX;
public uint ID;
public ulong OFFSET;
public ulong SIZE;
public IDOFFSET(int index, uint id, ulong offset, ulong size)
{
INDEX = index;
ID = id;
OFFSET = offset;
SIZE = size;
}
}
private DatabaseObjectModel BuildTocUTF(File[] files, ulong initialFileOffset, ref IDOFFSET[] sortedOffsets)
{
DatabaseTable dt = new DatabaseTable();
@ -571,6 +740,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
contentOffset = contentOffset.RoundUp(SectorAlignment);
tocLength = (ulong) _tmp_ma.Length;
/*
_tmp_om = BuildItocUTF(sortedOffsets);
_tmp_ma = new MemoryAccessor();
Document.Save(_tmp_om, dfUTF, _tmp_ma);
@ -581,6 +751,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
contentOffset += (ulong)_tmp_ma.Length;
contentOffset = contentOffset.RoundUp(SectorAlignment);
itocLength = (ulong) _tmp_ma.Length; // 21728
*/
_tmp_om = BuildEtocUTF(files);
_tmp_ma = new MemoryAccessor();
@ -614,7 +785,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
ulong etocOffset = 0;
etocOffset = contentOffset + contentSize;
DatabaseObjectModel utfHeader = BuildHeaderUTF(fsom, utfTOC_data.Length + 16 /*includes 16-byte 'TOC ' header from CPK*/, contentOffset, contentSize, etocOffset, etocLength + 16, itocOffset, itocLength + 16);
DatabaseObjectModel utfHeader = BuildHeaderUTF(fsom, utfTOC_data.Length + 16 /*includes 16-byte 'TOC ' header from CPK*/, contentOffset, contentSize, etocOffset, etocLength + 16, null, null); // itocOffset, itocLength + 16);
MemoryAccessor maUTFHeader = new MemoryAccessor();
Document.Save(utfHeader, dfUTF, maUTFHeader);
@ -622,6 +793,9 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
bw.WriteInt64(utfHeader_data.Length);
bw.WriteBytes(utfHeader_data);
int __unknown_checksum = -1677552896; // 9634460;
bw.WriteInt32(__unknown_checksum);
bw.Align(SectorAlignment);
bw.Accessor.Seek(-6, SeekOrigin.Current);
bw.WriteFixedLengthString("(c)CRI");
@ -634,6 +808,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
bw.Align(SectorAlignment);
/*
// here comes the ITOC (indexes TOC) UTF table chunk.
DatabaseObjectModel utfITOC = BuildItocUTF(sortedOffsets);
MemoryAccessor maUTFITOC = new MemoryAccessor();
@ -646,6 +821,7 @@ namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
bw.WriteInt32(255);
bw.WriteInt64(utfITOC_data.Length);
bw.WriteBytes(utfITOC_data);
*/
// here comes the file data. each file is aligned to FileAlignment bytes, apparently.
bw.Align(SectorAlignment);

33
Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/CPKFileMode.cs Normal file
View File

@ -0,0 +1,33 @@
//
// CPKFileMode.cs
//
// Author:
// Michael Becker <alcexhim@gmail.com>
//
// Copyright (c) 2020 Mike Becker's Software
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
using System;
namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
{
public enum CPKFileMode
{
IDOnly = 0,
FilenameOnly = 1,
IDFilename = 2,
FilenameGroup = 3,
IDGroup = 4,
FilenameIDGroup = 5
}
}

42
Plugins/UniversalEditor.Plugins.CRI/DataFormats/FileSystem/CPK/Internal/IDOFFSET.cs Normal file
View File

@ -0,0 +1,42 @@
//
// IDOFFSET.cs - internal structure for representing an ID, offset, and size for a file in a CPK archive
//
// Author:
// Michael Becker <alcexhim@gmail.com>
//
// Copyright (c) 2020 Mike Becker's Software
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
namespace UniversalEditor.Plugins.CRI.DataFormats.FileSystem.CPK
{
/// <summary>
/// Internal structure for representing an ID, offset, and size for a file in a CPK archive.
/// </summary>
internal struct IDOFFSET
{
public int INDEX;
public uint ID;
public ulong OFFSET;
public ulong SIZE;
public IDOFFSET(int index, uint id, ulong offset, ulong size)
{
INDEX = index;
ID = id;
OFFSET = offset;
SIZE = size;
}
}
}

3
Plugins/UniversalEditor.Plugins.CRI/UniversalEditor.Plugins.CRI.csproj
View File

@ -41,6 +41,8 @@
<Compile Include="DataFormats\Database\UTF\UTFDataFormat.cs" />
<Compile Include="DataFormats\Database\UTF\UTFColumnDataType.cs" />
<Compile Include="DataFormats\Database\UTF\UTFColumnStorageType.cs" />
<Compile Include="DataFormats\FileSystem\CPK\CPKFileMode.cs" />
<Compile Include="DataFormats\FileSystem\CPK\Internal\IDOFFSET.cs" />
</ItemGroup>
<ItemGroup>
<Folder Include="DataFormats\" />
@ -49,6 +51,7 @@
<Folder Include="DataFormats\FileSystem\AFS\" />
<Folder Include="DataFormats\Database\" />
<Folder Include="DataFormats\Database\UTF\" />
<Folder Include="DataFormats\FileSystem\CPK\Internal\" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Libraries\UniversalEditor.Core\UniversalEditor.Core.csproj">