From 3de240486cdcea4f0836d57e4713dd6863da4e89 Mon Sep 17 00:00:00 2001 From: alcexhim Date: Fri, 24 Oct 2014 17:16:21 -0400 Subject: [PATCH 1/8] Attempt to implement writing EBML compressed integers, needs heavy testing... --- .../DataFormats/Markup/EBML/EBMLDataFormat.cs | 111 +++++++++++++++++- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/CSharp/Plugins/UniversalEditor.Essential/DataFormats/Markup/EBML/EBMLDataFormat.cs b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/Markup/EBML/EBMLDataFormat.cs index ef48db7c..e071cd20 100644 --- a/CSharp/Plugins/UniversalEditor.Essential/DataFormats/Markup/EBML/EBMLDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/Markup/EBML/EBMLDataFormat.cs @@ -40,8 +40,8 @@ namespace UniversalEditor.DataFormats.Markup.EBML private MarkupElement ReadEBMLElement(Reader reader) { - long elementID = ReadEBMLElementID(reader); - long dataSize = ReadEBMLElementID(reader); + long elementID = ReadEBMLCompressedInteger(reader); + long dataSize = ReadEBMLCompressedInteger(reader); byte[] data = reader.ReadBytes(dataSize); MarkupTagElement tag = new MarkupTagElement(); @@ -67,7 +67,7 @@ namespace UniversalEditor.DataFormats.Markup.EBML return tag; } - private long ReadEBMLElementID(Reader reader) + private long ReadEBMLCompressedInteger(Reader reader) { byte[] buffer = reader.ReadBytes(8); reader.Seek(-8, SeekOrigin.Current); @@ -191,6 +191,111 @@ namespace UniversalEditor.DataFormats.Markup.EBML // which will be sufficient for the time being. throw new NotImplementedException("Unknown Element Size coding: 0x" + buffer[0].ToString("X")); } + private void WriteEBMLCompressedInteger(Writer writer, long value) + { + if (value <= 0x7F) + { + writer.WriteByte((byte)(value & 0x80)); + return; + } + else if (value <= 0x7FFF) + { + // two bytes + byte[] _buffer = new byte[2]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[1] | 0x40); + _buffer[1] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFF) + { + // three bytes + byte[] _buffer = new byte[3]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[2] | 0x20); + _buffer[1] = buffer[1]; + _buffer[2] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFFFF) + { + // four bytes + byte[] _buffer = new byte[4]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[3] | 0x10); + _buffer[1] = buffer[2]; + _buffer[2] = buffer[1]; + _buffer[3] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFFFFFF) + { + // five bytes + byte[] _buffer = new byte[5]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[4] | 0x08); + _buffer[1] = buffer[3]; + _buffer[2] = buffer[2]; + _buffer[3] = buffer[1]; + _buffer[4] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFFFFFFFF) + { + // six bytes + byte[] _buffer = new byte[6]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[5] | 0x04); + _buffer[1] = buffer[4]; + _buffer[2] = buffer[3]; + _buffer[3] = buffer[2]; + _buffer[4] = buffer[1]; + _buffer[5] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFFFFFFFFFF) + { + // seven bytes + byte[] _buffer = new byte[7]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[6] | 0x02); + _buffer[1] = buffer[5]; + _buffer[2] = buffer[4]; + _buffer[3] = buffer[3]; + _buffer[4] = buffer[2]; + _buffer[5] = buffer[1]; + _buffer[6] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + else if (value <= 0x7FFFFFFFFFFFFFFF) + { + // eight bytes + byte[] _buffer = new byte[8]; + byte[] buffer = BitConverter.GetBytes(value); + _buffer[0] = (byte)(buffer[7] | 0x01); + _buffer[1] = buffer[6]; + _buffer[2] = buffer[5]; + _buffer[3] = buffer[4]; + _buffer[4] = buffer[3]; + _buffer[5] = buffer[2]; + _buffer[6] = buffer[1]; + _buffer[7] = buffer[0]; + writer.WriteBytes(_buffer); + return; + } + + // Since modern computers do not easily deal with data coded in sizes greater than 64 bits, + // any larger Element Sizes are left undefined at the moment. Currently, the Element Size + // coding allows for an Element to grow to 72000 To, i.e. 7x10^16 octets or 72000 terabytes, + // which will be sufficient for the time being. + throw new NotImplementedException("Value cannot be represented as an EBML compressed integer: " + value.ToString()); + } protected override void SaveInternal(ObjectModel objectModel) { From fba13565405d4be326d2efca5eb16e652c10d5cf Mon Sep 17 00:00:00 2001 From: alcexhim Date: Fri, 24 Oct 2014 21:46:17 -0400 Subject: [PATCH 2/8] Now with short-sector support, untested, LOOKS BUGGY, definitely needs to be reorganized --- .../CompoundDocumentDataFormat.cs | 266 ++++++++++-------- 1 file changed, 155 insertions(+), 111 deletions(-) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs index 40a93be8..bbd4a41a 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs @@ -78,8 +78,31 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument /// private int mvarMasterSectorAllocationTableSize = 0; - private void ReadHeader(Reader reader) + private int mvarShortSectorFirstSectorID = 0; + + private int GetSectorPositionFromSectorID(int sectorID) { + if (sectorID < 0) return 0; + return (int)(512 + (sectorID * mvarSectorSize)); + } + private int GetShortSectorPositionFromSectorID(int sectorID) + { + if (sectorID < 0) return 0; + return (int)(sectorID * mvarShortSectorSize); + } + + private byte[] mvarShortSectorContainerStreamData = null; + + protected override void LoadInternal(ref ObjectModel objectModel) + { + FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel); + if (fsom == null) throw new ObjectModelNotSupportedException(); + + Reader reader = base.Accessor.Reader; + + // The header is always located at the beginning of the file, and its size is + // exactly 512 bytes. This implies that the first sector (0) always starts at + // file offset 512. byte[] validSignature = new byte[] { 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1 }; byte[] signature = reader.ReadBytes(8); if (!signature.Match(validSignature)) @@ -87,7 +110,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument throw new InvalidDataFormatException("File does not begin with { 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1 }"); } mvarUniqueIdentifier = reader.ReadGuid(); - + ushort MinorVersion = reader.ReadUInt16(); ushort MajorVersion = reader.ReadUInt16(); mvarFormatVersion = new Version(MajorVersion, MinorVersion); @@ -129,7 +152,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument #region Read Master Sector Allocation Table // First part of the master sector allocation table, containing 109 SecIDs int[] masterSectorAllocationTable = reader.ReadInt32Array(109); - + // TODO: test this! when MSAT contains more than 109 SecIDs int countForMSAT = (int)((double)mvarSectorSize / 4); int nextSectorForMSAT = mvarMasterSectorAllocationTableFirstSectorID; @@ -139,7 +162,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument Array.Resize(ref masterSectorAllocationTable, masterSectorAllocationTable.Length + countForMSAT); int[] masterSectorAllocationTablePart = reader.ReadInt32Array(countForMSAT); Array.Copy(masterSectorAllocationTablePart, 0, masterSectorAllocationTable, nextPositionForMSAT, masterSectorAllocationTablePart.Length); - + nextSectorForMSAT = masterSectorAllocationTablePart[masterSectorAllocationTablePart.Length - 1]; } #endregion @@ -165,150 +188,171 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument Array.Copy(sectorData, 0, data, (i * mvarSectorSize), mvarSectorSize); } #endregion + #region Read Short Sector Allocation Table + List shortSectorAllocationTable = new List(); + List shortSectorAllocationTableSectors = new List(); + if (mvarShortSectorAllocationTableFirstSectorID >= 0) + { + int sector = mvarShortSectorAllocationTableFirstSectorID; + while (sector >= 0) + { + shortSectorAllocationTableSectors.Add(sector); + sector = sectorAllocationTable[sector]; + } + } + + byte[] shortSectorAllocationTableData = new byte[mvarSectorSize * shortSectorAllocationTableSectors.Count]; + for (int i = 0; i < shortSectorAllocationTableSectors.Count; i++) + { + pos = GetSectorPositionFromSectorID(shortSectorAllocationTableSectors[i]); + reader.Accessor.Seek(pos, SeekOrigin.Begin); + byte[] sectorData = reader.ReadBytes(mvarSectorSize); + Array.Copy(sectorData, 0, shortSectorAllocationTableData, (i * mvarSectorSize), mvarSectorSize); + } + + Accessors.MemoryAccessor ma1 = new Accessors.MemoryAccessor(shortSectorAllocationTableData); + Reader shortSectorAllocationTableReader = new Reader(ma1); + while (!shortSectorAllocationTableReader.EndOfStream) + { + int sectorID = shortSectorAllocationTableReader.ReadInt32(); + shortSectorAllocationTable.Add(sectorID); + } + #endregion #region Read Sector Directory Entries Accessors.MemoryAccessor ma = new Accessors.MemoryAccessor(data); - reader = new Reader(ma); - - List files = new List(); + + Reader sectorReader = new Reader(ma); while (!reader.EndOfStream) { // The first directory entry always represents the root storage entry - string storageName = reader.ReadFixedLengthString(64, IO.Encoding.UTF16LittleEndian).TrimNull(); - ushort storageNameLength = reader.ReadUInt16(); + string storageName = sectorReader.ReadFixedLengthString(64, IO.Encoding.UTF16LittleEndian).TrimNull(); + if (String.IsNullOrEmpty(storageName)) break; + + ushort storageNameLength = sectorReader.ReadUInt16(); storageNameLength /= 2; if (storageNameLength > 0) storageNameLength -= 1; if (storageName.Length != storageNameLength) throw new InvalidDataFormatException("Sanity check: storage name length is not actual length of storage name"); - byte storageType = reader.ReadByte(); - byte storageNodeColor = reader.ReadByte(); + byte storageType = sectorReader.ReadByte(); + byte storageNodeColor = sectorReader.ReadByte(); - int leftChildNodeDirectoryID = reader.ReadInt32(); - int rightChildNodeDirectoryID = reader.ReadInt32(); + int leftChildNodeDirectoryID = sectorReader.ReadInt32(); + int rightChildNodeDirectoryID = sectorReader.ReadInt32(); // directory ID of the root node entry of the red-black tree of all members of the root storage - int rootNodeEntryDirectoryID = reader.ReadInt32(); + int rootNodeEntryDirectoryID = sectorReader.ReadInt32(); - Guid uniqueIdentifier = reader.ReadGuid(); - uint flags = reader.ReadUInt32(); - long creationTimestamp = reader.ReadInt64(); - long lastModificationTimestamp = reader.ReadInt64(); + Guid uniqueIdentifier = sectorReader.ReadGuid(); + uint flags = sectorReader.ReadUInt32(); + long creationTimestamp = sectorReader.ReadInt64(); + long lastModificationTimestamp = sectorReader.ReadInt64(); - int firstSectorOfStream = reader.ReadInt32(); - int streamOffset = GetSectorPositionFromSectorID(firstSectorOfStream); - int streamLength = reader.ReadInt32(); - int unused3 = reader.ReadInt32(); - - if (streamLength < mvarMinimumStandardStreamSize) + int firstSectorOfStream = sectorReader.ReadInt32(); + if (storageType == 0x05) { - // stored as a short-sector container stream - streamOffset = (int)(512 + (firstSectorOfStream * mvarShortSectorSize)); + // this is the root storage entry + mvarShortSectorFirstSectorID = firstSectorOfStream; + + #region Read Short Stream Container Stream + List shortStreamContainerStreamSectors = new List(); + { + int shortSectorDataSector = mvarShortSectorFirstSectorID; + while (shortSectorDataSector >= 0) + { + shortStreamContainerStreamSectors.Add(shortSectorDataSector); + shortSectorDataSector = sectorAllocationTable[shortSectorDataSector]; + } + } + byte[] shortStreamContainerStreamData = new byte[shortStreamContainerStreamSectors.Count * mvarSectorSize]; + int i = 0; + foreach (int sector in shortStreamContainerStreamSectors) + { + int wpos = GetSectorPositionFromSectorID(sector); + reader.Seek(wpos, SeekOrigin.Begin); + byte[] sectorData = reader.ReadBytes(mvarSectorSize); + Array.Copy(sectorData, 0, shortStreamContainerStreamData, i, sectorData.Length); + i += sectorData.Length; + } + mvarShortSectorContainerStreamData = shortStreamContainerStreamData; + #endregion } - File file = new File(); - file.Name = storageName; - file.Properties.Add("reader", reader); - file.Properties.Add("offset", streamOffset); - file.Properties.Add("length", streamLength); - file.DataRequest += file_DataRequest; - files.Add(file); - } - #endregion - } + int streamLength = sectorReader.ReadInt32(); + int unused3 = sectorReader.ReadInt32(); - private int GetSectorPositionFromSectorID(int sectorID) - { - if (sectorID < 0) return 0; - return (int)(512 + (sectorID * mvarSectorSize)); - } - - protected override void LoadInternal(ref ObjectModel objectModel) - { - FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel); - if (fsom == null) throw new ObjectModelNotSupportedException(); - - Reader reader = base.Accessor.Reader; - - // The header is always located at the beginning of the file, and its size is - // exactly 512 bytes. This implies that the first sector (0) always starts at - // file offset 512. - ReadHeader(reader); - - // TODO: read extra sectors if necessary - - int directoryEntryLength = 128; - int directoryEntryCount = (int)((mvarSectorAllocationTableSize * mvarSectorSize) / directoryEntryLength); - for (int i = 0; i < directoryEntryCount; i++) - { - // The first directory entry always represents the root storage entry - string storageName = reader.ReadFixedLengthString(64, IO.Encoding.UTF16LittleEndian).TrimNull(); - ushort storageNameLength = reader.ReadUInt16(); - storageNameLength /= 2; - storageNameLength -= 1; - if (storageName.Length != storageNameLength) throw new InvalidDataFormatException("Sanity check: storage name length is not actual length of storage name"); - - byte storageType = reader.ReadByte(); - byte storageNodeColor = reader.ReadByte(); - - int leftChildNodeDirectoryID = reader.ReadInt32(); - int rightChildNodeDirectoryID = reader.ReadInt32(); - // directory ID of the root node entry of the red-black tree of all members of the root storage - int rootNodeEntryDirectoryID = reader.ReadInt32(); - - Guid uniqueIdentifier = reader.ReadGuid(); - uint flags = reader.ReadUInt32(); - long creationTimestamp = reader.ReadInt64(); - long lastModificationTimestamp = reader.ReadInt64(); - - int firstSectorOfStream = reader.ReadInt32(); - int streamOffset = GetSectorPositionFromSectorID(firstSectorOfStream); - int streamLength = reader.ReadInt32(); - int unused3 = reader.ReadInt32(); - - /* - The directory entry of a stream contains the SecID of the first sector or - short-sector containing the stream data. All streams that are shorter than a - specific size given in the header are stored as a short-stream, thus inserted - into the short-stream container stream. In this case the SecID specifies the - first short-sector inside the short-stream container stream, and the - short-sector allocation table is used to build up the SecID chain of the - stream. - */ - - if (i == 0) + List sectors = new List(); + if (streamLength < mvarMinimumStandardStreamSize) { - // in the case of the Root Entry, the firstSectorOfStream is the SecID of - // the first sector and the streamLength is the size of the short-stream - // container stream - continue; + // use the short-sector allocation table + int sector = firstSectorOfStream; + while (sector >= 0) + { + sectors.Add(sector); + sector = shortSectorAllocationTable[sector]; + } } else { - - } - - if (streamLength < mvarMinimumStandardStreamSize) - { - // stored as a short-sector container stream - streamOffset = (int)(512 + (firstSectorOfStream * mvarShortSectorSize)); + // use the standard sector allocation table + int sector = firstSectorOfStream; + while (sector >= 0) + { + sectors.Add(sector); + sector = sectorAllocationTable[sector]; + } } File file = new File(); file.Name = storageName; file.Properties.Add("reader", reader); - file.Properties.Add("offset", streamOffset); + file.Properties.Add("sectors", sectors); file.Properties.Add("length", streamLength); file.DataRequest += file_DataRequest; fsom.Files.Add(file); } + #endregion } + private Reader shortSectorReader = null; + private void file_DataRequest(object sender, DataRequestEventArgs e) { File file = (sender as File); Reader reader = (Reader)file.Properties["reader"]; - int offset = (int)file.Properties["offset"]; + if (shortSectorReader == null) shortSectorReader = new Reader(new Accessors.MemoryAccessor(mvarShortSectorContainerStreamData)); + List sectors = (List)file.Properties["sectors"]; int length = (int)file.Properties["length"]; - reader.Accessor.Seek(offset, SeekOrigin.Begin); - e.Data = reader.ReadBytes(length); + + byte[] realdata = new byte[length]; + if (length < mvarMinimumStandardStreamSize) + { + // use the short-sector allocation table and short stream container stream + + byte[] data = new byte[sectors.Count * mvarShortSectorSize]; + int start = 0; + foreach (int sector in sectors) + { + int pos = GetShortSectorPositionFromSectorID(sector); + shortSectorReader.Accessor.Seek(pos, SeekOrigin.Begin); + byte[] sectorData = shortSectorReader.ReadBytes(mvarShortSectorSize); + Array.Copy(sectorData, 0, data, start, sectorData.Length); + start += (int)mvarShortSectorSize; + } + e.Data = data; + } + else + { + byte[] data = new byte[sectors.Count * mvarSectorSize]; + int start = 0; + foreach (int sector in sectors) + { + int pos = GetSectorPositionFromSectorID(sector); + reader.Accessor.Seek(pos, SeekOrigin.Begin); + byte[] sectorData = reader.ReadBytes(mvarSectorSize); + Array.Copy(sectorData, 0, data, start, sectorData.Length); + start += (int)mvarSectorSize; + } + e.Data = data; + } } protected override void SaveInternal(ObjectModel objectModel) From 257d61688202c5d5c629bb40beca7186bcf30be4 Mon Sep 17 00:00:00 2001 From: alcexhim Date: Fri, 24 Oct 2014 22:11:30 -0400 Subject: [PATCH 3/8] Trim the DataRequest so it only returns actual data, not padding (which could contain other sectors' data) --- .../CompoundDocument/CompoundDocumentDataFormat.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs index bbd4a41a..c647fb3e 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs @@ -326,7 +326,6 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument if (length < mvarMinimumStandardStreamSize) { // use the short-sector allocation table and short stream container stream - byte[] data = new byte[sectors.Count * mvarShortSectorSize]; int start = 0; foreach (int sector in sectors) @@ -337,7 +336,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument Array.Copy(sectorData, 0, data, start, sectorData.Length); start += (int)mvarShortSectorSize; } - e.Data = data; + Array.Copy(data, 0, realdata, 0, realdata.Length); } else { @@ -351,8 +350,9 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument Array.Copy(sectorData, 0, data, start, sectorData.Length); start += (int)mvarSectorSize; } - e.Data = data; + Array.Copy(data, 0, realdata, 0, realdata.Length); } + e.Data = realdata; } protected override void SaveInternal(ObjectModel objectModel) From a037dadaa228c4f567e469629cd3fd9ba4c198f3 Mon Sep 17 00:00:00 2001 From: alcexhim Date: Sat, 25 Oct 2014 09:50:59 -0400 Subject: [PATCH 4/8] Properly advance the next sector for MSAT --- .../Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs index c647fb3e..dfd5c64b 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs @@ -163,7 +163,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument int[] masterSectorAllocationTablePart = reader.ReadInt32Array(countForMSAT); Array.Copy(masterSectorAllocationTablePart, 0, masterSectorAllocationTable, nextPositionForMSAT, masterSectorAllocationTablePart.Length); - nextSectorForMSAT = masterSectorAllocationTablePart[masterSectorAllocationTablePart.Length - 1]; + nextSectorForMSAT = masterSectorAllocationTablePart[nextSectorForMSAT]; } #endregion #region Read Sector Allocation Table From d4ea17f320a5804f5aee971c1f1240d17daa8feb Mon Sep 17 00:00:00 2001 From: alcexhim Date: Sat, 25 Oct 2014 09:53:37 -0400 Subject: [PATCH 5/8] Properly advance the next sector for MSAT --- .../Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs index dfd5c64b..c647fb3e 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs @@ -163,7 +163,7 @@ namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument int[] masterSectorAllocationTablePart = reader.ReadInt32Array(countForMSAT); Array.Copy(masterSectorAllocationTablePart, 0, masterSectorAllocationTable, nextPositionForMSAT, masterSectorAllocationTablePart.Length); - nextSectorForMSAT = masterSectorAllocationTablePart[nextSectorForMSAT]; + nextSectorForMSAT = masterSectorAllocationTablePart[masterSectorAllocationTablePart.Length - 1]; } #endregion #region Read Sector Allocation Table From 926815e76e9f5ba8d98af3ed486aad17bcf5f3ec Mon Sep 17 00:00:00 2001 From: alcexhim Date: Sat, 25 Oct 2014 09:55:07 -0400 Subject: [PATCH 6/8] Optimize seeking with cast to System.IO.SeekOrigin --- .../Accessors/FileAccessor.cs | 23 ++----------------- .../UniversalEditor.Core/IO/SeekOrigin.cs | 6 ++--- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/CSharp/Libraries/UniversalEditor.Core/Accessors/FileAccessor.cs b/CSharp/Libraries/UniversalEditor.Core/Accessors/FileAccessor.cs index c3856648..e8c084df 100644 --- a/CSharp/Libraries/UniversalEditor.Core/Accessors/FileAccessor.cs +++ b/CSharp/Libraries/UniversalEditor.Core/Accessors/FileAccessor.cs @@ -19,28 +19,9 @@ namespace UniversalEditor.Accessors set { mvarFileStream.SetLength(value); } } - public override void Seek(long length, SeekOrigin position) + public override void Seek(long length, SeekOrigin origin) { - System.IO.SeekOrigin origin = System.IO.SeekOrigin.Begin; - switch (position) - { - case SeekOrigin.Begin: - { - origin = System.IO.SeekOrigin.Begin; - break; - } - case SeekOrigin.Current: - { - origin = System.IO.SeekOrigin.Current; - break; - } - case SeekOrigin.End: - { - origin = System.IO.SeekOrigin.End; - break; - } - } - mvarFileStream.Seek(length, origin); + mvarFileStream.Seek(length, (System.IO.SeekOrigin)origin); } internal override int ReadInternal(byte[] buffer, int offset, int count) diff --git a/CSharp/Libraries/UniversalEditor.Core/IO/SeekOrigin.cs b/CSharp/Libraries/UniversalEditor.Core/IO/SeekOrigin.cs index 21c7926d..7d514fed 100644 --- a/CSharp/Libraries/UniversalEditor.Core/IO/SeekOrigin.cs +++ b/CSharp/Libraries/UniversalEditor.Core/IO/SeekOrigin.cs @@ -7,8 +7,8 @@ namespace UniversalEditor.IO { public enum SeekOrigin { - Begin, - Current, - End + Begin = 0, + Current = 1, + End = 2 } } From 3727fa78874c86db3981f2ffc1f98c9f94c52ce0 Mon Sep 17 00:00:00 2001 From: alcexhim Date: Sat, 25 Oct 2014 10:29:27 -0400 Subject: [PATCH 7/8] Tabify --- .../Picture/TBODY/TBODYDataFormat.cs | 14 ++-- .../Model/Avalanche/VBUFDataFormat.cs | 66 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia/Picture/TBODY/TBODYDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia/Picture/TBODY/TBODYDataFormat.cs index f67d5ee4..550b6e42 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia/Picture/TBODY/TBODYDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia/Picture/TBODY/TBODYDataFormat.cs @@ -26,12 +26,12 @@ namespace UniversalEditor.DataFormats.Multimedia.Picture.TBODY protected override void LoadInternal(ref ObjectModel objectModel) { - // A TBODY file is just a DDS file without a header + // A TBODY file is just a DDS file without a header IO.Reader br = base.Accessor.Reader; byte[] data = br.ReadToEnd(); - // We have to add the DDS header - MemoryAccessor ma = new MemoryAccessor(); + // We have to add the DDS header + MemoryAccessor ma = new MemoryAccessor(); IO.Writer bw = new IO.Writer(ma); bw.WriteUInt32(DirectDrawSurfaceDataFormat.DDS_MAGIC); bw.WriteUInt32((uint)124); // data size @@ -71,10 +71,10 @@ namespace UniversalEditor.DataFormats.Multimedia.Picture.TBODY ma = new MemoryAccessor(data1); DirectDrawSurfaceDataFormat dds = new DirectDrawSurfaceDataFormat(); - Document doc = new Document(objectModel, new DirectDrawSurfaceDataFormat(), ma); - doc.InputAccessor.Open(); - doc.Load(); - doc.InputAccessor.Close(); + Document doc = new Document(objectModel, new DirectDrawSurfaceDataFormat(), ma); + doc.InputAccessor.Open(); + doc.Load(); + doc.InputAccessor.Close(); } protected override void SaveInternal(ObjectModel objectModel) diff --git a/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia3D/Model/Avalanche/VBUFDataFormat.cs b/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia3D/Model/Avalanche/VBUFDataFormat.cs index 64c1f4b0..c0ba755c 100644 --- a/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia3D/Model/Avalanche/VBUFDataFormat.cs +++ b/CSharp/Plugins/UniversalEditor.Plugins.Avalanche/DataFormats/Multimedia3D/Model/Avalanche/VBUFDataFormat.cs @@ -21,9 +21,9 @@ namespace UniversalEditor.DataFormats.Multimedia3D.Model.Avalanche } return _dfr; } - - protected override void LoadInternal(ref ObjectModel objectModel) - { + + protected override void LoadInternal(ref ObjectModel objectModel) + { ModelObjectModel model = (objectModel as ModelObjectModel); if (model == null) throw new ObjectModelNotSupportedException(); @@ -39,41 +39,41 @@ namespace UniversalEditor.DataFormats.Multimedia3D.Model.Avalanche //br.BaseStream.Seek(0, System.IO.SeekOrigin.Begin); ModelSurface surf = new ModelSurface(); - int count = (int)(brIBUF.Accessor.Length / 2); + int count = (int)(brIBUF.Accessor.Length / 2); for (int i = 0; i < count; i++) { float vx = br.ReadSingle(); float vz = br.ReadSingle(); float vy = br.ReadSingle(); - vx *= 100; - vy *= 100; - vz *= 100; - vz *= -1; + vx *= 100; + vy *= 100; + vz *= 100; + vz *= -1; float tu = br.ReadHalf(); - float tv = br.ReadHalf(); - tv = 1 - tv; + float tv = br.ReadHalf(); + tv = 1 - tv; - uint unknown1 = br.ReadUInt32(); - uint vertexColor = br.ReadUInt32(); + uint unknown1 = br.ReadUInt32(); + uint vertexColor = br.ReadUInt32(); ModelVertex vtx = new ModelVertex(vx, vy, vz, tu, tv); // vx, vz, vy in original script?? surf.Vertices.Add(vtx); } - /* + /* for (int x = 0; x < surf.Vertices.Count; x++) { float nx = br.ReadSingle(); float ny = br.ReadSingle(); float nz = br.ReadSingle(); - nz *= -1; + nz *= -1; float un = br.ReadSingle(); - - surf.Vertices[x].Normal = new PositionVector3(nx, ny, nz); - surf.Vertices[x].OriginalNormal = new PositionVector3(nx, ny, nz); + + surf.Vertices[x].Normal = new PositionVector3(nx, ny, nz); + surf.Vertices[x].OriginalNormal = new PositionVector3(nx, ny, nz); - // not sure why this needs to be here... - br.BaseStream.Seek(10, System.IO.SeekOrigin.Current); + // not sure why this needs to be here... + br.BaseStream.Seek(10, System.IO.SeekOrigin.Current); } while (!brIBUF.EndOfStream) { @@ -82,21 +82,21 @@ namespace UniversalEditor.DataFormats.Multimedia3D.Model.Avalanche ushort fc = (ushort)(brIBUF.ReadUInt16() + 1); surf.Triangles.Add(surf.Vertices[fa], surf.Vertices[fb], surf.Vertices[fc]); } - */ + */ - for (int i = 0; i < surf.Vertices.Count - 3; i += 3) - { - surf.Triangles.Add(surf.Vertices[i], surf.Vertices[i + 1], surf.Vertices[i + 2]); - } + for (int i = 0; i < surf.Vertices.Count - 3; i += 3) + { + surf.Triangles.Add(surf.Vertices[i], surf.Vertices[i + 1], surf.Vertices[i + 2]); + } - ModelMaterial matDefault = new ModelMaterial(); - matDefault.Name = "default"; - matDefault.EmissiveColor = Color.FromRGBA(255, 255, 255, 255); - foreach (ModelTriangle tri in surf.Triangles) - { - matDefault.Triangles.Add(tri); - } - model.Materials.Add(matDefault); + ModelMaterial matDefault = new ModelMaterial(); + matDefault.Name = "default"; + matDefault.EmissiveColor = Color.FromRGBA(255, 255, 255, 255); + foreach (ModelTriangle tri in surf.Triangles) + { + matDefault.Triangles.Add(tri); + } + model.Materials.Add(matDefault); model.Surfaces.Add(surf); /* @@ -116,7 +116,7 @@ namespace UniversalEditor.DataFormats.Multimedia3D.Model.Avalanche )))) else (Print "Aborted.") } */ - brIBUF.Close(); + brIBUF.Close(); } protected override void SaveInternal(ObjectModel objectModel) From 61db8164c725ccbb2d4d293855fcad001b21f0fc Mon Sep 17 00:00:00 2001 From: alcexhim Date: Sat, 25 Oct 2014 21:52:11 -0400 Subject: [PATCH 8/8] Begin to implement Abstract Syntax Notation (ASN) v1 and the related DER-encoded Security Certificate formats --- .../AbstractSyntax/DER/DERDataFormat.cs | 43 ++++++++++++++++ .../DER/DERCertificateDataFormat.cs | 50 +++++++++++++++++++ .../AbstractSyntaxObjectModel.cs | 20 ++++++++ .../SecurityCertificateObjectModel.cs | 28 +++++++++++ .../UniversalEditor.Essential.csproj | 4 ++ 5 files changed, 145 insertions(+) create mode 100644 CSharp/Plugins/UniversalEditor.Essential/DataFormats/AbstractSyntax/DER/DERDataFormat.cs create mode 100644 CSharp/Plugins/UniversalEditor.Essential/DataFormats/SecurityCertificate/DER/DERCertificateDataFormat.cs create mode 100644 CSharp/Plugins/UniversalEditor.Essential/ObjectModels/AbstractSyntax/AbstractSyntaxObjectModel.cs create mode 100644 CSharp/Plugins/UniversalEditor.Essential/ObjectModels/SecurityCertificate/SecurityCertificateObjectModel.cs diff --git a/CSharp/Plugins/UniversalEditor.Essential/DataFormats/AbstractSyntax/DER/DERDataFormat.cs b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/AbstractSyntax/DER/DERDataFormat.cs new file mode 100644 index 00000000..f59f910c --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/AbstractSyntax/DER/DERDataFormat.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using UniversalEditor.IO; +using UniversalEditor.ObjectModels.AbstractSyntax; + +namespace UniversalEditor.DataFormats.AbstractSyntax.DER +{ + public class DERDataFormat : DataFormat + { + private static DataFormatReference _dfr = null; + public override DataFormatReference MakeReference() + { + if (_dfr == null) + { + _dfr = base.MakeReference(); + _dfr.Capabilities.Add(typeof(AbstractSyntaxObjectModel), DataFormatCapabilities.All); + } + return _dfr; + } + + protected override void LoadInternal(ref ObjectModel objectModel) + { + AbstractSyntaxObjectModel asn = (objectModel as AbstractSyntaxObjectModel); + if (asn == null) throw new ObjectModelNotSupportedException(); + + Reader reader = base.Accessor.Reader; + while (!reader.EndOfStream) + { + byte identifier = reader.ReadByte(); + byte tagClass = (byte)identifier.GetBits(7, 2); + byte primitiveOrConstructed = (byte)identifier.GetBits(6, 1); + byte tagNumber = (byte)identifier.GetBits(0, 5); + } + } + + protected override void SaveInternal(ObjectModel objectModel) + { + throw new NotImplementedException(); + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Essential/DataFormats/SecurityCertificate/DER/DERCertificateDataFormat.cs b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/SecurityCertificate/DER/DERCertificateDataFormat.cs new file mode 100644 index 00000000..f1cd382f --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Essential/DataFormats/SecurityCertificate/DER/DERCertificateDataFormat.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +using UniversalEditor.ObjectModels.AbstractSyntax; +using UniversalEditor.DataFormats.AbstractSyntax.DER; + +using UniversalEditor.IO; +using UniversalEditor.ObjectModels.SecurityCertificate; + +namespace UniversalEditor.DataFormats.SecurityCertificate.DER +{ + public class DERCertificateDataFormat : DERDataFormat + { + private static DataFormatReference _dfr = null; + public override DataFormatReference MakeReference() + { + if (_dfr == null) + { + _dfr = new DataFormatReference(GetType()); + _dfr.Capabilities.Add(typeof(SecurityCertificateObjectModel), DataFormatCapabilities.All); + _dfr.Filters.Add("Security certificate (Binary-encoded DER)", new byte?[][] { new byte?[] { (byte)0x30, (byte)0x82 } }, new string[] { "*.cer", "*.der", "*.p7b" }); + _dfr.Filters.Add("Security certificate (Base64-encoded DER)", new byte?[][] { new byte?[] { (byte)'-', (byte)'-', (byte)'-', (byte)'-', (byte)'-', (byte)'B', (byte)'E', (byte)'G', (byte)'I', (byte)'N', (byte)' ', (byte)'C', (byte)'E', (byte)'R', (byte)'T', (byte)'I', (byte)'F', (byte)'I', (byte)'C', (byte)'A', (byte)'T', (byte)'E', (byte)'-', (byte)'-', (byte)'-', (byte)'-', (byte)'-', (byte)'\r', (byte)'\n' } }, new string[] { "*.cer", "*.der", "*.p7b" }); + } + return _dfr; + } + + protected override void BeforeLoadInternal(Stack objectModels) + { + base.BeforeLoadInternal(objectModels); + objectModels.Push(new AbstractSyntaxObjectModel()); + } + protected override void AfterLoadInternal(Stack objectModels) + { + base.AfterLoadInternal(objectModels); + AbstractSyntaxObjectModel asn = (objectModels.Pop() as AbstractSyntaxObjectModel); + SecurityCertificateObjectModel cer = (objectModels.Pop() as SecurityCertificateObjectModel); + } + + protected override void BeforeSaveInternal(Stack objectModels) + { + base.BeforeSaveInternal(objectModels); + SecurityCertificateObjectModel cer = (objectModels.Pop() as SecurityCertificateObjectModel); + AbstractSyntaxObjectModel asn = new AbstractSyntaxObjectModel(); + + objectModels.Push(asn); + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/AbstractSyntax/AbstractSyntaxObjectModel.cs b/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/AbstractSyntax/AbstractSyntaxObjectModel.cs new file mode 100644 index 00000000..2dd8eba0 --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/AbstractSyntax/AbstractSyntaxObjectModel.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace UniversalEditor.ObjectModels.AbstractSyntax +{ + public class AbstractSyntaxObjectModel : ObjectModel + { + public override void Clear() + { + throw new NotImplementedException(); + } + + public override void CopyTo(ObjectModel where) + { + throw new NotImplementedException(); + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/SecurityCertificate/SecurityCertificateObjectModel.cs b/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/SecurityCertificate/SecurityCertificateObjectModel.cs new file mode 100644 index 00000000..a9c4fc6f --- /dev/null +++ b/CSharp/Plugins/UniversalEditor.Essential/ObjectModels/SecurityCertificate/SecurityCertificateObjectModel.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace UniversalEditor.ObjectModels.SecurityCertificate +{ + public class SecurityCertificateObjectModel : ObjectModel + { + private static ObjectModelReference _omr = null; + public override ObjectModelReference MakeReference() + { + if (_omr == null) + { + _omr = base.MakeReference(); + _omr.Title = "Security certificate"; + _omr.Path = new string[] { "Security", "Certificate" }; + } + return _omr; + } + public override void Clear() + { + } + public override void CopyTo(ObjectModel where) + { + } + } +} diff --git a/CSharp/Plugins/UniversalEditor.Essential/UniversalEditor.Essential.csproj b/CSharp/Plugins/UniversalEditor.Essential/UniversalEditor.Essential.csproj index 3e6a2c1a..2ba6b648 100644 --- a/CSharp/Plugins/UniversalEditor.Essential/UniversalEditor.Essential.csproj +++ b/CSharp/Plugins/UniversalEditor.Essential/UniversalEditor.Essential.csproj @@ -44,6 +44,7 @@ + @@ -60,10 +61,12 @@ + + @@ -95,6 +98,7 @@ +