Begin to implement the Microsoft OLE Compound Document Format, with documentation

CSharp/Plugins/UniversalEditor.Plugins.Microsoft.VisualStudio/DataFormats/PropertyList/Microsoft/VisualStudio/SUODataFormat.cs

@@ -0,0 +1,51 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument;
+using UniversalEditor.ObjectModels.FileSystem;
+using UniversalEditor.ObjectModels.PropertyList;
+
+namespace UniversalEditor.DataFormats.PropertyList.Microsoft.VisualStudio
+{
+	public class SUODataFormat : CompoundDocumentDataFormat
+	{
+		private static DataFormatReference _dfr = null;
+		public override DataFormatReference MakeReference()
+		{
+			if (_dfr == null)
+			{
+				_dfr = new DataFormatReference(GetType());
+				_dfr.Capabilities.Add(typeof(PropertyListObjectModel), DataFormatCapabilities.All);
+				_dfr.Filters.Add("Visual Studio Solution Options", new string[] { "*.suo" });
+			}
+			return _dfr;
+		}
+
+		protected override void BeforeLoadInternal(Stack<ObjectModel> objectModels)
+		{
+			base.BeforeLoadInternal(objectModels);
+			objectModels.Push(new FileSystemObjectModel());
+		}
+		protected override void AfterLoadInternal(Stack<ObjectModel> objectModels)
+		{
+			base.AfterLoadInternal(objectModels);
+			FileSystemObjectModel fsom = (objectModels.Pop() as FileSystemObjectModel);
+			PropertyListObjectModel plom = (objectModels.Pop() as PropertyListObjectModel);
+
+			for (int i = 0; i < fsom.Files.Count; i++ )
+				fsom.Files[i].Save(@"C:\Temp\SUO\" + fsom.Files[i].Name);
+		}
+		protected override void BeforeSaveInternal(Stack<ObjectModel> objectModels)
+		{
+			base.BeforeSaveInternal(objectModels);
+
+			PropertyListObjectModel plom = (objectModels.Pop() as PropertyListObjectModel);
+			FileSystemObjectModel fsom = new FileSystemObjectModel();
+
+
+
+			objectModels.Push(fsom);
+		}
+	}
+}

CSharp/Plugins/UniversalEditor.Plugins.Microsoft.VisualStudio/UniversalEditor.Plugins.Microsoft.VisualStudio.csproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
@@ -36,6 +36,7 @@
     <Reference Include="System.Core" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="DataFormats\PropertyList\Microsoft\VisualStudio\SUODataFormat.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="DataFormats\Project\Microsoft\VisualStudio\VisualStudioProjectDataFormat.cs" />
     <Compile Include="DataFormats\Solution\Microsoft\VisualStudio\VisualStudioSolutionDataFormat.cs" />
@@ -53,6 +54,10 @@
       <Project>{30467E5C-05BC-4856-AADC-13906EF4CADD}</Project>
       <Name>UniversalEditor.Essential</Name>
     </ProjectReference>
+    <ProjectReference Include="..\UniversalEditor.Plugins.Microsoft\UniversalEditor.Plugins.Microsoft.csproj">
+      <Project>{4698bc3f-ec29-42eb-9aed-3d8f9983a108}</Project>
+      <Name>UniversalEditor.Plugins.Microsoft</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentDataFormat.cs

@@ -0,0 +1,308 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+using UniversalEditor.IO;
+using UniversalEditor.ObjectModels.FileSystem;
+
+namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument
+{
+	public class CompoundDocumentDataFormat : DataFormat
+	{
+		private static DataFormatReference _dfr = null;
+		public override DataFormatReference MakeReference()
+		{
+			if (_dfr == null)
+			{
+				_dfr = base.MakeReference();
+				_dfr.Capabilities.Add(typeof(FileSystemObjectModel), DataFormatCapabilities.All);
+				_dfr.ExportOptions.Add(new CustomOptionNumber("SectorSize", "&Sector size (in bytes):", 512, 128));
+				_dfr.ExportOptions.Add(new CustomOptionNumber("ShortSectorSize", "S&hort sector size (in bytes):", 64));
+				_dfr.ExportOptions.Add(new CustomOptionNumber("MinimumStandardStreamSize", "&Minimum standard stream size (in bytes):", 4096, 4096));
+				_dfr.Sources.Add("http://www.openoffice.org/sc/compdocfileformat.pdf");
+			}
+			return _dfr;
+		}
+
+		private Guid mvarUniqueIdentifier = Guid.Empty;
+		public Guid UniqueIdentifier { get { return mvarUniqueIdentifier; } set { mvarUniqueIdentifier = value; } }
+
+		private Version mvarFormatVersion = new Version(3, 62);
+		public Version FormatVersion { get { return mvarFormatVersion; } set { mvarFormatVersion = value; } }
+
+		private Endianness mvarEndianness = Endianness.LittleEndian;
+		public Endianness Endianness { get { return mvarEndianness; } set { mvarEndianness = value; } }
+
+		private uint mvarSectorSize = 512;
+		public uint SectorSize { get { return mvarSectorSize; } set { mvarSectorSize = value; } }
+
+		private uint mvarShortSectorSize = 64;
+		public uint ShortSectorSize { get { return mvarShortSectorSize; } set { mvarShortSectorSize = value; } }
+
+		/// <summary>
+		/// Total number of sectors used for the sector allocation table
+		/// </summary>
+		private uint mvarSectorAllocationTableSize = 0;
+		/// <summary>
+		/// Sector ID of the first sector of the directory stream
+		/// </summary>
+		private uint mvarDirectoryStreamFirstSectorID = 0;
+
+		private uint mvarMinimumStandardStreamSize = 4096;
+		/// <summary>
+		/// Minimum size of a standard stream (in bytes). Minimum allowed and most-used size is
+		/// 4096 bytes. Streams with an actual size smaller than (and not equal to) this value
+		/// are stored as short-streams.
+		/// </summary>
+		public uint MinimumStandardStreamSize { get { return mvarMinimumStandardStreamSize; } set { mvarMinimumStandardStreamSize = value; } }
+
+		/// <summary>
+		/// Sector ID of the first sector of the short-sector allocation table (or
+		/// <see cref="CompoundDocumentKnownSectorID.EndOfChain" /> if not extant).
+		/// </summary>
+		private int mvarShortSectorAllocationTableFirstSectorID = 0;
+		/// <summary>
+		/// Total number of sectors used for the short-sector allocation table.
+		/// </summary>
+		private int mvarShortSectorAllocationTableSize = 0;
+		/// <summary>
+		/// Sector ID of the first sector of the master sector allocation table (or
+		/// <see cref="CompoundDocumentKnownSectorID.EndOfChain" /> if no additional sectors
+		/// used).
+		/// </summary>
+		private int mvarMasterSectorAllocationTableFirstSectorID = 0;
+		/// <summary>
+		/// Total number of sectors used for the master sector allocation table.
+		/// </summary>
+		private int mvarMasterSectorAllocationTableSize = 0;
+
+		private void ReadHeader(Reader reader)
+		{
+			byte[] validSignature = new byte[] { 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1 };
+			byte[] signature = reader.ReadBytes(8);
+			if (!signature.Match(validSignature))
+			{
+				throw new InvalidDataFormatException("File does not begin with { 0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1 }");
+			}
+			mvarUniqueIdentifier = reader.ReadGuid();
+			
+			ushort MinorVersion = reader.ReadUInt16();
+			ushort MajorVersion = reader.ReadUInt16();
+			mvarFormatVersion = new Version(MajorVersion, MinorVersion);
+
+			byte[] ByteOrderIdentifier = reader.ReadBytes(2);
+			if (ByteOrderIdentifier[0] == 0xFE && ByteOrderIdentifier[1] == 0xFF)
+			{
+				mvarEndianness = Endianness.LittleEndian;
+			}
+			else if (ByteOrderIdentifier[0] == 0xFF && ByteOrderIdentifier[1] == 0xFE)
+			{
+				mvarEndianness = Endianness.BigEndian;
+			}
+			else
+			{
+				throw new InvalidDataFormatException("Invalid value for byte order (" + ByteOrderIdentifier[0].ToString("X").PadLeft(2, '0') + ", " + ByteOrderIdentifier[1].ToString("X").PadLeft(2, '0') + ")");
+			}
+
+			ushort SectorSize = reader.ReadUInt16();
+			mvarSectorSize = (uint)(Math.Pow(2, SectorSize));
+
+			ushort ShortSectorSize = reader.ReadUInt16();
+			mvarShortSectorSize = (uint)(Math.Pow(2, ShortSectorSize));
+
+			if (ShortSectorSize > SectorSize) throw new InvalidDataFormatException("Short sector size (" + ShortSectorSize.ToString() + ") exceeds sector size (" + SectorSize.ToString() + ")");
+
+			byte[] unused1 = reader.ReadBytes(10);
+
+			mvarSectorAllocationTableSize = reader.ReadUInt32();
+			mvarDirectoryStreamFirstSectorID = reader.ReadUInt32();
+			uint unused2 = reader.ReadUInt32();
+			mvarMinimumStandardStreamSize = reader.ReadUInt32();
+
+			mvarShortSectorAllocationTableFirstSectorID = reader.ReadInt32();
+			mvarShortSectorAllocationTableSize = reader.ReadInt32();
+			mvarMasterSectorAllocationTableFirstSectorID = reader.ReadInt32();
+			mvarMasterSectorAllocationTableSize = reader.ReadInt32();
+
+			// First part of the master sector allocation table, containing 109 SecIDs
+			int[] masterSectorAllocationTable = reader.ReadInt32Array(109);
+			if (mvarMasterSectorAllocationTableFirstSectorID != (int)CompoundDocumentKnownSectorID.EndOfChain)
+			{
+				int nextSectorForMSAT = masterSectorAllocationTable[masterSectorAllocationTable.Length - 1];
+			}
+
+			int pos = GetSectorPositionFromSectorID(masterSectorAllocationTable[0]);
+			reader.Accessor.Seek(pos, SeekOrigin.Begin);
+			int[] sectorAllocationTable = reader.ReadInt32Array((int)(mvarSectorSize / 4));
+
+			List<int> directorySectors = new List<int>();
+			int currentSector = (int)mvarDirectoryStreamFirstSectorID;
+			while (currentSector != -2)
+			{
+				directorySectors.Add(currentSector);
+				currentSector = sectorAllocationTable[currentSector];
+			}
+
+			byte[] data = new byte[mvarSectorSize * directorySectors.Count];
+			for (int i = 0; i < directorySectors.Count; i++)
+			{
+				pos = GetSectorPositionFromSectorID(directorySectors[i]);
+				reader.Accessor.Seek(pos, SeekOrigin.Begin);
+				byte[] sectorData = reader.ReadBytes(mvarSectorSize);
+				Array.Copy(sectorData, 0, data, (i * mvarSectorSize), mvarSectorSize);
+			}
+
+			Accessors.MemoryAccessor ma = new Accessors.MemoryAccessor(data);
+			reader = new Reader(ma);
+
+			List<File> files = new List<File>();
+			while (!reader.EndOfStream)
+			{
+				// The first directory entry always represents the root storage entry
+				string storageName = reader.ReadFixedLengthString(64, IO.Encoding.UTF16LittleEndian).TrimNull();
+				ushort storageNameLength = reader.ReadUInt16();
+				storageNameLength /= 2;
+				if (storageNameLength > 0) storageNameLength -= 1;
+				if (storageName.Length != storageNameLength) throw new InvalidDataFormatException("Sanity check: storage name length is not actual length of storage name");
+
+				byte storageType = reader.ReadByte();
+				byte storageNodeColor = reader.ReadByte();
+
+				int leftChildNodeDirectoryID = reader.ReadInt32();
+				int rightChildNodeDirectoryID = reader.ReadInt32();
+				// directory ID of the root node entry of the red-black tree of all members of the root storage
+				int rootNodeEntryDirectoryID = reader.ReadInt32();
+
+				Guid uniqueIdentifier = reader.ReadGuid();
+				uint flags = reader.ReadUInt32();
+				long creationTimestamp = reader.ReadInt64();
+				long lastModificationTimestamp = reader.ReadInt64();
+
+				int firstSectorOfStream = reader.ReadInt32();
+				int streamOffset = GetSectorPositionFromSectorID(firstSectorOfStream);
+				int streamLength = reader.ReadInt32();
+				int unused3 = reader.ReadInt32();
+
+				if (streamLength < mvarMinimumStandardStreamSize)
+				{
+					// stored as a short-sector container stream
+					streamOffset = (int)(512 + (firstSectorOfStream * mvarShortSectorSize));
+				}
+
+				File file = new File();
+				file.Name = storageName;
+				file.Properties.Add("reader", reader);
+				file.Properties.Add("offset", streamOffset);
+				file.Properties.Add("length", streamLength);
+				file.DataRequest += file_DataRequest;
+				files.Add(file);
+			}
+		}
+
+		private int GetSectorPositionFromSectorID(int sectorID)
+		{
+			if (sectorID < 0) return 0;
+			return (int)(512 + (sectorID * mvarSectorSize));
+		}
+
+		protected override void LoadInternal(ref ObjectModel objectModel)
+		{
+			FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel);
+			if (fsom == null) throw new ObjectModelNotSupportedException();
+
+			Reader reader = base.Accessor.Reader;
+
+			// The header is always located at the beginning of the file, and its size is
+			// exactly 512 bytes. This implies that the first sector (0) always starts at
+			// file offset 512.
+			ReadHeader(reader);
+
+			// TODO: read extra sectors if necessary
+
+			int directoryEntryLength = 128;
+			int directoryEntryCount = (int)((mvarSectorAllocationTableSize * mvarSectorSize) / directoryEntryLength);
+			for (int i = 0; i < directoryEntryCount; i++)
+			{
+				// The first directory entry always represents the root storage entry
+				string storageName = reader.ReadFixedLengthString(64, IO.Encoding.UTF16LittleEndian).TrimNull();
+				ushort storageNameLength = reader.ReadUInt16();
+				storageNameLength /= 2;
+				storageNameLength -= 1;
+				if (storageName.Length != storageNameLength) throw new InvalidDataFormatException("Sanity check: storage name length is not actual length of storage name");
+
+				byte storageType = reader.ReadByte();
+				byte storageNodeColor = reader.ReadByte();
+
+				int leftChildNodeDirectoryID = reader.ReadInt32();
+				int rightChildNodeDirectoryID = reader.ReadInt32();
+				// directory ID of the root node entry of the red-black tree of all members of the root storage
+				int rootNodeEntryDirectoryID = reader.ReadInt32();
+
+				Guid uniqueIdentifier = reader.ReadGuid();
+				uint flags = reader.ReadUInt32();
+				long creationTimestamp = reader.ReadInt64();
+				long lastModificationTimestamp = reader.ReadInt64();
+
+				int firstSectorOfStream = reader.ReadInt32();
+				int streamOffset = GetSectorPositionFromSectorID(firstSectorOfStream);
+				int streamLength = reader.ReadInt32();
+				int unused3 = reader.ReadInt32();
+
+				/*
+				The directory entry of a stream contains the SecID of the first sector or
+				short-sector containing the stream data. All streams that are shorter than a
+				specific size given in the header are stored as a short-stream, thus inserted
+				into the short-stream container stream. In this case the SecID specifies the
+				first short-sector inside the short-stream container stream, and the
+				short-sector allocation table is used to build up the SecID chain of the
+				stream.
+				*/
+
+				if (i == 0)
+				{
+					// in the case of the Root Entry, the firstSectorOfStream is the SecID of
+					// the first sector and the streamLength is the size of the short-stream
+					// container stream
+					continue;
+				}
+				else
+				{
+
+				}
+
+				if (streamLength < mvarMinimumStandardStreamSize)
+				{
+					// stored as a short-sector container stream
+					streamOffset = (int)(512 + (firstSectorOfStream * mvarShortSectorSize));
+				}
+
+				File file = new File();
+				file.Name = storageName;
+				file.Properties.Add("reader", reader);
+				file.Properties.Add("offset", streamOffset);
+				file.Properties.Add("length", streamLength);
+				file.DataRequest += file_DataRequest;
+				fsom.Files.Add(file);
+			}
+		}
+
+		private void file_DataRequest(object sender, DataRequestEventArgs e)
+		{
+			File file = (sender as File);
+			Reader reader = (Reader)file.Properties["reader"];
+			int offset = (int)file.Properties["offset"];
+			int length = (int)file.Properties["length"];
+			reader.Accessor.Seek(offset, SeekOrigin.Begin);
+			e.Data = reader.ReadBytes(length);
+		}
+
+		protected override void SaveInternal(ObjectModel objectModel)
+		{
+			FileSystemObjectModel fsom = (objectModel as FileSystemObjectModel);
+			if (fsom == null) throw new ObjectModelNotSupportedException();
+
+			Writer writer = base.Accessor.Writer;
+		}
+	}
+}

CSharp/Plugins/UniversalEditor.Plugins.Microsoft/DataFormats/FileSystem/Microsoft/CompoundDocument/CompoundDocumentKnownSectorID.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace UniversalEditor.DataFormats.FileSystem.Microsoft.CompoundDocument
+{
+	public enum CompoundDocumentKnownSectorID
+	{
+		/// <summary>
+		/// Free sector; may exist in the file, but is not part of any stream.
+		/// </summary>
+		Free = -1,
+		/// <summary>
+		/// Trailing section ID in a section ID chain.
+		/// </summary>
+		EndOfChain = -2,
+		/// <summary>
+		/// Sector is used by the Sector Allocation Table (SAT)
+		/// </summary>
+		SectorAllocationTable = -3,
+		/// <summary>
+		/// Sector is used by the Master Sector Allocation Table (MSAT)
+		/// </summary>
+		MasterSectionAllocationTable = -4
+	}
+}

CSharp/Plugins/UniversalEditor.Plugins.Microsoft/UniversalEditor.Plugins.Microsoft.csproj

@@ -37,6 +37,8 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="DataFormats\DataLink\UniversalDataLink\UDLDataFormat.cs" />
+    <Compile Include="DataFormats\FileSystem\Microsoft\CompoundDocument\CompoundDocumentDataFormat.cs" />
+    <Compile Include="DataFormats\FileSystem\Microsoft\CompoundDocument\CompoundDocumentKnownSectorID.cs" />
     <Compile Include="DataFormats\FileSystem\Microsoft\MSCompressed\MSCompressedCompressionMethod.cs" />
     <Compile Include="DataFormats\FileSystem\Microsoft\MSCompressed\MSCompressedDataFormat.cs" />
     <Compile Include="DataFormats\FileSystem\Microsoft\MSCompressed\MSCompressedKWAJCompressionMethod.cs" />
@@ -64,7 +66,9 @@
       <Name>UniversalEditor.Plugins.FileSystem</Name>
     </ProjectReference>
   </ItemGroup>
-  <ItemGroup />
+  <ItemGroup>
+    <Folder Include="ObjectModels\Text\Formatted\XPS\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
@@ -73,13 +77,5 @@
   <Target Name="AfterBuild">
   </Target>
   -->
-  <ItemGroup>
-    <Folder Include="DataFormats\Text\" />
-    <Folder Include="DataFormats\Text\Formatted\" />
-    <Folder Include="DataFormats\Text\Formatted\XPS\" />
-    <Folder Include="ObjectModels\" />
-    <Folder Include="ObjectModels\Text\" />
-    <Folder Include="ObjectModels\Text\Formatted\" />
-    <Folder Include="ObjectModels\Text\Formatted\XPS\" />
-  </ItemGroup>
+  <ItemGroup />
 </Project>