From 83719504cfc39b546960af9658fefc3cdd1955bf Mon Sep 17 00:00:00 2001 From: myocytebd Date: Tue, 25 Oct 2022 04:05:59 +0800 Subject: [PATCH] Enable threading PoC --- AssetTools.NET/AssetsTools.NET.csproj | 8 +- AssetTools.NET/Extra/AssetHelper.cs | 4 +- .../Extra/AssetsManager/AssetsFileInstance.cs | 15 +- .../Extra/AssetsManager/AssetsManager.cs | 9 +- AssetTools.NET/Extra/BundleHelper.cs | 2 +- AssetTools.NET/Extra/Net35Polyfill.cs | 16 ++ .../AssetTypeClass/AssetTypeInstance.cs | 4 +- .../AssetTypeClass/AssetTypeTemplateField.cs | 4 +- .../AssetsFileFormat/AssetFileInfo.cs | 2 +- .../AssetsFileFormat/AssetFileList.cs | 2 +- .../Standard/AssetsFileFormat/AssetsFile.cs | 40 +-- .../AssetsFileFormat/AssetsFileDependency.cs | 4 +- .../AssetsFileDependencyList.cs | 2 +- .../AssetsFileFormat/AssetsFileHeader.cs | 2 +- .../AssetsFileReaderMMapImpl.cs | 148 ++++++++++ .../AssetsFileReaderStreamImpl.cs | 115 ++++++++ .../AssetsFileStatefulReader.cs | 266 ++++++++++++++++++ .../AssetsFileFormat/IAssetsFileReaderImpl.cs | 91 ++++++ .../Standard/AssetsFileFormat/PreloadList.cs | 2 +- .../Standard/AssetsFileFormat/TypeField_0D.cs | 2 +- .../Standard/AssetsFileFormat/TypeTree.cs | 2 +- .../Standard/AssetsFileFormat/Type_0D.cs | 2 +- .../AssetsFileTable/AssetFileInfoEx.cs | 2 +- .../AssetsFileTable/AssetsFileTable.cs | 6 +- .../BundleReplacerFromAssets.cs | 2 +- .../Standard/TextureFileFormat/TextureFile.cs | 4 +- 26 files changed, 691 insertions(+), 65 deletions(-) create mode 100644 AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderMMapImpl.cs create mode 100644 AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderStreamImpl.cs create mode 100644 AssetTools.NET/Standard/AssetsFileFormat/AssetsFileStatefulReader.cs create mode 100644 AssetTools.NET/Standard/AssetsFileFormat/IAssetsFileReaderImpl.cs diff --git a/AssetTools.NET/AssetsTools.NET.csproj b/AssetTools.NET/AssetsTools.NET.csproj index 39b02db..69b6163 100644 --- a/AssetTools.NET/AssetsTools.NET.csproj +++ b/AssetTools.NET/AssetsTools.NET.csproj @@ -1,10 +1,11 @@  - netstandard2.0;net35;net40 + netstandard2.0;net35;net40;net462 Library false AnyCPU 8 + true bin\x64\Debug\ @@ -18,10 +19,7 @@ - - - - + diff --git a/AssetTools.NET/Extra/AssetHelper.cs b/AssetTools.NET/Extra/AssetHelper.cs index d654cdd..08840e1 100644 --- a/AssetTools.NET/Extra/AssetHelper.cs +++ b/AssetTools.NET/Extra/AssetHelper.cs @@ -98,7 +98,7 @@ public static ushort GetScriptIndex(AssetsFile file, AssetFileInfoEx info) public static string GetAssetNameFast(AssetsFile file, ClassDatabaseFile cldb, AssetFileInfoEx info) { ClassDatabaseType type = FindAssetClassByID(cldb, info.curFileType); - AssetsFileReader reader = file.reader; + var reader = file.reader; if (file.typeTree.hasTypeTree) { @@ -168,7 +168,7 @@ public static string GetAssetNameFast(AssetsFile file, ClassDatabaseFile cldb, A //no classdatabase but may not work public static string GetAssetNameFastNaive(AssetsFile file, AssetFileInfoEx info) { - AssetsFileReader reader = file.reader; + var reader = file.reader; if (AssetsFileExtra.HasName(info.curFileType)) { diff --git a/AssetTools.NET/Extra/AssetsManager/AssetsFileInstance.cs b/AssetTools.NET/Extra/AssetsManager/AssetsFileInstance.cs index 65626bb..a972a3a 100644 --- a/AssetTools.NET/Extra/AssetsManager/AssetsFileInstance.cs +++ b/AssetTools.NET/Extra/AssetsManager/AssetsFileInstance.cs @@ -17,13 +17,11 @@ public class AssetsFileInstance //for monobehaviours public Dictionary monoIdToName = new Dictionary(); - public Stream AssetsStream => file.readerPar; - - public AssetsFileInstance(Stream stream, string filePath, string root) + public AssetsFileInstance(AssetsFileStatefulReader reader, string filePath, string root) { path = Path.GetFullPath(filePath); name = Path.Combine(root, Path.GetFileName(path)); - file = new AssetsFile(new AssetsFileReader(stream)); + file = new AssetsFile(reader); table = new AssetsFileTable(file); dependencies.AddRange( Enumerable.Range(0, file.dependencies.dependencyCount) @@ -31,15 +29,8 @@ public AssetsFileInstance(Stream stream, string filePath, string root) ); } public AssetsFileInstance(FileStream stream, string root) + : this(AssetsFileReaderHelper.createReader(stream), stream.Name, root) { - path = stream.Name; - name = Path.Combine(root, Path.GetFileName(path)); - file = new AssetsFile(new AssetsFileReader(stream)); - table = new AssetsFileTable(file); - dependencies.AddRange( - Enumerable.Range(0, file.dependencies.dependencyCount) - .Select(d => (AssetsFileInstance)null) - ); } public AssetsFileInstance GetDependency(AssetsManager am, int depIdx) diff --git a/AssetTools.NET/Extra/AssetsManager/AssetsManager.cs b/AssetTools.NET/Extra/AssetsManager/AssetsManager.cs index 8d6f390..e6c0d18 100644 --- a/AssetTools.NET/Extra/AssetsManager/AssetsManager.cs +++ b/AssetTools.NET/Extra/AssetsManager/AssetsManager.cs @@ -12,6 +12,7 @@ public class AssetsManager { public bool updateAfterLoad = true; public bool useTemplateFieldCache = false; + public bool useThreadSafeReader = true; public ClassDatabasePackage classPackage; public ClassDatabaseFile classFile; public List files = new List(); @@ -26,7 +27,8 @@ public AssetsFileInstance LoadAssetsFile(Stream stream, string path, bool loadDe int index = files.FindIndex(f => f.path.ToLower() == Path.GetFullPath(path).ToLower()); if (index == -1) { - instance = new AssetsFileInstance(stream, path, root); + var reader = AssetsFileReaderHelper.createReader(stream, useThreadSafeReader, true); + instance = new AssetsFileInstance(reader, path, root); instance.parentBundle = bunInst; files.Add(instance); } @@ -391,11 +393,6 @@ public AssetTypeTemplateField GetTemplateBaseField(AssetsFile file, AssetFileInf { baseField.FromClassDatabase(classFile, AssetHelper.FindAssetClassByID(classFile, fixedId), 0); } - - if (useTemplateFieldCache) - { - templateFieldCache[fixedId] = baseField; - } } return baseField; diff --git a/AssetTools.NET/Extra/BundleHelper.cs b/AssetTools.NET/Extra/BundleHelper.cs index d500516..77f2ec4 100644 --- a/AssetTools.NET/Extra/BundleHelper.cs +++ b/AssetTools.NET/Extra/BundleHelper.cs @@ -36,7 +36,7 @@ public static AssetsFile LoadAssetFromBundle(AssetBundleFile bundle, int index) { bundle.GetFileRange(index, out long offset, out long length); Stream stream = new SegmentStream(bundle.reader.BaseStream, offset, length); - AssetsFileReader reader = new AssetsFileReader(stream); + var reader = AssetsFileReaderHelper.createReader(stream, false, false); return new AssetsFile(reader); } diff --git a/AssetTools.NET/Extra/Net35Polyfill.cs b/AssetTools.NET/Extra/Net35Polyfill.cs index b0ff11b..40bebc4 100644 --- a/AssetTools.NET/Extra/Net35Polyfill.cs +++ b/AssetTools.NET/Extra/Net35Polyfill.cs @@ -24,5 +24,21 @@ public static void CopyToCompat(this Stream input, Stream output, long bytes = - bytes -= read; } } + public static void CopyToCompat(this AssetsFileStatefulReader reader, Stream output, long bytes = -1, int bufferSize = 80 * 1024) + { + byte[] buffer = new byte[bufferSize]; + int read; + + //set to largest value so we always go over buffer (hopefully) + if (bytes == -1) + bytes = long.MaxValue; + + //bufferSize will always be an int so if bytes is larger, it's also under the size of an int + while (bytes > 0 && (read = reader.Read(buffer, 0, (int)Math.Min(buffer.Length, bytes))) > 0) + { + output.Write(buffer, 0, read); + bytes -= read; + } + } } } diff --git a/AssetTools.NET/Standard/AssetTypeClass/AssetTypeInstance.cs b/AssetTools.NET/Standard/AssetTypeClass/AssetTypeInstance.cs index 6bbe5a8..015e50a 100644 --- a/AssetTools.NET/Standard/AssetTypeClass/AssetTypeInstance.cs +++ b/AssetTools.NET/Standard/AssetTypeClass/AssetTypeInstance.cs @@ -5,7 +5,7 @@ public class AssetTypeInstance public int baseFieldCount; public AssetTypeValueField[] baseFields; public byte[] memoryToClear; - public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileReader reader, long filePos) + public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileStatefulReader reader, long filePos) { reader.bigEndian = false; reader.Position = filePos; @@ -18,7 +18,7 @@ public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileReader r this.baseFields[i] = atvf; } } - public AssetTypeInstance(AssetTypeTemplateField baseField, AssetsFileReader reader, long filePos) + public AssetTypeInstance(AssetTypeTemplateField baseField, AssetsFileStatefulReader reader, long filePos) : this(new[] { baseField }, reader, filePos) { } public static AssetTypeValueField GetDummyAssetTypeField() diff --git a/AssetTools.NET/Standard/AssetTypeClass/AssetTypeTemplateField.cs b/AssetTools.NET/Standard/AssetTypeClass/AssetTypeTemplateField.cs index 0b4dc10..f8de040 100644 --- a/AssetTools.NET/Standard/AssetTypeClass/AssetTypeTemplateField.cs +++ b/AssetTools.NET/Standard/AssetTypeClass/AssetTypeTemplateField.cs @@ -85,7 +85,7 @@ public bool FromClassDatabase(ClassDatabaseFile file, ClassDatabaseType type, ui } return true; } - public AssetTypeValueField MakeValue(AssetsFileReader reader) + public AssetTypeValueField MakeValue(AssetsFileStatefulReader reader) { AssetTypeValueField valueField = new AssetTypeValueField(); valueField.templateField = this; @@ -93,7 +93,7 @@ public AssetTypeValueField MakeValue(AssetsFileReader reader) return valueField; } - public AssetTypeValueField ReadType(AssetsFileReader reader, AssetTypeValueField valueField) + public AssetTypeValueField ReadType(AssetsFileStatefulReader reader, AssetTypeValueField valueField) { if (valueField.templateField.isArray) { diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetFileInfo.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetFileInfo.cs index 1a7cc7f..8bdb738 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetFileInfo.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetFileInfo.cs @@ -26,7 +26,7 @@ public static int GetSize(uint version) if (0x0F <= version && version <= 0x10) size += 1; return size; } - public void Read(uint version, AssetsFileReader reader) + public void Read(uint version, AssetsFileStatefulReader reader) { reader.Align(); if (version >= 0x0E) diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetFileList.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetFileList.cs index 988d11f..3296710 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetFileList.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetFileList.cs @@ -20,7 +20,7 @@ public uint GetSizeBytes(uint version) return (uint)AssetFileInfo.GetSize(version) * sizeFiles + 4; } } - public void Read(uint version, AssetsFileReader reader) + public void Read(uint version, AssetsFileStatefulReader reader) { sizeFiles = reader.ReadUInt32(); fileInfs = new AssetFileInfo[sizeFiles]; diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFile.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFile.cs index 06c1cb1..0a7c216 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFile.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFile.cs @@ -17,13 +17,12 @@ public class AssetsFile public uint assetTablePos; public uint assetCount; - public AssetsFileReader reader; - public Stream readerPar; + public AssetsFileStatefulReader reader { get { return _reader.derive(); } } + public AssetsFileStatefulReader _reader; - public AssetsFile(AssetsFileReader reader) + public AssetsFile(AssetsFileStatefulReader reader) { - this.reader = reader; - readerPar = reader.BaseStream; + this._reader = reader; header = new AssetsFileHeader(); header.Read(reader); @@ -33,18 +32,18 @@ public AssetsFile(AssetsFileReader reader) assetCount = reader.ReadUInt32(); reader.Align(); - assetTablePos = (uint)reader.BaseStream.Position; + assetTablePos = (uint)reader.Position; int assetInfoSize = AssetFileInfo.GetSize(header.format); if (0x0F <= header.format && header.format <= 0x10) { //for these two versions, the asset info is not aligned //for the last entry, so we have to do some weird stuff - reader.BaseStream.Position += ((assetInfoSize + 3) >> 2 << 2) * (assetCount - 1) + assetInfoSize; + reader.Position += ((assetInfoSize + 3) >> 2 << 2) * (assetCount - 1) + assetInfoSize; } else { - reader.BaseStream.Position += AssetFileInfo.GetSize(header.format) * assetCount; + reader.Position += AssetFileInfo.GetSize(header.format) * assetCount; } if (header.format > 0x0B) { @@ -58,7 +57,7 @@ public AssetsFile(AssetsFileReader reader) public void Close() { - readerPar.Dispose(); + _reader.impl.Dispose(); } public void Write(AssetsFileWriter writer, long filePos, List replacers, uint fileID = 0, ClassDatabaseFile typeMeta = null) @@ -121,11 +120,11 @@ public void Write(AssetsFileWriter writer, long filePos, List re List newAssetInfos = new List(); // Collect unchanged assets (that aren't getting removed) - reader.Position = assetTablePos; + _reader.Position = assetTablePos; for (int i = 0; i < assetCount; i++) { AssetFileInfo oldAssetInfo = new AssetFileInfo(); - oldAssetInfo.Read(header.format, reader); + oldAssetInfo.Read(header.format, _reader); oldAssetInfosByPathId.Add(oldAssetInfo.index, oldAssetInfo); if (replacersByPathId.ContainsKey(oldAssetInfo.index)) @@ -226,8 +225,8 @@ public void Write(AssetsFileWriter writer, long filePos, List re else { AssetFileInfo oldAssetInfo = oldAssetInfosByPathId[newAssetInfo.index]; - reader.Position = header.firstFileOffset + oldAssetInfo.curFileOffset; - reader.BaseStream.CopyToCompat(writer.BaseStream, oldAssetInfo.curFileSize); + _reader.Position = header.firstFileOffset + oldAssetInfo.curFileOffset; + _reader.CopyToCompat(writer.BaseStream, oldAssetInfo.curFileSize); } newAssetInfo.curFileSize = (uint)(writer.Position - (newFirstFileOffset + newAssetInfo.curFileOffset)); @@ -266,11 +265,18 @@ public void Write(AssetsFileWriter writer, long filePos, List re public static bool IsAssetsFile(string filePath) { - using AssetsFileReader reader = new AssetsFileReader(filePath); - return IsAssetsFile(reader, 0, reader.BaseStream.Length); + using var reader = AssetsFileReaderHelper.createStreamReader(filePath, false); + return IsAssetsFile(reader, 0, reader.streamImpl.Length); } - public static bool IsAssetsFile(AssetsFileReader reader, long offset, long length) + // Temporary polyfill. + public static bool IsAssetsFile(AssetsFileReader legacyReader, long offset, long length) + { + var readerImpl = AssetsFileReaderHelper.createStreamReaderImpl(legacyReader.BaseStream); + var reader = new AssetsFileStatefulReader(readerImpl, true); + return IsAssetsFile(reader, offset, length); + } + public static bool IsAssetsFile(AssetsFileStatefulReader reader, long offset, long length) { //todo - not fully implemented if (length < 0x30) @@ -295,7 +301,7 @@ public static bool IsAssetsFile(AssetsFileReader reader, long offset, long lengt string possibleVersion = ""; char curChar; - while (reader.Position < reader.BaseStream.Length && (curChar = (char)reader.ReadByte()) != 0x00) + while (reader.Position < reader.streamImpl.Length && (curChar = (char)reader.ReadByte()) != 0x00) { possibleVersion += curChar; if (possibleVersion.Length > 0xFF) diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependency.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependency.cs index 6de5378..1025318 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependency.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependency.cs @@ -11,7 +11,7 @@ public struct GUID128 { public long mostSignificant; public long leastSignificant; - public void Read(AssetsFileReader reader) + public void Read(AssetsFileStatefulReader reader) { mostSignificant = reader.ReadInt64(); leastSignificant = reader.ReadInt64(); @@ -27,7 +27,7 @@ public void Write(AssetsFileWriter writer) public int type; public string assetPath; public string originalAssetPath; - public void Read(AssetsFileReader reader) + public void Read(AssetsFileStatefulReader reader) { bufferedPath = reader.ReadNullTerminated(); guid = new GUID128(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependencyList.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependencyList.cs index 872e634..608dbb8 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependencyList.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileDependencyList.cs @@ -9,7 +9,7 @@ public class AssetsFileDependencyList { public int dependencyCount; public List dependencies; - public void Read(AssetsFileReader reader) + public void Read(AssetsFileStatefulReader reader) { dependencyCount = reader.ReadInt32(); dependencies = new List(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileHeader.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileHeader.cs index 14ee5d8..b743948 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileHeader.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileHeader.cs @@ -23,7 +23,7 @@ public int GetSizeBytes() else return 0x14; } - public void Read(AssetsFileReader reader) + public void Read(AssetsFileStatefulReader reader) { metadataSize = reader.ReadUInt32(); fileSize = reader.ReadUInt32(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderMMapImpl.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderMMapImpl.cs new file mode 100644 index 0000000..42c8736 --- /dev/null +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderMMapImpl.cs @@ -0,0 +1,148 @@ +#if !NET35 +using System; +using System.IO; +using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; +using System.Text; + +namespace AssetsTools.NET +{ + public class AssetsFileReaderMMapImpl : IAssetsFileReaderImpl + { + public MemoryMappedViewAccessor mmapView; + internal IntPtr ptr; + public readonly long initialSize; + public readonly string filePath; + + public AssetsFileReaderMMapImpl(MemoryMappedViewAccessor mmapView, long size, string filePath = null) + { + this.filePath = filePath; + initialSize = (size < 0) ? mmapView.Capacity : size; + Init(mmapView); + } + + public AssetsFileReaderMMapImpl(FileStream fileStream) + { + filePath = fileStream.Name; + initialSize = fileStream.Length; + // MemoryMappedFile.CreateFromFile(FileStream, ...) requires .NET 4.6+. + var mmapFile = MemoryMappedFile.CreateFromFile(filePath); + Init(mmapFile.CreateViewAccessor()); + } + + internal void Init(MemoryMappedViewAccessor mmapView) + { + this.mmapView = mmapView; + unsafe { + byte* bytes = null; + mmapView.SafeMemoryMappedViewHandle.AcquirePointer(ref bytes); + ptr = (IntPtr)bytes; + } + } + public void Dispose() + { + mmapView.SafeMemoryMappedViewHandle.ReleasePointer(); + mmapView.Dispose(); + } + + public long InitialSize + { + get { return initialSize; } + } + public string BackingFile + { + get { return filePath; } + } + public byte ReadByte(long position) + { + return mmapView.ReadByte(position); + } + public sbyte ReadSByte(long position) + { + return mmapView.ReadSByte(position); + } + public short ReadInt16(long position) + { + return mmapView.ReadInt16(position); + } + public ushort ReadUInt16(long position) + { + return mmapView.ReadUInt16(position); + } + public int ReadInt32(long position) + { + return mmapView.ReadInt32(position); + } + public uint ReadUInt32(long position) + { + return mmapView.ReadUInt32(position); + } + public long ReadInt64(long position) + { + return mmapView.ReadInt64(position); + } + public ulong ReadUInt64(long position) + { + return mmapView.ReadUInt64(position); + } + public float ReadSingle(long position) + { + return mmapView.ReadSingle(position); + } + public double ReadDouble(long position) + { + return mmapView.ReadDouble(position); + } + public int Read(long position, byte[] buffer, int count) + { + int readCount = (int)Math.Max(0, Math.Min(initialSize - position, count)); + unsafe { + Marshal.Copy((IntPtr)((byte*)ptr + position), buffer, 0, readCount); + } + return readCount; + } + public byte[] ReadBytes(long position, int count) + { + var bytes = new byte[count]; + unsafe { + Marshal.Copy((IntPtr)((byte*)ptr + position), bytes, 0, count); + } + return bytes; + } + public string ReadStringLength(long position, int len) + { + unsafe { + return ReadUTF8String(position, len); + } + } + public string ReadNullTerminated(long position, out int rlen) + { + unsafe { + byte* bytes = (byte*)ptr; + rlen = AssetsFileReaderHelper.strlen(bytes + position) + 1; + return ReadUTF8String(position, rlen - 1); + } + } + // For .NET 4.0 compatibility. + public unsafe string ReadUTF8String(long position, int len) + { +#if NET46_OR_GREATER + byte* bytes = (byte*)ptr; + return Encoding.UTF8.GetString(bytes + position, len); +#else + return Encoding.UTF8.GetString(ReadBytes(position, len), 0, len); +#endif + } + } + + public static partial class AssetsFileReaderHelper + { + public static IAssetsFileReaderImpl createMMapReaderImpl(string filePath) + { + var mmapFile = MemoryMappedFile.CreateFromFile(filePath); + long size = (new FileInfo(filePath)).Length; + return new AssetsFileReaderMMapImpl(mmapFile.CreateViewAccessor(), size, filePath); + } + } +} +#endif diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderStreamImpl.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderStreamImpl.cs new file mode 100644 index 0000000..60726fa --- /dev/null +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileReaderStreamImpl.cs @@ -0,0 +1,115 @@ +using System; +using System.IO; +using System.Text; + +namespace AssetsTools.NET +{ + public class AssetsFileReaderStreamImpl : BinaryReader, IAssetsFileReaderImpl, IAssetsFileReaderStreamExtra + { + public AssetsFileReaderStreamImpl(Stream stream) + : base(stream) + { + } + + public long InitialSize + { + get { return BaseStream.Length; } + } + public string BackingFile + { + get { return (BaseStream as FileStream)?.Name; } + } + public byte ReadByte(long position) + { + Check(position); + return base.ReadByte(); + } + public sbyte ReadSByte(long position) + { + Check(position); + return base.ReadSByte(); + } + public short ReadInt16(long position) + { + Check(position); + return base.ReadInt16(); + } + public ushort ReadUInt16(long position) + { + Check(position); + return base.ReadUInt16(); + } + public int ReadInt32(long position) + { + Check(position); + return base.ReadInt32(); + } + public uint ReadUInt32(long position) + { + Check(position); + return base.ReadUInt32(); + } + public long ReadInt64(long position) + { + Check(position); + return base.ReadInt64(); + } + public ulong ReadUInt64(long position) + { + Check(position); + return base.ReadUInt64(); + } + public float ReadSingle(long position) + { + Check(position); + return base.ReadSingle(); + } + public double ReadDouble(long position) + { + Check(position); + return base.ReadDouble(); + } + public int Read(long position, byte[] buffer, int count) + { + BaseStream.Seek(position, SeekOrigin.Begin); + return base.Read(buffer, 0, count); + } + public byte[] ReadBytes(long position, int count) + { + Check(position); + return base.ReadBytes(count); + } + public string ReadStringLength(long position, int len) + { + Check(position); + return Encoding.UTF8.GetString(ReadBytes(len)); + } + public string ReadNullTerminated(long position, out int rlen) + { + Check(position); + long savedPosition = BaseStream.Position; + string output = ""; + char curChar; + while ((curChar = ReadChar()) != 0x00) + { + output += curChar; + } + rlen = (int)(BaseStream.Position - savedPosition); + return output; + } + + public long Length + { + get { return BaseStream.Length; } + } + public void UpdatePosition(long position) + { + BaseStream.Position = position; + } + public void Check(long position) + { + if (position != BaseStream.Position) + throw new InvalidOperationException(); + } + } +} diff --git a/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileStatefulReader.cs b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileStatefulReader.cs new file mode 100644 index 0000000..3df2be8 --- /dev/null +++ b/AssetTools.NET/Standard/AssetsFileFormat/AssetsFileStatefulReader.cs @@ -0,0 +1,266 @@ +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices.ComTypes; +using System.Text; +using AssetsTools.NET.Extra; +using Mono.Cecil; + +namespace AssetsTools.NET +{ + public class AssetsFileStatefulReader : IDisposable + { + //todo, this should default to bigEndian = false + //since it's more likely little endian than big endian + public bool bigEndian = true; + public readonly bool shared; + public bool threadSafe; + public IAssetsFileReaderImpl impl; + public IAssetsFileReaderStreamExtra streamImpl; + internal long position = -1; + + public AssetsFileStatefulReader(IAssetsFileReaderImpl impl, bool shared) + { + this.impl = impl; + this.shared = shared; + streamImpl = impl as IAssetsFileReaderStreamExtra; + threadSafe = streamImpl == null; + position = 0; + } + + internal AssetsFileStatefulReader(AssetsFileStatefulReader other) + { + if (!other.shared) throw new InvalidOperationException(); + impl = other.impl; + shared = true; + streamImpl = other.streamImpl; + threadSafe = other.threadSafe; + } + + public void Close() + { + if (!shared) + impl.Dispose(); + } + public void Dispose() { Close(); } + + public byte ReadByte() + { + byte value = impl.ReadByte(position); + position++; + return value; + } + public sbyte ReadSByte() + { + sbyte value = impl.ReadSByte(position); + position++; + return value; + } + public bool ReadBoolean() { return ReadByte() != 0; } + public short ReadInt16() + { + unchecked + { + short value = impl.ReadInt16(position); + position += 2; + return bigEndian ? (short)ReverseShort((ushort)value) : value; + } + } + public ushort ReadUInt16() + { + unchecked + { + ushort value = impl.ReadUInt16(position); + position += 2; + return bigEndian ? ReverseShort(value) : value; + } + } + public int ReadInt24() + { + unchecked + { + return bigEndian ? (int)ReverseInt((uint)System.BitConverter.ToInt32(ReadBytes(3).Concat(new byte[] { 0 }).ToArray(), 0)) : + System.BitConverter.ToInt32(ReadBytes(3).Concat(new byte[] { 0 }).ToArray(), 0); + } + } + public uint ReadUInt24() + { + unchecked + { + return bigEndian ? ReverseInt(System.BitConverter.ToUInt32(ReadBytes(3).Concat(new byte[] { 0 }).ToArray(), 0)) : + System.BitConverter.ToUInt32(ReadBytes(3).Concat(new byte[] { 0 }).ToArray(), 0); + } + } + public int ReadInt32() + { + unchecked + { + int value = impl.ReadInt32(position); + position += 4; + return bigEndian ? (int)ReverseInt((uint)value) : value; + } + } + public uint ReadUInt32() + { + unchecked + { + uint value = impl.ReadUInt32(position); + position += 4; + return bigEndian ? ReverseInt(value) : value; + } + } + public long ReadInt64() + { + unchecked + { + long value = impl.ReadInt64(position); + position += 8; + return bigEndian ? (long)ReverseLong((ulong)value) : value; + } + } + public ulong ReadUInt64() + { + unchecked + { + ulong value = impl.ReadUInt64(position); + position += 8; + return bigEndian ? ReverseLong(value) : value; + } + } + public float ReadSingle() + { + float value = impl.ReadSingle(position); + position += 4; + return value; + } + public double ReadDouble() + { + double value = impl.ReadDouble(position); + position += 8; + return value; + } + public ushort ReverseShort(ushort value) + { + return (ushort)(((value & 0xFF00) >> 8) | (value & 0x00FF) << 8); + } + public uint ReverseInt(uint value) + { + value = (value >> 16) | (value << 16); + return ((value & 0xFF00FF00) >> 8) | ((value & 0x00FF00FF) << 8); + } + public ulong ReverseLong(ulong value) + { + value = (value >> 32) | (value << 32); + value = ((value & 0xFFFF0000FFFF0000) >> 16) | ((value & 0x0000FFFF0000FFFF) << 16); + return ((value & 0xFF00FF00FF00FF00) >> 8) | ((value & 0x00FF00FF00FF00FF) << 8); + } + public void Align() + { + long pad = 4 - (position % 4); + if (pad != 4) position += pad; + streamImpl?.UpdatePosition(position); + } + public void Align8() + { + long pad = 8 - (position % 8); + if (pad != 8) position += pad; + streamImpl?.UpdatePosition(position); + } + public void Align16() + { + long pad = 16 - (position % 16); + if (pad != 16) position += pad; + streamImpl?.UpdatePosition(position); + } + public int Read(byte[] buffer, int offset, int count) + { + int readCount = impl.Read(position + offset, buffer, count); + position = position + offset + readCount; + return readCount; + } + public byte[] ReadBytes(int count) + { + byte[] bytes = impl.ReadBytes(position, count); + position += count; + return bytes; + } + public string ReadStringLength(int len) + { + string value = impl.ReadStringLength(position, len); + position += len; + return value; + } + public string ReadNullTerminated() + { + string value = impl.ReadNullTerminated(position, out int rlen); + position += rlen; + return value; + } + public static string ReadNullTerminatedArray(byte[] bytes, uint pos) + { + return AssetsFileReaderHelper.ReadNullTerminatedArray(bytes, pos); + } + public string ReadCountString() + { + byte length = ReadByte(); + return ReadStringLength(length); + } + public string ReadCountStringInt16() + { + ushort length = ReadUInt16(); + return ReadStringLength(length); + } + public string ReadCountStringInt32() + { + int length = ReadInt32(); + return ReadStringLength(length); + } + + public long InitialSize + { + get { return impl.InitialSize; } + } + public string BackingFile + { + get { return impl.BackingFile; } + } + public long Position + { + get { return position; } + set { + position = value; + streamImpl?.UpdatePosition(value); + } + } + + public AssetsFileStatefulReader clone() + { + var readerClone = new AssetsFileStatefulReader(this); + readerClone.position = position; + return readerClone; + } + + public AssetsFileStatefulReader derive(long newPosition = -1) + { + var readerClone = new AssetsFileStatefulReader(this); + readerClone.position = newPosition; + return readerClone; + } + } + + public static partial class AssetsFileReaderHelper + { + public static AssetsFileStatefulReader createReader(string filePath, bool shared = true) + { + return new AssetsFileStatefulReader(createReaderImpl(filePath), shared); + } + public static AssetsFileStatefulReader createStreamReader(string filePath, bool shared = true) + { + return new AssetsFileStatefulReader(createStreamReaderImpl(filePath), shared); + } + public static AssetsFileStatefulReader createReader(Stream stream, bool useThreadSafeReader = true, bool shared = true) + { + return new AssetsFileStatefulReader(createReaderImpl(stream, useThreadSafeReader), shared); + } + } +} diff --git a/AssetTools.NET/Standard/AssetsFileFormat/IAssetsFileReaderImpl.cs b/AssetTools.NET/Standard/AssetsFileFormat/IAssetsFileReaderImpl.cs new file mode 100644 index 0000000..0670080 --- /dev/null +++ b/AssetTools.NET/Standard/AssetsFileFormat/IAssetsFileReaderImpl.cs @@ -0,0 +1,91 @@ +using System; +using System.IO; + +namespace AssetsTools.NET +{ + // Native endian (=little-endian) stateless reader interface. Big endian is swapped in StatefulReader. + public interface IAssetsFileReaderImpl : IDisposable + { + long InitialSize { get; } + string BackingFile { get; } + byte ReadByte(long position); + sbyte ReadSByte(long position); + short ReadInt16(long position); + ushort ReadUInt16(long position); + int ReadInt32(long position); + uint ReadUInt32(long position); + long ReadInt64(long position); + ulong ReadUInt64(long position); + float ReadSingle(long position); + double ReadDouble(long position); + int Read(long position, byte[] buffer, int count); + byte[] ReadBytes(long position, int count); + string ReadStringLength(long position, int len); + string ReadNullTerminated(long position, out int len); // len: including null-terminator. + } + + public interface IAssetsFileReaderStreamExtra + { + long Length { get; } + void UpdatePosition(long position); + } + + public static partial class AssetsFileReaderHelper + { + public static IAssetsFileReaderImpl createReaderImpl(string filePath) + { +#if NET35 + return createStreamReaderImpl(filePath); +#else + return createMMapReaderImpl(filePath); +#endif + } + public static IAssetsFileReaderImpl createReaderImpl(Stream stream, bool useThreadSafeReader = true) + { + if (!useThreadSafeReader) { + return createStreamReaderImpl(stream); + } else { + if (stream is FileStream fs) { + return createReaderImpl(fs.Name); + } else { + throw new InvalidOperationException(); + } + } + } + + public static AssetsFileReaderStreamImpl createStreamReaderImpl(string filePath) + { + return new AssetsFileReaderStreamImpl(File.OpenRead(filePath)); + } + public static AssetsFileReaderStreamImpl createStreamReaderImpl(Stream stream) + { + return new AssetsFileReaderStreamImpl(stream); + } + + public static unsafe int strlen(byte[] bytes, long pos) + { + fixed (byte* s = bytes) + return strlen(s + pos); + } + public static unsafe int strlen(byte* s) + { + byte* p = s; + while(*p++ != '\0'); + return (int)(p - s - 1); + } + public static string ReadNullTerminatedArray(byte[] bytes, long pos) + { + unsafe { + fixed (byte* pb = bytes) + return ReadNullTerminatedArray(pb, pos); + } + } + public static unsafe string ReadNullTerminatedArray(byte* bytes, long pos) + { + // Unlike Encoding.ASCII.GetString: + // String(sbyte*) (w/o Encoding) variants accept 8-bit value. (or Encoding=Latin1, N/A in .NET framework) + // String(sbyte*) (w/o length) handles null-termination; String(sbyte*,int,int) does not. + return new String((sbyte*)bytes + pos); + } + } +} diff --git a/AssetTools.NET/Standard/AssetsFileFormat/PreloadList.cs b/AssetTools.NET/Standard/AssetsFileFormat/PreloadList.cs index 2de34da..7b099d8 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/PreloadList.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/PreloadList.cs @@ -7,7 +7,7 @@ public class PreloadList public int len; public List items; - public void Read(AssetsFileReader reader) + public void Read(AssetsFileStatefulReader reader) { len = reader.ReadInt32(); items = new List(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/TypeField_0D.cs b/AssetTools.NET/Standard/AssetsFileFormat/TypeField_0D.cs index adaff89..14d05af 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/TypeField_0D.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/TypeField_0D.cs @@ -13,7 +13,7 @@ public class TypeField_0D public uint index; public uint flags; public byte[] unknown; - public void Read(AssetsFileReader reader, uint format) + public void Read(AssetsFileStatefulReader reader, uint format) { version = reader.ReadUInt16(); depth = reader.ReadByte(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/TypeTree.cs b/AssetTools.NET/Standard/AssetsFileFormat/TypeTree.cs index 2c30cb3..1f8840a 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/TypeTree.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/TypeTree.cs @@ -14,7 +14,7 @@ public class TypeTree public uint dwUnknown; - public void Read(AssetsFileReader reader, uint version) + public void Read(AssetsFileStatefulReader reader, uint version) { unityVersion = reader.ReadNullTerminated(); this.version = reader.ReadUInt32(); diff --git a/AssetTools.NET/Standard/AssetsFileFormat/Type_0D.cs b/AssetTools.NET/Standard/AssetsFileFormat/Type_0D.cs index 61be0af..fa61f1f 100644 --- a/AssetTools.NET/Standard/AssetsFileFormat/Type_0D.cs +++ b/AssetTools.NET/Standard/AssetsFileFormat/Type_0D.cs @@ -28,7 +28,7 @@ public class Type_0D public int dependenciesCount; public int[] dependencies; - public void Read(bool hasTypeTree, AssetsFileReader reader, uint version) + public void Read(bool hasTypeTree, AssetsFileStatefulReader reader, uint version) { classId = reader.ReadInt32(); if (version >= 0x10) unknown16_1 = reader.ReadByte(); diff --git a/AssetTools.NET/Standard/AssetsFileTable/AssetFileInfoEx.cs b/AssetTools.NET/Standard/AssetsFileTable/AssetFileInfoEx.cs index b0e39f1..0d53867 100644 --- a/AssetTools.NET/Standard/AssetsFileTable/AssetFileInfoEx.cs +++ b/AssetTools.NET/Standard/AssetsFileTable/AssetFileInfoEx.cs @@ -11,7 +11,7 @@ public class AssetFileInfoEx : AssetFileInfo public bool ReadName(AssetsFile file, out string str) { str = string.Empty; - AssetsFileReader reader = file.reader; + var reader = file.reader; if (AssetsFileExtra.HasName(curFileType)) { reader.Position = absoluteFilePos; diff --git a/AssetTools.NET/Standard/AssetsFileTable/AssetsFileTable.cs b/AssetTools.NET/Standard/AssetsFileTable/AssetsFileTable.cs index 22b571a..adb4e40 100644 --- a/AssetTools.NET/Standard/AssetsFileTable/AssetsFileTable.cs +++ b/AssetTools.NET/Standard/AssetsFileTable/AssetsFileTable.cs @@ -6,8 +6,7 @@ namespace AssetsTools.NET public class AssetsFileTable { public AssetsFile file; - public AssetsFileReader reader; - public Stream readerPar; + public AssetsFileStatefulReader reader; public AssetFileInfoEx[] assetFileInfo; public uint assetFileInfoCount; @@ -19,9 +18,8 @@ public AssetsFileTable(AssetsFile file) { this.file = file; reader = file.reader; - readerPar = file.readerPar; reader.bigEndian = file.header.endianness == 1 ? true : false; - reader.BaseStream.Position = file.assetTablePos; + reader.Position = file.assetTablePos; assetFileInfoCount = file.assetCount; assetFileInfo = new AssetFileInfoEx[assetFileInfoCount]; for (int i = 0; i < assetFileInfoCount; i++) diff --git a/AssetTools.NET/Standard/BundleReplacer/BundleReplacerFromAssets.cs b/AssetTools.NET/Standard/BundleReplacer/BundleReplacerFromAssets.cs index 762a517..0f5d9b9 100644 --- a/AssetTools.NET/Standard/BundleReplacer/BundleReplacerFromAssets.cs +++ b/AssetTools.NET/Standard/BundleReplacer/BundleReplacerFromAssets.cs @@ -52,7 +52,7 @@ public override bool Init(AssetsFileReader entryReader, long entryPos, long entr return false; SegmentStream stream = new SegmentStream(entryReader.BaseStream, entryPos, entrySize); - AssetsFileReader reader = new AssetsFileReader(stream); + var reader = AssetsFileReaderHelper.createReader(stream, false, false); assetsFile = new AssetsFile(reader); return true; } diff --git a/AssetTools.NET/Standard/TextureFileFormat/TextureFile.cs b/AssetTools.NET/Standard/TextureFileFormat/TextureFile.cs index 95f9f39..cb8e4be 100644 --- a/AssetTools.NET/Standard/TextureFileFormat/TextureFile.cs +++ b/AssetTools.NET/Standard/TextureFileFormat/TextureFile.cs @@ -254,9 +254,9 @@ public byte[] GetTextureData(AssetsFileInstance inst) public byte[] GetTextureData(AssetsFile file) { string path = null; - if (file.readerPar is FileStream fs) + if (file.reader.BackingFile != null) { - path = Path.GetDirectoryName(fs.Name); + path = Path.GetDirectoryName(file.reader.BackingFile); } return GetTextureData(path); }