Skip to content

Thread-safe reader PoC #87

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: assetstoolsv2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions AssetTools.NET/AssetsTools.NET.csproj
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFrameworks>netstandard2.0;net35;net40</TargetFrameworks>
<TargetFrameworks>netstandard2.0;net35;net40;net462</TargetFrameworks>
<OutputType>Library</OutputType>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<Platforms>AnyCPU</Platforms>
<LangVersion>8</LangVersion>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|x64'">
<OutputPath>bin\x64\Debug\</OutputPath>
Expand All @@ -18,10 +19,7 @@
<None Remove="*.nupkg" />
<None Remove="*.nuspec" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net35'">
<PackageReference Include="Mono.Cecil" Version="0.10.4" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'net40'">
<ItemGroup Condition="'$(TargetFramework.TrimEnd(`0123456789.`))' == 'net'">
<PackageReference Include="Mono.Cecil" Version="0.10.4" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
Expand Down
4 changes: 2 additions & 2 deletions AssetTools.NET/Extra/AssetHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ public static ushort GetScriptIndex(AssetsFile file, AssetFileInfoEx info)
public static string GetAssetNameFast(AssetsFile file, ClassDatabaseFile cldb, AssetFileInfoEx info)
{
ClassDatabaseType type = FindAssetClassByID(cldb, info.curFileType);
AssetsFileReader reader = file.reader;
var reader = file.reader;

if (file.typeTree.hasTypeTree)
{
Expand Down Expand Up @@ -168,7 +168,7 @@ public static string GetAssetNameFast(AssetsFile file, ClassDatabaseFile cldb, A
//no classdatabase but may not work
public static string GetAssetNameFastNaive(AssetsFile file, AssetFileInfoEx info)
{
AssetsFileReader reader = file.reader;
var reader = file.reader;

if (AssetsFileExtra.HasName(info.curFileType))
{
Expand Down
15 changes: 3 additions & 12 deletions AssetTools.NET/Extra/AssetsManager/AssetsFileInstance.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,29 +17,20 @@ public class AssetsFileInstance
//for monobehaviours
public Dictionary<uint, string> monoIdToName = new Dictionary<uint, string>();

public Stream AssetsStream => file.readerPar;

public AssetsFileInstance(Stream stream, string filePath, string root)
public AssetsFileInstance(AssetsFileStatefulReader reader, string filePath, string root)
{
path = Path.GetFullPath(filePath);
name = Path.Combine(root, Path.GetFileName(path));
file = new AssetsFile(new AssetsFileReader(stream));
file = new AssetsFile(reader);
table = new AssetsFileTable(file);
dependencies.AddRange(
Enumerable.Range(0, file.dependencies.dependencyCount)
.Select(d => (AssetsFileInstance)null)
);
}
public AssetsFileInstance(FileStream stream, string root)
: this(AssetsFileReaderHelper.createReader(stream), stream.Name, root)
{
path = stream.Name;
name = Path.Combine(root, Path.GetFileName(path));
file = new AssetsFile(new AssetsFileReader(stream));
table = new AssetsFileTable(file);
dependencies.AddRange(
Enumerable.Range(0, file.dependencies.dependencyCount)
.Select(d => (AssetsFileInstance)null)
);
}

public AssetsFileInstance GetDependency(AssetsManager am, int depIdx)
Expand Down
9 changes: 3 additions & 6 deletions AssetTools.NET/Extra/AssetsManager/AssetsManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ public class AssetsManager
{
public bool updateAfterLoad = true;
public bool useTemplateFieldCache = false;
public bool useThreadSafeReader = true;
public ClassDatabasePackage classPackage;
public ClassDatabaseFile classFile;
public List<AssetsFileInstance> files = new List<AssetsFileInstance>();
Expand All @@ -26,7 +27,8 @@ public AssetsFileInstance LoadAssetsFile(Stream stream, string path, bool loadDe
int index = files.FindIndex(f => f.path.ToLower() == Path.GetFullPath(path).ToLower());
if (index == -1)
{
instance = new AssetsFileInstance(stream, path, root);
var reader = AssetsFileReaderHelper.createReader(stream, useThreadSafeReader, true);
instance = new AssetsFileInstance(reader, path, root);
instance.parentBundle = bunInst;
files.Add(instance);
}
Expand Down Expand Up @@ -391,11 +393,6 @@ public AssetTypeTemplateField GetTemplateBaseField(AssetsFile file, AssetFileInf
{
baseField.FromClassDatabase(classFile, AssetHelper.FindAssetClassByID(classFile, fixedId), 0);
}

if (useTemplateFieldCache)
{
templateFieldCache[fixedId] = baseField;
}
}

return baseField;
Expand Down
2 changes: 1 addition & 1 deletion AssetTools.NET/Extra/BundleHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public static AssetsFile LoadAssetFromBundle(AssetBundleFile bundle, int index)
{
bundle.GetFileRange(index, out long offset, out long length);
Stream stream = new SegmentStream(bundle.reader.BaseStream, offset, length);
AssetsFileReader reader = new AssetsFileReader(stream);
var reader = AssetsFileReaderHelper.createReader(stream, false, false);
return new AssetsFile(reader);
}

Expand Down
16 changes: 16 additions & 0 deletions AssetTools.NET/Extra/Net35Polyfill.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,5 +24,21 @@ public static void CopyToCompat(this Stream input, Stream output, long bytes = -
bytes -= read;
}
}
public static void CopyToCompat(this AssetsFileStatefulReader reader, Stream output, long bytes = -1, int bufferSize = 80 * 1024)
{
byte[] buffer = new byte[bufferSize];
int read;

//set to largest value so we always go over buffer (hopefully)
if (bytes == -1)
bytes = long.MaxValue;

//bufferSize will always be an int so if bytes is larger, it's also under the size of an int
while (bytes > 0 && (read = reader.Read(buffer, 0, (int)Math.Min(buffer.Length, bytes))) > 0)
{
output.Write(buffer, 0, read);
bytes -= read;
}
}
}
}
4 changes: 2 additions & 2 deletions AssetTools.NET/Standard/AssetTypeClass/AssetTypeInstance.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ public class AssetTypeInstance
public int baseFieldCount;
public AssetTypeValueField[] baseFields;
public byte[] memoryToClear;
public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileReader reader, long filePos)
public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileStatefulReader reader, long filePos)
{
reader.bigEndian = false;
reader.Position = filePos;
Expand All @@ -18,7 +18,7 @@ public AssetTypeInstance(AssetTypeTemplateField[] baseFields, AssetsFileReader r
this.baseFields[i] = atvf;
}
}
public AssetTypeInstance(AssetTypeTemplateField baseField, AssetsFileReader reader, long filePos)
public AssetTypeInstance(AssetTypeTemplateField baseField, AssetsFileStatefulReader reader, long filePos)
: this(new[] { baseField }, reader, filePos) { }

public static AssetTypeValueField GetDummyAssetTypeField()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,15 +85,15 @@ public bool FromClassDatabase(ClassDatabaseFile file, ClassDatabaseType type, ui
}
return true;
}
public AssetTypeValueField MakeValue(AssetsFileReader reader)
public AssetTypeValueField MakeValue(AssetsFileStatefulReader reader)
{
AssetTypeValueField valueField = new AssetTypeValueField();
valueField.templateField = this;
valueField = ReadType(reader, valueField);
return valueField;
}

public AssetTypeValueField ReadType(AssetsFileReader reader, AssetTypeValueField valueField)
public AssetTypeValueField ReadType(AssetsFileStatefulReader reader, AssetTypeValueField valueField)
{
if (valueField.templateField.isArray)
{
Expand Down
2 changes: 1 addition & 1 deletion AssetTools.NET/Standard/AssetsFileFormat/AssetFileInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public static int GetSize(uint version)
if (0x0F <= version && version <= 0x10) size += 1;
return size;
}
public void Read(uint version, AssetsFileReader reader)
public void Read(uint version, AssetsFileStatefulReader reader)
{
reader.Align();
if (version >= 0x0E)
Expand Down
2 changes: 1 addition & 1 deletion AssetTools.NET/Standard/AssetsFileFormat/AssetFileList.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public uint GetSizeBytes(uint version)
return (uint)AssetFileInfo.GetSize(version) * sizeFiles + 4;
}
}
public void Read(uint version, AssetsFileReader reader)
public void Read(uint version, AssetsFileStatefulReader reader)
{
sizeFiles = reader.ReadUInt32();
fileInfs = new AssetFileInfo[sizeFiles];
Expand Down
40 changes: 23 additions & 17 deletions AssetTools.NET/Standard/AssetsFileFormat/AssetsFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,12 @@ public class AssetsFile
public uint assetTablePos;
public uint assetCount;

public AssetsFileReader reader;
public Stream readerPar;
public AssetsFileStatefulReader reader { get { return _reader.derive(); } }
public AssetsFileStatefulReader _reader;

public AssetsFile(AssetsFileReader reader)
public AssetsFile(AssetsFileStatefulReader reader)
{
this.reader = reader;
readerPar = reader.BaseStream;
this._reader = reader;

header = new AssetsFileHeader();
header.Read(reader);
Expand All @@ -33,18 +32,18 @@ public AssetsFile(AssetsFileReader reader)

assetCount = reader.ReadUInt32();
reader.Align();
assetTablePos = (uint)reader.BaseStream.Position;
assetTablePos = (uint)reader.Position;

int assetInfoSize = AssetFileInfo.GetSize(header.format);
if (0x0F <= header.format && header.format <= 0x10)
{
//for these two versions, the asset info is not aligned
//for the last entry, so we have to do some weird stuff
reader.BaseStream.Position += ((assetInfoSize + 3) >> 2 << 2) * (assetCount - 1) + assetInfoSize;
reader.Position += ((assetInfoSize + 3) >> 2 << 2) * (assetCount - 1) + assetInfoSize;
}
else
{
reader.BaseStream.Position += AssetFileInfo.GetSize(header.format) * assetCount;
reader.Position += AssetFileInfo.GetSize(header.format) * assetCount;
}
if (header.format > 0x0B)
{
Expand All @@ -58,7 +57,7 @@ public AssetsFile(AssetsFileReader reader)

public void Close()
{
readerPar.Dispose();
_reader.impl.Dispose();
}

public void Write(AssetsFileWriter writer, long filePos, List<AssetsReplacer> replacers, uint fileID = 0, ClassDatabaseFile typeMeta = null)
Expand Down Expand Up @@ -121,11 +120,11 @@ public void Write(AssetsFileWriter writer, long filePos, List<AssetsReplacer> re
List<AssetFileInfo> newAssetInfos = new List<AssetFileInfo>();

// Collect unchanged assets (that aren't getting removed)
reader.Position = assetTablePos;
_reader.Position = assetTablePos;
for (int i = 0; i < assetCount; i++)
{
AssetFileInfo oldAssetInfo = new AssetFileInfo();
oldAssetInfo.Read(header.format, reader);
oldAssetInfo.Read(header.format, _reader);
oldAssetInfosByPathId.Add(oldAssetInfo.index, oldAssetInfo);

if (replacersByPathId.ContainsKey(oldAssetInfo.index))
Expand Down Expand Up @@ -226,8 +225,8 @@ public void Write(AssetsFileWriter writer, long filePos, List<AssetsReplacer> re
else
{
AssetFileInfo oldAssetInfo = oldAssetInfosByPathId[newAssetInfo.index];
reader.Position = header.firstFileOffset + oldAssetInfo.curFileOffset;
reader.BaseStream.CopyToCompat(writer.BaseStream, oldAssetInfo.curFileSize);
_reader.Position = header.firstFileOffset + oldAssetInfo.curFileOffset;
_reader.CopyToCompat(writer.BaseStream, oldAssetInfo.curFileSize);
}

newAssetInfo.curFileSize = (uint)(writer.Position - (newFirstFileOffset + newAssetInfo.curFileOffset));
Expand Down Expand Up @@ -266,11 +265,18 @@ public void Write(AssetsFileWriter writer, long filePos, List<AssetsReplacer> re

public static bool IsAssetsFile(string filePath)
{
using AssetsFileReader reader = new AssetsFileReader(filePath);
return IsAssetsFile(reader, 0, reader.BaseStream.Length);
using var reader = AssetsFileReaderHelper.createStreamReader(filePath, false);
return IsAssetsFile(reader, 0, reader.streamImpl.Length);
}

public static bool IsAssetsFile(AssetsFileReader reader, long offset, long length)
// Temporary polyfill.
public static bool IsAssetsFile(AssetsFileReader legacyReader, long offset, long length)
{
var readerImpl = AssetsFileReaderHelper.createStreamReaderImpl(legacyReader.BaseStream);
var reader = new AssetsFileStatefulReader(readerImpl, true);
return IsAssetsFile(reader, offset, length);
}
public static bool IsAssetsFile(AssetsFileStatefulReader reader, long offset, long length)
{
//todo - not fully implemented
if (length < 0x30)
Expand All @@ -295,7 +301,7 @@ public static bool IsAssetsFile(AssetsFileReader reader, long offset, long lengt

string possibleVersion = "";
char curChar;
while (reader.Position < reader.BaseStream.Length && (curChar = (char)reader.ReadByte()) != 0x00)
while (reader.Position < reader.streamImpl.Length && (curChar = (char)reader.ReadByte()) != 0x00)
{
possibleVersion += curChar;
if (possibleVersion.Length > 0xFF)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public struct GUID128
{
public long mostSignificant;
public long leastSignificant;
public void Read(AssetsFileReader reader)
public void Read(AssetsFileStatefulReader reader)
{
mostSignificant = reader.ReadInt64();
leastSignificant = reader.ReadInt64();
Expand All @@ -27,7 +27,7 @@ public void Write(AssetsFileWriter writer)
public int type;
public string assetPath;
public string originalAssetPath;
public void Read(AssetsFileReader reader)
public void Read(AssetsFileStatefulReader reader)
{
bufferedPath = reader.ReadNullTerminated();
guid = new GUID128();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ public class AssetsFileDependencyList
{
public int dependencyCount;
public List<AssetsFileDependency> dependencies;
public void Read(AssetsFileReader reader)
public void Read(AssetsFileStatefulReader reader)
{
dependencyCount = reader.ReadInt32();
dependencies = new List<AssetsFileDependency>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public int GetSizeBytes()
else
return 0x14;
}
public void Read(AssetsFileReader reader)
public void Read(AssetsFileStatefulReader reader)
{
metadataSize = reader.ReadUInt32();
fileSize = reader.ReadUInt32();
Expand Down
Loading