Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2939e5d

Browse files
authored
Merge pull request#4098 from erri120/task/4042-3
Unify hashing code
2 parents35d22e4 +dc06e5b commit2939e5d

File tree

33 files changed

+1023
-190
lines changed

33 files changed

+1023
-190
lines changed

‎Directory.Packages.props‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<PackageVersionInclude="CommunityToolkit.HighPerformance"Version="8.4.0" />
44
<PackageVersionInclude="Mutagen.Bethesda.Fallout4"Version="0.51.3" />
55
<PackageVersionInclude="SharpZipLib"Version="1.4.2" />
6-
<PackageVersionInclude="TUnit"Version="0.57.24" />
6+
<PackageVersionInclude="TUnit"Version="0.86.5" />
77
<PackageVersionInclude="Avalonia.AvaloniaEdit"Version="11.3.0" />
88
<PackageVersionInclude="Avalonia.Labs.Panels"Version="11.3.1" />
99
<PackageVersionInclude="Avalonia.Skia"Version="11.3.5" />
@@ -56,8 +56,8 @@
5656
<PackageVersionInclude="OpenTelemetry.Extensions.Hosting"Version="1.12.0" />
5757
<PackageVersionInclude="Polly.Core"Version="8.6.3" />
5858
<PackageVersionInclude="Polly"Version="8.6.3" />
59-
<PackageVersionInclude="Verify"Version="30.11.0" />
60-
<PackageVersionInclude="Verify.TUnit"Version="30.11.0" />
59+
<PackageVersionInclude="Verify"Version="31.4.1" />
60+
<PackageVersionInclude="Verify.TUnit"Version="31.4.1" />
6161
<PackageVersionInclude="YamlDotNet"Version="16.3.0" />
6262
<PackageVersionInclude="ZLinq"Version="1.5.2" />
6363
<PackageVersionInclude="ZstdSharp.Port"Version="0.8.6" />

‎src/NexusMods.Abstractions.GameLocators/LocationId.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ namespace NexusMods.Abstractions.GameLocators;
99
[ValueObject<ushort>]
1010
publicreadonlypartialstructLocationId
1111
{
12-
privatestaticreadonlyFNV1a16PoolHashPool=new(name:nameof(LocationId));
12+
privatestaticreadonlyStringHashPool<ushort,FNV1a16Hasher>HashPool=new(name:nameof(LocationId));
1313

1414
/// <summary>
1515
/// Converts the string to a LocationId.

‎src/NexusMods.Backend/Game/GameLocationsService.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
usingSystem.Collections.Concurrent;
22
usingSystem.Collections.Frozen;
3+
usingSystem.IO.Hashing;
34
usingDynamicData.Kernel;
45
usingMicrosoft.Extensions.DependencyInjection;
56
usingMicrosoft.Extensions.Logging;
@@ -139,6 +140,6 @@ private static async ValueTask<Optional<Hash>> GetExistingHash(IDb hashesDb, Abs
139140
privatestaticasyncValueTask<Hash>GetMinimalHash(AbsolutePathfile,CancellationTokencancellationToken)
140141
{
141142
awaitusingvarfileStream=file.Read();
142-
returnawaitMultiHasher.MinimalHash(fileStream,cancellationToken:cancellationToken);
143+
returnawaitMinimalHash.HashAsync<Hash,XxHash3,Xx3Hasher>(fileStream,cancellationToken:cancellationToken);
143144
}
144145
}

‎src/NexusMods.Collections/InstallCollectionDownloadJob.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -283,9 +283,7 @@ private FomodXmlInstaller GetFomodXmlInstaller(CancellationToken cancellationTok
283283
awaitParallel.ForEachAsync(libraryArchive.Children,async(child,token)=>
284284
{
285285
awaitusingvarstream=awaitFileStore.GetFileStream(child.AsLibraryFile().Hash,token);
286-
usingvarhasher=MD5.Create();
287-
varhash=awaithasher.ComputeHashAsync(stream,token);
288-
varmd5=Md5Value.From(hash);
286+
varmd5=awaitMd5Hasher.HashAsync(stream,cancellationToken:token);
289287

290288
varfile=child.AsLibraryFile();
291289
hashes[md5]=newHashMapping()

‎src/NexusMods.DataModel/Synchronizer/DbFunctions/FNV1aHashScalar.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public override void Execute(ReadOnlyChunk chunk, WritableVector vector)
3030
}
3131

3232
varelement=elements[(int)row];
33-
varhash=FNV1a.Hash16(element.GetSpan());
33+
varhash=FNV1a16Hasher.Hash(element.GetSpan());
3434
output[(int)row]=hash;
3535
outputMask[row]=true;
3636
}

‎src/NexusMods.Library/AddLibraryFileJob.cs‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
usingSystem.Collections.Concurrent;
2+
usingSystem.IO.Hashing;
23
usingSystem.Security.Cryptography;
34
usingDynamicData.Kernel;
45
usingMicrosoft.Extensions.DependencyInjection;
@@ -61,21 +62,29 @@ internal class AddLibraryFileJob : IJobDefinitionWithStart<AddLibraryFileJob, Li
6162
returntopFile;
6263
}
6364

64-
privateasyncTask<LibraryFile.New>AnalyzeFile(IJobContext<AddLibraryFileJob>context,AbsolutePathfilePath,boolisNestedFile=false)
65+
privatestaticreadonlyMultiHasherBuilder<Hash,XxHash3,Xx3Hasher,Md5Value,MD5,Md5Hasher>Hasher=MultiHasherBuilder.Start()
66+
.AddHasher<Hash,XxHash3,Xx3Hasher>()
67+
.AddHasher<Md5Value,MD5,Md5Hasher>();
68+
69+
privatestaticasyncValueTask<(Hash,Optional<Md5Value>)>HashAsync(AbsolutePathfilePath,boolisNestedFile,CancellationTokencancellationToken)
6570
{
66-
varisArchive=isNestedFile?awaitCheckIfNestedArchiveAsync(filePath):awaitCheckIfArchiveAsync(filePath);
67-
varhash=awaitfilePath.XxHash3Async(token:context.CancellationToken);
71+
awaitusingvarstream=filePath.Read();
6872

69-
// TODO: hash once with both algorithms
70-
varmd5=Optional<Md5Value>.None;
71-
if(!isNestedFile)
73+
if(isNestedFile)
7274
{
73-
awaitusingvarfileStream=filePath.Read();
74-
usingvaralgo=MD5.Create();
75-
varrawHash=awaitalgo.ComputeHashAsync(fileStream,cancellationToken:context.CancellationToken);
76-
md5=Md5Value.From(rawHash);
75+
varhash=awaitXx3Hasher.HashAsync(stream,cancellationToken:cancellationToken).ConfigureAwait(false);
76+
return(hash,Optional<Md5Value>.None);
7777
}
7878

79+
varmultiHash=awaitHasher.HashAsync(stream,cancellationToken:cancellationToken);
80+
return(multiHash.Hash1,multiHash.Hash2);
81+
}
82+
83+
privateasyncTask<LibraryFile.New>AnalyzeFile(IJobContext<AddLibraryFileJob>context,AbsolutePathfilePath,boolisNestedFile=false)
84+
{
85+
varisArchive=isNestedFile?awaitCheckIfNestedArchiveAsync(filePath):awaitCheckIfArchiveAsync(filePath);
86+
87+
var(hash,md5)=awaitHashAsync(filePath,isNestedFile:isNestedFile,cancellationToken:context.CancellationToken);
7988
varlibraryFile=CreateLibraryFile(Transaction,filePath,hash,md5);
8089

8190
if(isArchive)

‎src/NexusMods.Sdk/Hashes/Crc32Value.cs‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
usingSystem.IO.Hashing;
12
usingSystem.Text.Json.Serialization;
23
usingJetBrains.Annotations;
34
usingTransparentValueObjects;
@@ -11,3 +12,28 @@ namespace NexusMods.Sdk.Hashes;
1112
[ValueObject<uint>]
1213
[JsonConverter(typeof(Crc32JsonConverter))]
1314
publicreadonlypartialstructCrc32Value;
15+
16+
[PublicAPI]
17+
publicclass Crc32Hasher:IStreamingHasher<Crc32Value,Crc32,Crc32Hasher>
18+
{
19+
publicstaticCrc32ValueHash(ReadOnlySpan<byte>input)
20+
{
21+
varhash=Crc32.HashToUInt32(input);
22+
returnCrc32Value.From(hash);
23+
}
24+
25+
publicstaticCrc32Initialize()=>new();
26+
27+
publicstaticCrc32Update(Crc32state,ReadOnlySpan<byte>input)
28+
{
29+
state.Append(input);
30+
returnstate;
31+
}
32+
33+
publicstaticCrc32ValueFinish(Crc32state)=>Crc32Value.From(state.GetCurrentHashAsUInt32());
34+
35+
publicstaticValueTask<Crc32Value>HashAsync(Streamstream,intbufferSize=IStreamingHasher<Crc32Value,Crc32,Crc32Hasher>.DefaultBufferSize,CancellationTokencancellationToken=default)
36+
{
37+
returnStreamingHasher<Crc32Value,Crc32,Crc32Hasher>.HashAsync(stream,bufferSize,cancellationToken);
38+
}
39+
}

‎src/NexusMods.Sdk/Hashes/FNV1a.cs‎

Lines changed: 54 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -4,74 +4,93 @@
44
namespaceNexusMods.Sdk.Hashes;
55

66
/// <summary>
7-
///FNV Non-Cryptographic Hash Algorithm.
7+
///Hasher using 16-bit FNV1a algorithm.
88
/// </summary>
99
[PublicAPI]
1010
[SuppressMessage("ReSharper","InconsistentNaming")]
11-
publicstaticclassFNV1a
11+
publicclassFNV1a16Hasher:IStringHasher<ushort,FNV1a16Hasher>
12+
{
13+
publicstaticushortHash(ReadOnlySpan<byte>input)=>MixToShort(FNV1a32Hasher.Hash(input));
14+
publicstaticushortHash(ReadOnlySpan<char>input)=>MixToShort(FNV1a32Hasher.Hash(input));
15+
16+
privatestaticushortMixToShort(uinthash)
17+
{
18+
return(ushort)((hash>>16)^(hash&0xFFFF));
19+
}
20+
}
21+
22+
/// <summary>
23+
/// Hasher using 32-bit FNV1a algorithm.
24+
/// </summary>
25+
[PublicAPI]
26+
[SuppressMessage("ReSharper","InconsistentNaming")]
27+
publicclassFNV1a32Hasher:IStringHasher<uint,FNV1a32Hasher>
1228
{
1329
// https://datatracker.ietf.org/doc/html/draft-eastlake-fnv-35#name-fnv-constants
14-
privateconstuintPrime32=0x01000193;
15-
privateconstuintOffset32=0x811C9DC5;
30+
privateconstuintPrime=0x01000193;
31+
privateconstuintOffset=0x811C9DC5;
1632

17-
publicstaticuintHash32(ReadOnlySpan<char>input)
33+
publicstaticuintHash(ReadOnlySpan<byte>input)
1834
{
19-
varhash=Offset32;
35+
varhash=Offset;
2036
for(vari=0;i<input.Length;i++)
2137
{
2238
varcurrent=input[i];
2339
hash^=current;
24-
hash*=Prime32;
40+
hash*=Prime;
2541
}
2642

2743
returnhash;
2844
}
2945

30-
publicstaticuintHash32(ReadOnlySpan<byte>input)
46+
publicstaticuintHash(ReadOnlySpan<char>input)
3147
{
32-
varhash=Offset32;
48+
varhash=Offset;
3349
for(vari=0;i<input.Length;i++)
3450
{
3551
varcurrent=input[i];
3652
hash^=current;
37-
hash*=Prime32;
53+
hash*=Prime;
3854
}
3955

4056
returnhash;
4157
}
42-
43-
publicstaticushortHash16(ReadOnlySpan<char>input)=>MixToShort(Hash32(input));
44-
publicstaticushortHash16(ReadOnlySpan<byte>input)=>MixToShort(Hash32(input));
45-
46-
/// <summary>
47-
/// Mixes a 32-bit hash into a 16-bit unsigned short by XOR-ing the higher and lower 16 bits.
48-
/// </summary>
49-
privatestaticushortMixToShort(uinthash)
50-
{
51-
return(ushort)((hash>>16)^(hash&0xFFFF));
52-
}
5358
}
5459

5560
/// <summary>
56-
///String hash poolusing32-bit FNV1ahashes.
61+
///Hasherusing64-bit FNV1aalgorithm.
5762
/// </summary>
5863
[PublicAPI]
5964
[SuppressMessage("ReSharper","InconsistentNaming")]
60-
publicclassFNV1a32Pool:AStringHashPool<uint>
65+
publicclassFNV1a64Hasher:IStringHasher<ulong,FNV1a64Hasher>
6166
{
62-
publicFNV1a32Pool(stringname):base(name){}
67+
// https://datatracker.ietf.org/doc/html/draft-eastlake-fnv-35#name-fnv-constants
68+
privateconstulongPrime=0x00000100_000001B3;
69+
privateconstulongOffset=0xCBF29CE4_84222325;
6370

64-
protectedoverrideuintHash(stringinput)=>FNV1a.Hash32(input);
65-
}
71+
publicstaticulongHash(ReadOnlySpan<byte>input)
72+
{
73+
varhash=Offset;
74+
for(vari=0;i<input.Length;i++)
75+
{
76+
varcurrent=input[i];
77+
hash^=current;
78+
hash*=Prime;
79+
}
6680

67-
/// <summary>
68-
/// String hash pool using 16-bit FNV1a hashes.
69-
/// </summary>
70-
[PublicAPI]
71-
[SuppressMessage("ReSharper","InconsistentNaming")]
72-
publicclassFNV1a16Pool:AStringHashPool<ushort>
73-
{
74-
publicFNV1a16Pool(stringname):base(name){}
81+
returnhash;
82+
}
83+
84+
publicstaticulongHash(ReadOnlySpan<char>input)
85+
{
86+
varhash=Offset;
87+
for(vari=0;i<input.Length;i++)
88+
{
89+
varcurrent=input[i];
90+
hash^=current;
91+
hash*=Prime;
92+
}
7593

76-
protectedoverrideushortHash(stringinput)=>FNV1a.Hash16(input);
94+
returnhash;
95+
}
7796
}
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
usingJetBrains.Annotations;
2+
3+
namespaceNexusMods.Sdk.Hashes;
4+
5+
/// <summary>
6+
/// Represents a hash algorithm.
7+
/// </summary>
8+
[PublicAPI]
9+
publicinterfaceIHasher<outTHash,TSelf>
10+
whereTHash: unmanaged,IEquatable<THash>
11+
whereTSelf:IHasher<THash,TSelf>
12+
{
13+
/// <summary>
14+
/// Hashes the input.
15+
/// </summary>
16+
staticabstractTHashHash(ReadOnlySpan<byte>input);
17+
}
18+
19+
/// <summary>
20+
/// Represents a hash algorithm.
21+
/// </summary>
22+
[PublicAPI]
23+
publicinterfaceIStringHasher<outTHash,TSelf>:IHasher<THash,TSelf>
24+
whereTHash: unmanaged,IEquatable<THash>
25+
whereTSelf:IStringHasher<THash,TSelf>
26+
{
27+
/// <summary>
28+
/// Hashes the input.
29+
/// </summary>
30+
staticabstractTHashHash(ReadOnlySpan<char>input);
31+
}
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
usingJetBrains.Annotations;
2+
3+
namespaceNexusMods.Sdk.Hashes;
4+
5+
/// <summary>
6+
/// Represents a streaming hash algorithm.
7+
/// </summary>
8+
[PublicAPI]
9+
publicinterfaceIStreamingHasher<THash,TState,TSelf>:IHasher<THash,TSelf>
10+
whereTHash: unmanaged,IEquatable<THash>
11+
whereTSelf:IStreamingHasher<THash,TState,TSelf>
12+
{
13+
publicconstintDefaultBufferSize=1024*8;
14+
15+
/// <summary>
16+
/// Hashes the stream contents until the end.
17+
/// </summary>
18+
staticabstractValueTask<THash>HashAsync(Streamstream,intbufferSize=DefaultBufferSize,CancellationTokencancellationToken=default);
19+
20+
/// <summary>
21+
/// Initializes the streaming state.
22+
/// </summary>
23+
staticabstractTStateInitialize();
24+
25+
/// <summary>
26+
/// Updates the streaming state.
27+
/// </summary>
28+
staticabstractTStateUpdate(TStatestate,ReadOnlySpan<byte>input);
29+
30+
/// <summary>
31+
/// Updates the streaming state.
32+
/// </summary>
33+
staticvirtualTStateUpdate(TStatestate,byte[]input,intoffset,intcount)=>TSelf.Update(state,input.AsSpan(start:offset,length:count));
34+
35+
/// <summary>
36+
/// Finalizes the streaming state.
37+
/// </summary>
38+
staticabstractTHashFinish(TStatestate);
39+
}
40+
41+
[PublicAPI]
42+
publicstaticclassStreamingHasher<THash,TState,THasher>
43+
whereTHash: unmanaged,IEquatable<THash>
44+
whereTHasher:IStreamingHasher<THash,TState,THasher>
45+
{
46+
/// <inheritdoc cref="IStreamingHasher{THash,TState,TSelf}.HashAsync"/>
47+
publicstaticasyncValueTask<THash>HashAsync(
48+
Streamstream,
49+
intbufferSize,
50+
CancellationTokencancellationToken)
51+
{
52+
varbuffer=GC.AllocateUninitializedArray<byte>(bufferSize);
53+
54+
varstate=THasher.Initialize();
55+
while(!cancellationToken.IsCancellationRequested)
56+
{
57+
varbytesRead=awaitstream.ReadAsync(buffer,cancellationToken).ConfigureAwait(false);
58+
if(bytesRead==0)break;
59+
60+
state=THasher.Update(state,buffer,offset:0,count:bytesRead);
61+
}
62+
63+
cancellationToken.ThrowIfCancellationRequested();
64+
returnTHasher.Finish(state);
65+
}
66+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp