Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

[AI Evaluation] Add versioning to reporting and caching#6070

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
peterwald merged 10 commits intomainfrompewaldsc/versioning
Mar 13, 2025
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
Show all changes
10 commits
Select commitHold shift + click to select a range
1d038cd
[AI Evaluation] Add versioning to reporting and caching
peterwaldMar 10, 2025
f3bbe08
Move CacheVersion key down into the DistributedCachingChatClient
peterwaldMar 10, 2025
3a06929
Update src/Libraries/Microsoft.Extensions.AI/ChatCompletion/Distribut…
stephentoubMar 11, 2025
f332da2
Update src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSh…
peterwaldMar 11, 2025
0883cc0
Update src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSh…
peterwaldMar 11, 2025
3923045
Rollback formatVersion init change
peterwaldMar 11, 2025
7739a01
Merge branch 'main' into pewaldsc/versioning
peterwaldMar 12, 2025
4e74bf3
Make version const internal, update comments.
peterwaldMar 13, 2025
61617ff
Add assertion for report version
peterwaldMar 13, 2025
30fb676
Make ReportVersion property readonly
peterwaldMar 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,4 +30,10 @@ public static class Defaults
/// in the <see cref="IResponseCacheProvider"/>'s cache before they are considered expired and evicted.
/// </summary>
public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14);

/// <summary>
/// Defines the version number for the reporting format. If and when the serialized format undergoes
/// breaking changes, this number will be incremented.
/// </summary>
internal const int ReportingFormatVersion = 1;
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -37,6 +37,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting;
/// The <see cref="Evaluation.EvaluationResult"/> for the <see cref="ScenarioRun"/> corresponding to the
/// <see cref="ScenarioRunResult"/> being constructed.
/// </param>
/// <param name="formatVersion">The version of the format used to persist the current <see cref="ScenarioRunResult"/>.</param>
[method: JsonConstructor]
public sealed class ScenarioRunResult(
string scenarioName,
Expand All@@ -45,7 +46,8 @@ public sealed class ScenarioRunResult(
DateTime creationTime,
IList<ChatMessage> messages,
ChatResponse modelResponse,
EvaluationResult evaluationResult)
EvaluationResult evaluationResult,
int? formatVersion = null)
{
/// <summary>
/// Initializes a new instance of the <see cref="ScenarioRunResult"/> class.
Expand DownExpand Up@@ -81,6 +83,11 @@ public ScenarioRunResult(
{
}

/// <summary>
/// Gets the version of the format used to persist the current <see cref="ScenarioRunResult"/>.
/// </summary>
public int? FormatVersion { get; } = formatVersion ?? Defaults.ReportingFormatVersion;

/// <summary>
/// Gets or sets the <see cref="ScenarioRun.ScenarioName"/>.
/// </summary>
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -107,6 +107,11 @@ protected override async Task WriteCacheStreamingAsync(string key, IReadOnlyList
/// The generated cache key is not guaranteed to be stable across releases of the library.
/// </para>
/// </remarks>
protected override string GetCacheKey(params ReadOnlySpan<object?> values) =>
AIJsonUtilities.HashDataToString(values, _jsonSerializerOptions);
protected override string GetCacheKey(params ReadOnlySpan<object?> values)
{
// Bump the cache version to invalidate existing caches if the serialization format changes in a breaking way.
const int CacheVersion = 1;

return AIJsonUtilities.HashDataToString([CacheVersion, .. values], _jsonSerializerOptions);
}
}
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ public void SerializeScenarioRunResult()
messages: [new ChatMessage(ChatRole.User, "prompt")],
modelResponse: new ChatResponse(new ChatMessage(ChatRole.Assistant, "response")),
evaluationResult: new EvaluationResult(booleanMetric, numericMetric, stringMetric, metricWithNoValue));
Assert.Equal(Defaults.ReportingFormatVersion, entry.FormatVersion);

string json = JsonSerializer.Serialize(entry, SerializerContext.Default.ScenarioRunResult);
ScenarioRunResult? deserialized = JsonSerializer.Deserialize<ScenarioRunResult>(json, SerializerContext.Default.ScenarioRunResult);
Expand All@@ -49,6 +50,7 @@ public void SerializeScenarioRunResult()
Assert.Equal(entry.CreationTime, deserialized.CreationTime);
Assert.True(entry.Messages.SequenceEqual(deserialized.Messages, ChatMessageComparer.Instance));
Assert.Equal(entry.ModelResponse, deserialized.ModelResponse, ChatResponseComparer.Instance);
Assert.Equal(entry.FormatVersion, deserialized.FormatVersion);

ValidateEquivalence(entry.EvaluationResult, deserialized.EvaluationResult);
}
Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp