Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
Expand Down Expand Up @@ -79,8 +80,8 @@ public ValueTask<EvaluationResult> EvaluateAsync(

var (score, duration) = TimingHelper.ExecuteWithTiming(() =>
{
var references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference));
var hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text);
var references = context.References.Select(reference => SimpleWordTokenizer.WordTokenize(reference).ToArray()).ToArray();
var hypothesis = SimpleWordTokenizer.WordTokenize(modelResponse.Text).ToArray();
return BLEUAlgorithm.SentenceBLEU(references, hypothesis, BLEUAlgorithm.DefaultBLEUWeights, SmoothingFunction.Method4);
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public sealed class BLEUEvaluatorContext : EvaluationContext
/// <param name="references">
/// The reference responses against which the response that is being evaluated is compared.
/// </param>
public BLEUEvaluatorContext(params string[] references)
: this(references as IEnumerable<string>)
public BLEUEvaluatorContext(IEnumerable<string> references)
: this(references.ToArray())
{
}

Expand All @@ -52,11 +52,12 @@ public BLEUEvaluatorContext(params string[] references)
/// <param name="references">
/// The reference responses against which the response that is being evaluated is compared.
/// </param>
public BLEUEvaluatorContext(IEnumerable<string> references)
public BLEUEvaluatorContext(params string[] references)
: base(
name: BLEUContextName,
contents: [.. references.Select(c => new TextContent(c))])
{
References = [.. references];
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ namespace Microsoft.Extensions.AI.Evaluation.NLP.Common;
/// </summary>
internal static class BLEUAlgorithm
{
internal static int ClosestRefLength(IEnumerable<IEnumerable<string>> references, int hypLength)
internal static int ClosestRefLength(string[][] references, int hypLength)
{
if (!references.Any())
{
Expand All @@ -27,7 +27,7 @@ internal static int ClosestRefLength(IEnumerable<IEnumerable<string>> references
int smallestDiff = int.MaxValue;
foreach (var reference in references)
{
int refLength = reference.Count();
int refLength = reference.Length;
int diff = Math.Abs(refLength - hypLength);
if (diff < smallestDiff ||
(diff == smallestDiff && refLength < closestRefLength))
Expand Down Expand Up @@ -55,26 +55,26 @@ internal static double BrevityPenalty(int closestRefLength, int hypLength)
return Math.Exp(1 - ((double)closestRefLength / hypLength));
}

internal static RationalNumber ModifiedPrecision(IEnumerable<IEnumerable<string>> references, IEnumerable<string> hypothesis, int n = 1)
internal static RationalNumber ModifiedPrecision(string[][] references, string[] hypothesis, int n = 1)
{
if (n <= 0)
{
Throw.ArgumentOutOfRangeException(nameof(n), $"`{nameof(n)}` must be greater than zero.");
}

if (!references.Any() || !hypothesis.Any())
if (references.Length == 0 || hypothesis.Length == 0)
{
return RationalNumber.Zero;
}

var hyp = hypothesis.CreateNGrams(n);
var hypCounts = new MatchCounter<NGram<string>>(hyp);
var hypGrams = hypothesis.AsSpan().CreateNGrams(n);
var hypCounts = new MatchCounter<NGram<string>>(hypGrams);

Dictionary<NGram<string>, int> maxCounts = [];

foreach (var rf in references)
{
IEnumerable<NGram<string>> refGrams = rf.CreateNGrams(n);
IEnumerable<NGram<string>> refGrams = rf.AsSpan().CreateNGrams(n);
var refCounts = new MatchCounter<NGram<string>>(refGrams);

foreach (var ct in refCounts)
Expand Down Expand Up @@ -123,25 +123,28 @@ internal static double[] EqualWeights(int n)
}

double[] weights = new double[n];
#if NET8_0_OR_GREATER
Array.Fill(weights, 1.0 / n);
#else
for (int i = 0; i < n; i++)
{
weights[i] = 1.0 / n;
}

#endif
return weights;
}

internal static readonly double[] DefaultBLEUWeights = EqualWeights(4);

internal static double SentenceBLEU(IEnumerable<IEnumerable<string>> references, IEnumerable<string> hypothesis,
internal static double SentenceBLEU(string[][] references, string[] hypothesis,
double[]? weights = null, Func<RationalNumber[], int, double[]>? smoothingFunction = null)
{
if (references == null || !references.Any())
if (references == null || references.Length == 0)
{
Throw.ArgumentNullException(nameof(references), $"'{nameof(references)}' cannot be null or empty.");
}

if (hypothesis == null || !hypothesis.Any())
if (hypothesis == null || hypothesis.Length == 0)
{
Throw.ArgumentNullException(nameof(hypothesis), $"'{nameof(hypothesis)}' cannot be null or empty.");
}
Expand Down Expand Up @@ -171,7 +174,7 @@ internal static double SentenceBLEU(IEnumerable<IEnumerable<string>> references,
precisionValues[i] = prec;
}

int hypLen = hypothesis.Count();
int hypLen = hypothesis.Length;
int closestRefLength = ClosestRefLength(references, hypLen);
double brevityPenalty = BrevityPenalty(closestRefLength, hypLen);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ public void AddRange(IEnumerable<T> items)
}
}

public string ToDebugString() => string.Concat(_counts.Select(v => $"{v.Key}: {v.Value}, "));
public string ToDebugString() => string.Join(",", _counts.Select(v => $"{v.Key}: {v.Value}"));

public IEnumerator<KeyValuePair<T, int>> GetEnumerator() => _counts.GetEnumerator();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

using System;
using System.Collections.Generic;
using System.Linq;
using Microsoft.Shared.Diagnostics;

namespace Microsoft.Extensions.AI.Evaluation.NLP.Common;
Expand All @@ -14,28 +13,36 @@ internal static class NGramExtensions
public static NGram<T> CreateNGram<T>(this ReadOnlySpan<T> values)
where T : IEquatable<T> => new(values);

internal static IEnumerable<NGram<T>> CreateNGrams<T>(this Span<T> input, int n)
where T : IEquatable<T>
=> CreateNGrams((ReadOnlySpan<T>)input, n);

/// <summary>
/// Create a sequence of n-grams from the input sequence.
/// </summary>
/// <param name="input">The input sequence of items.</param>
/// <param name="n">The size of each n-gram.</param>
internal static IEnumerable<NGram<T>> CreateNGrams<T>(this IEnumerable<T> input, int n)
internal static List<NGram<T>> CreateNGrams<T>(this ReadOnlySpan<T> input, int n)
where T : IEquatable<T>
{
if (n <= 0)
{
Throw.ArgumentOutOfRangeException(nameof(n), $"'{nameof(n)}' must be greater than zero.");
}

T[] output = [.. input.Take(n)];
List<NGram<T>> nGrams = [];

ReadOnlySpan<T> output = input.Slice(0, Math.Min(n, input.Length));

while (output.Length == n)
{
yield return new NGram<T>(output);
nGrams.Add(new NGram<T>(output));

input = input.Skip(1);
output = [.. input.Take(n)];
input = input.Slice(1);
output = input.Slice(0, Math.Min(n, input.Length));
}

return nGrams;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

<PropertyGroup>
<InjectCollectionBuilderAttributesOnLegacy>true</InjectCollectionBuilderAttributesOnLegacy>
<InjectSystemIndexOnLegacy>true</InjectSystemIndexOnLegacy>
</PropertyGroup>

<ItemGroup>
Expand Down
Loading
Loading