Skip to content

6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 2 commits into from
Sep 16, 2023
Merged

6 more naive methods for Tensor Primitives. #92142

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
651d230
6 more naive methods
michaelgsharp Sep 15, 2023
75769c4
updates from pr comments
michaelgsharp Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 27 additions & 21 deletions src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
public static class TensorPrimitives
public static partial class TensorPrimitives
{
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
}
}
5 changes: 4 additions & 1 deletion src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
<data name="Argument_DestinationTooShort" xml:space="preserve">
<value>Destination is too short.</value>
</data>
<data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
<value>Input span arguments must not be empty.</value>
</data>
<data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
<value>Input span arguments must all have the same length.</value>
</data>
</root>
</root>
157 changes: 157 additions & 0 deletions src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
destination[i] = MathF.Tanh(x[i]);
}
}

/// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}
if (x.Length == 0 || y.Length == 0)

@tannergoodingtannergoodingSep 18, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.

That being said, is it desirable to throw here rather than allowing it to just return NaN?

{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float dotprod = 0f;
float magx = 0f;
float magy = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
magx += x[i] * x[i];
magy += y[i] * y[i];
}

return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.
}

/// <summary>
/// Compute the distance between two points in Euclidean space.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The Euclidean distance.</returns>
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}
if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float distance = 0f;

for (int i = 0; i < x.Length; i++)
{
float dist = x[i] - y[i];
distance += dist * dist;
}

return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.
}

/// <summary>
/// A mathematical operation that takes two vectors and returns a scalar.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="y">The second tensor, represented as a span.</param>
/// <returns>The dot product.</returns>
/// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
{
if (x.Length != y.Length)
{
ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
}

float dotprod = 0f;

for (int i = 0; i < x.Length; i++)
{
dotprod += x[i] * y[i];
}

return dotprod;
Comment thread
stephentoub marked this conversation as resolved.
}

/// <summary>
/// A mathematical operation that takes a vector and returns the L2 norm.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <returns>The L2 norm.</returns>
public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2

@tannergoodingtannergoodingSep 18, 2023•
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.

The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())

{
float magx = 0f;

for (int i = 0; i < x.Length; i++)
{
magx += x[i] * x[i];
}

return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.
}

/// <summary>
/// A function that takes a collection of real numbers and returns a probability distribution.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
/// <exception cref="ArgumentException">Destination is too short.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}
if (x.Length == 0)
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

float expSum = 0f;

for (int i = 0; i < x.Length; i++)
{
expSum += MathF.Pow((float)Math.E, x[i]);

@tannergoodingtannergoodingSep 18, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?

Why (float)Math.E and not MathF.E?

@stephentoubstephentoubSep 18, 2023

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.

}

for (int i = 0; i < x.Length; i++)
{
destination[i] = MathF.Exp(x[i]) / expSum;
}
}

/// <summary>
/// A function that takes a real number and returns a value between 0 and 1.
/// </summary>
/// <param name="x">The first tensor, represented as a span.</param>
/// <param name="destination">The destination tensor.</param>
/// <exception cref="ArgumentException">Destination is too short.</exception>
/// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
{
if (x.Length > destination.Length)
{
ThrowHelper.ThrowArgument_DestinationTooShort();
}
if (x.Length == 0)
{
ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
}

for (int i = 0; i < x.Length; i++)
{
destination[i] = 1f / (1 + MathF.Exp(-x[i]));
}
}
}
}
4 changes: 4 additions & 0 deletions src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
[DoesNotReturn]
public static void ThrowArgument_SpansMustHaveSameLength() =>
throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

[DoesNotReturn]
public static void ThrowArgument_SpansMustBeNonEmpty() =>
throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
}
}
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all 
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Force GitHub README to respect dark mode
(function() {
var style = document.createElement('style');
style.textContent = '
.markdown-body {
color-scheme: dark light;
}
.markdown-body pre { background: #161b22 !important; }
.markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }
.markdown-body table th, .markdown-body table td { border-color: #30363d !important; }
.markdown-body img { background: #0d1117; }
.markdown-body blockquote { border-left-color: #8b949e; }
.markdown-body hr { border-color: #30363d; }
';
document.head.appendChild(style);
})();
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Highlight search terms from Google/DuckDuckGo/Bing referrer
(function() {
var ref = document.referrer;
var terms = [];
if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {
var url = new URL(ref);
var q = url.searchParams.get('q') || url.searchParams.get('p');
if (q) {
terms = q.split(/\s+/).filter(function(t) { return t.length > 2; });
}
}
if (terms.length === 0) return;
var style = document.createElement('style');
style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';
document.head.appendChild(style);
function highlight(node) {
if (node.nodeType === 3) { // text node
var text = node.textContent;
var found = false;
terms.forEach(function(term) {
var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi');
if (regex.test(text)) {
found = true;
var frag = document.createDocumentFragment();
var parts = text.split(regex);
parts.forEach(function(part, i) {
if (i % 2 === 0) {
frag.appendChild(document.createTextNode(part));
} else {
var span = document.createElement('span');
span.className = 'userscript-highlight';
span.textContent = part;
frag.appendChild(span);
}
});
node.parentNode.replaceChild(frag, node);
}
});
} else if (node.nodeType === 1 && node.childNodes) { // element
var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];
if (!skipTags.includes(node.tagName)) {
Array.from(node.childNodes).forEach(highlight);
}
}
}
highlight(document.body);
// Re-highlight on dynamic content
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1 || node.nodeType === 3) highlight(node);
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Strip utm_, fbclid, gclid, etc. from all links on page
(function() {
var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',
'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',
'ref', 'ref_src', 'source', 'medium', 'campaign'];
function cleanUrl(url) {
try {
var u = new URL(url, window.location.origin);
var changed = false;
trackingParams.forEach(function(p) {
if (u.searchParams.has(p)) {
u.searchParams.delete(p);
changed = true;
}
});
return changed ? u.toString() : url;
} catch (e) {
return url;
}
}
function cleanLinks() {
document.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
cleanLinks();
var observer = new MutationObserver(function(mutations) {
mutations.forEach(function(m) {
m.addedNodes.forEach(function(node) {
if (node.nodeType === 1) {
if (node.tagName === 'A') cleanLinks();
node.querySelectorAll('a[href]').forEach(function(a) {
var clean = cleanUrl(a.href);
if (clean !== a.href) a.href = clean;
});
}
});
});
});
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Auto-enable theater mode on YouTube
(function() {
function tryTheater() {
var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]');
if (btn && !btn.classList.contains('activated')) {
btn.click();
}
}
// Try immediately
tryTheater();
// Try after navigation (SPA)
var lastUrl = location.href;
setInterval(function() {
if (location.href !== lastUrl) {
lastUrl = location.href;
setTimeout(tryTheater, 500);
}
}, 1000);
// Also try on player load
var observer = new MutationObserver(tryTheater);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Remove or un-stick sticky/fixed headers that block content
(function() {
function unstick() {
document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) {
if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {
el.style.position = 'static';
el.style.top = 'auto';
el.style.zIndex = 'auto';
}
});
}
unstick();
var observer = new MutationObserver(unstick);
observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });
})();
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading

, 'i');
if (__m === '*' || __re.test(location.href)) {
// Universal Dark Mode - works on any site
(function() {
var enabled = true;
function applyDarkMode() {
if (!enabled) return;
// Create style element if it doesn't exist
var style = document.getElementById('universal-dark-mode-style');
if (!style) {
style = document.createElement('style');
style.id = 'universal-dark-mode-style';
document.head.appendChild(style);
}
// Dark mode CSS - inverts colors but preserves images/video
style.textContent = '
/* Invert everything except media */
html {
filter: invert(1) hue-rotate(180deg) !important;
background: #1a1a2e !important;
}
/* Restore images, videos, iframes, canvas */
img, video, iframe, canvas, svg, picture, [style*="background-image"] {
filter: invert(1) hue-rotate(180deg) !important;
}
/* Preserve specific elements that should not be inverted */
.no-dark-mode, .no-dark-mode *,
[data-theme="light"], [data-theme="light"],
.ace_editor, .ace_editor *,
.CodeMirror, .CodeMirror *,
.monaco-editor, .monaco-editor *,
.markdown-body pre, .markdown-body pre *,
.highlight, .highlight *,
pre code, pre code * {
filter: none !important;
}
/* Fix common UI elements */
.modal, .popup, .dropdown-menu, .tooltip, .popover {
filter: invert(1) hue-rotate(180deg) !important;
background: #2d2d44 !important;
border-color: #444 !important;
}
/* Scrollbars */
::-webkit-scrollbar { background: #1a1a2e !important; }
::-webkit-scrollbar-thumb { background: #444 !important; }
::-webkit-scrollbar-thumb:hover { background: #555 !important; }
/* Selection */
::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }
';
}
function removeDarkMode() {
var style = document.getElementById('universal-dark-mode-style');
if (style) style.remove();
}
// Toggle with Alt+Shift+D
document.addEventListener('keydown', function(e) {
if (e.altKey && e.shiftKey && e.key === 'D') {
e.preventDefault();
enabled = !enabled;
if (enabled) {
applyDarkMode();
console.log('[Universal Dark Mode] Enabled');
} else {
removeDarkMode();
console.log('[Universal Dark Mode] Disabled');
}
}
});
// Apply on load
applyDarkMode();
// Re-apply on dynamic content
var observer = new MutationObserver(function(mutations) {
if (enabled && !document.getElementById('universal-dark-mode-style')) {
applyDarkMode();
}
});
observer.observe(document.head, { childList: true });
console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');
})();
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();
6 more naive methods for Tensor Primitives. by michaelgsharp · Pull Request #92142 · dotnet/runtime · GitHub
Skip to content
6 more naive methods for Tensor Primitives.#92142

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

stephentoub
merged 2 commits into

from

Sep 16, 2023
 Merged


6 more naive methods for Tensor Primitives.
#92142
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


2 commits


Select commit
Hold shift + click to select a range
651d230

6 more naive methods


michaelgsharp Sep 15, 2023
75769c4

updates from pr comments


michaelgsharp Sep 15, 2023

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


48 changes: 27 additions & 21 deletions
src/libraries/System.Numerics.Tensors/ref/System.Numerics.Tensors.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -6,27 +6,33 @@

namespace System.Numerics.Tensors
{
 public static class TensorPrimitives
 public static partial class TensorPrimitives
 {
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { throw null; }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { throw null; }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { throw null; }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { throw null; }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { throw null; }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { throw null; }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { throw null; }
 public static void Add(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Add(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float multiplier, System.Span<float> destination) { }
 public static void AddMultiply(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> multiplier, System.Span<float> destination) { }
 public static void Cosh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float CosineSimilarity(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static float Distance(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Divide(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Divide(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static float Dot(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y) { throw null; }
 public static void Exp(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static float L2Normalize(System.ReadOnlySpan<float> x) { throw null; }
 public static void Log(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Multiply(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, float addend, System.Span<float> destination) { }
 public static void MultiplyAdd(System.ReadOnlySpan<float> x, float y, System.ReadOnlySpan<float> addend, System.Span<float> destination) { }
 public static void Negate(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sigmoid(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Sinh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void SoftMax(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, System.ReadOnlySpan<float> y, System.Span<float> destination) { }
 public static void Subtract(System.ReadOnlySpan<float> x, float y, System.Span<float> destination) { }
 public static void Tanh(System.ReadOnlySpan<float> x, System.Span<float> destination) { }
 }
}

5 changes: 4 additions & 1 deletion
src/libraries/System.Numerics.Tensors/src/Resources/Strings.resx

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -120,7 +120,10 @@
 <data name="Argument_DestinationTooShort" xml:space="preserve">
 <value>Destination is too short.</value>
 </data>
 <data name="Argument_SpansMustBeNonEmpty" xml:space="preserve">
 <value>Input span arguments must not be empty.</value>
 </data>
 <data name="Argument_SpansMustHaveSameLength" xml:space="preserve">
 <value>Input span arguments must all have the same length.</value>
 </data>
</root>
</root>

157 changes: 157 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -253,5 +253,162 @@ public static void Tanh(ReadOnlySpan<float> x, Span<float> destination)
 destination[i] = MathF.Tanh(x[i]);
 }
 }

 /// <summary>Computes the cosine similarity between two non-zero vectors.</summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The cosine similarity between the two vectors.</returns>
Comment thread
michaelgsharp marked this conversation as resolved.

 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float CosineSimilarity(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We only need one of the checks here since we've already determined the lengths are equal.
That being said, is it desirable to throw here rather than allowing it to just return NaN?
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float dotprod = 0f;
 float magx = 0f;
 float magy = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 magx += x[i] * x[i];
 magy += y[i] * y[i];
 }

 return dotprod / (MathF.Sqrt(magx) * MathF.Sqrt(magy));
Comment thread
jeffhandley marked this conversation as resolved.

 }

 /// <summary>
 /// Compute the distance between two points in Euclidean space.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The Euclidean distance.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' and '<paramref name="y" />' must not be empty.</exception>
 public static float Distance(ReadOnlySpan<float> x, ReadOnlySpan<float> y)
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }
 if (x.Length == 0 || y.Length == 0)
Comment thread
tannergooding marked this conversation as resolved.

 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float distance = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 float dist = x[i] - y[i];
 distance += dist * dist;
 }

 return MathF.Sqrt(distance);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes two vectors and returns a scalar.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="y">The second tensor, represented as a span.</param>
 /// <returns>The dot product.</returns>
 /// <exception cref="ArgumentException">Length of '<paramref name="x" />' must be same as length of '<paramref name="y" />'.</exception>
 public static float Dot(ReadOnlySpan<float> x, ReadOnlySpan<float> y) // BLAS1: dot
 {
 if (x.Length != y.Length)
 {
 ThrowHelper.ThrowArgument_SpansMustHaveSameLength();
 }

 float dotprod = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 dotprod += x[i] * y[i];
 }

 return dotprod;
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A mathematical operation that takes a vector and returns the L2 norm.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <returns>The L2 norm.</returns>
 public static float L2Normalize(ReadOnlySpan<float> x) // BLAS1: nrm2
@tannergoodingtannergoodingSep 18, 2023•

edited

Loading

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why is this L2Normalize? The standard API computes the "euclidean norm", that is the length or magnitude of the vector.
The name should likely just be Norm (or Length which is what it's called for Vector2/3/4.Length())
 {
 float magx = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 magx += x[i] * x[i];
 }

 return MathF.Sqrt(magx);
Comment thread
stephentoub marked this conversation as resolved.

 }

 /// <summary>
 /// A function that takes a collection of real numbers and returns a probability distribution.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void SoftMax(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 float expSum = 0f;

 for (int i = 0; i < x.Length; i++)
 {
 expSum += MathF.Pow((float)Math.E, x[i]);
@tannergoodingtannergoodingSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why MathF.Pow and not MathF.Exp?
Why (float)Math.E and not MathF.E?
@stephentoubstephentoubSep 18, 2023

Copy link


Copy Markdown

Member
Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding, can you submit a cleanup PR with the changes you think should be made? Thanks.
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = MathF.Exp(x[i]) / expSum;
 }
 }

 /// <summary>
 /// A function that takes a real number and returns a value between 0 and 1.
 /// </summary>
 /// <param name="x">The first tensor, represented as a span.</param>
 /// <param name="destination">The destination tensor.</param>
 /// <exception cref="ArgumentException">Destination is too short.</exception>
 /// <exception cref="ArgumentException">'<paramref name="x" />' must not be empty.</exception>
 public static void Sigmoid(ReadOnlySpan<float> x, Span<float> destination)
 {
 if (x.Length > destination.Length)
 {
 ThrowHelper.ThrowArgument_DestinationTooShort();
 }
 if (x.Length == 0)
 {
 ThrowHelper.ThrowArgument_SpansMustBeNonEmpty();
 }

 for (int i = 0; i < x.Length; i++)
 {
 destination[i] = 1f / (1 + MathF.Exp(-x[i]));
 }
 }
 }
}

4 changes: 4 additions & 0 deletions
src/libraries/System.Numerics.Tensors/src/System/ThrowHelper.cs

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -14,5 +14,9 @@ public static void ThrowArgument_DestinationTooShort() =>
 [DoesNotReturn]
 public static void ThrowArgument_SpansMustHaveSameLength() =>
 throw new ArgumentException(SR.Argument_SpansMustHaveSameLength);

 [DoesNotReturn]
 public static void ThrowArgument_SpansMustBeNonEmpty() =>
 throw new ArgumentException(SR.Argument_SpansMustBeNonEmpty);
 }
}
Loading