Repository navigation
Cobalt 100: ARM64 callee-save pairs crossing page and cache boundaries #665
Description
Investigate whether the Apple ARM64 callee-save pair boundary penalty also occurs on Azure Cobalt 100 / Neoverse N2. This is a hardware/reproduction measurement, not a PR comparison. Please verify that the returned machine is Cobalt 100; a macOS fallback does not answer this question.
The snippet has 28 benchmark cases:
- Native dependent STP/LDP versus scalar STR/LDR loops at 16-byte alignment, 8-byte alignment without crossing, a 32-byte crossing, a 64-byte crossing, and an actual OS-page crossing. Both pages are allocated and pre-touched. The exact AArch64 kernels are embedded so JIT instruction selection cannot remove the access being measured.
- Managed nine- versus eight-integer callee-save methods, with caller SP deliberately placed at offsets 0 through 128 (step 16) within the actual OS page. Variable-sized stack allocations establish SP, and the requested placement and return values are checked. Setup prints the addresses; JIT disassembly should confirm the nine-save misaligned pairs and the eight-save aligned control.
Use the unpatched .NET 10 runtime; no runtime source build or PR is needed. The current proposed fix is Apple-only, so a normal Linux PR comparison would be a no-op.
Local positive control on an unpatched Apple M4 CoreRun completed all 28 cases: native pair CrossPage was 14.26 ns versus 1.34 ns without a page crossing (scalar CrossPage 1.17 ns); managed Odd at SP offsets 16/32/48/64 was 16.9–18.1 ns versus 1.8–1.9 ns elsewhere, while Even stayed around 1.75–1.78 ns. The managed layout/return checks and expected nine-save prolog were also verified on installed .NET 10.0.3.
@EgorBot ubuntu24_azure_cobalt100 --runtimes net10.0 --filter "*" --launchCount 2 --warmupCount 3 --iterationCount 10 --iterationTime 300 --envvars DOTNET_TieredCompilation:0 DOTNET_JitDisasm:*Integers
using System;
using System.ComponentModel;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Running;
public static class Program
{
public static void Main(string[] args)
{
if (args.Length == 1 && (args[0] == "--validate" || args[0] == "--validate-managed"))
{
if (args[0] == "--validate")
{
foreach (PairBoundary boundary in Enum.GetValues<PairBoundary>())
{
var test = new NativePairs { Boundary = boundary };
test.Setup();
try
{
if (test.Pair() != 3 || test.Scalar() != 3)
{
throw new InvalidOperationException("Native kernel returned the wrong value.");
}
}
finally
{
test.Cleanup();
}
}
}
foreach (int offset in new[] { 0, 16, 32, 48, 64, 80, 96, 112, 128 })
{
var test = new ManagedPairs { EntrySpOffset = offset };
test.Setup();
if (test.Odd() != 45L * ManagedPairs.Calls || test.Even() != 36L * ManagedPairs.Calls)
{
throw new InvalidOperationException("Managed test returned the wrong value.");
}
}
Console.WriteLine("VALIDATION PASSED");
return;
}
BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args);
}
}
public enum PairBoundary
{
Aligned16,
Aligned8,
Cross32,
Cross64,
CrossPage
}
public unsafe class NativePairs
{
public const int Operations = 32768;
private byte* _data;
private byte* _address;
private void* _code;
private nuint _pageSize;
private delegate* unmanaged<byte*, nuint, ulong> _pair;
private delegate* unmanaged<byte*, nuint, ulong> _scalar;
[ParamsAllValues]
public PairBoundary Boundary { get; set; }
[GlobalSetup]
public void Setup()
{
if (RuntimeInformation.ProcessArchitecture != Architecture.Arm64 || !BitConverter.IsLittleEndian)
{
throw new PlatformNotSupportedException("This benchmark requires little-endian ARM64.");
}
_pageSize = (nuint)Environment.SystemPageSize;
_data = (byte*)NativeMemory.AlignedAlloc(3 * _pageSize, _pageSize);
if (_data is null)
{
throw new OutOfMemoryException();
}
NativeMemory.Clear(_data, 3 * _pageSize);
int offset = Boundary switch
{
PairBoundary.Aligned16 => -32,
PairBoundary.Aligned8 => -24,
PairBoundary.Cross32 => -40,
PairBoundary.Cross64 => -72,
PairBoundary.CrossPage => -8,
_ => throw new ArgumentOutOfRangeException(nameof(Boundary))
};
_address = _data + _pageSize + offset;
int anonymous = OperatingSystem.IsMacOS() ? 0x1000 : 0x20;
_code = Mmap(null, _pageSize, 3, 2 | anonymous, -1, 0);
if (_code == (void*)(-1))
{
throw new Win32Exception(Marshal.GetLastPInvokeError());
}
// x0 = address, x1 = iteration count; only caller-saved registers are changed.
ReadOnlySpan<uint> pair =
[
0xd2800029, // mov x9, #1
0xd280004a, // mov x10, #2
0xa9002809, // stp x9, x10, [x0]
0xa9402809, // ldp x9, x10, [x0]
0xf1000421, // subs x1, x1, #1
0x54ffffa1, // b.ne to stp
0x8b0a0120, // add x0, x9, x10
0xd65f03c0 // ret
];
ReadOnlySpan<uint> scalar =
[
0xd2800029, // mov x9, #1
0xd280004a, // mov x10, #2
0xf9000009, // str x9, [x0]
0xf900040a, // str x10, [x0, #8]
0xf9400009, // ldr x9, [x0]
0xf940040a, // ldr x10, [x0, #8]
0xf1000421, // subs x1, x1, #1
0x54ffff61, // b.ne to first str
0x8b0a0120, // add x0, x9, x10
0xd65f03c0 // ret
];
pair.CopyTo(new Span<uint>(_code, pair.Length));
scalar.CopyTo(new Span<uint>((byte*)_code + 128, scalar.Length));
if (Mprotect(_code, _pageSize, 5) != 0)
{
throw new Win32Exception(Marshal.GetLastPInvokeError());
}
if (OperatingSystem.IsMacOS())
{
InvalidateInstructionCache(_code, _pageSize);
}
else
{
ClearInstructionCache(_code, (byte*)_code + _pageSize);
}
_pair = (delegate* unmanaged<byte*, nuint, ulong>)_code;
_scalar = (delegate* unmanaged<byte*, nuint, ulong>)((byte*)_code + 128);
if (_pair(_address, 1) != 3 || _scalar(_address, 1) != 3)
{
throw new InvalidOperationException("Native kernel validation failed.");
}
Console.WriteLine($"NATIVE {Boundary}: page={_pageSize}, mod32={(nuint)_address % 32}, " +
$"mod64={(nuint)_address % 64}, modPage={(nuint)_address % _pageSize}");
}
[Benchmark(Baseline = true, OperationsPerInvoke = Operations)]
public ulong Pair() => _pair(_address, Operations);
[Benchmark(OperationsPerInvoke = Operations)]
public ulong Scalar() => _scalar(_address, Operations);
[GlobalCleanup]
public void Cleanup()
{
NativeMemory.AlignedFree(_data);
if (Munmap(_code, _pageSize) != 0)
{
throw new Win32Exception(Marshal.GetLastPInvokeError());
}
}
[DllImport("libc", EntryPoint = "mmap", SetLastError = true)]
private static extern void* Mmap(void* address, nuint length, int protection, int flags, int fd, nint offset);
[DllImport("libc", EntryPoint = "mprotect", SetLastError = true)]
private static extern int Mprotect(void* address, nuint length, int protection);
[DllImport("libc", EntryPoint = "munmap", SetLastError = true)]
private static extern int Munmap(void* address, nuint length);
[DllImport("libSystem.B.dylib", EntryPoint = "sys_icache_invalidate")]
private static extern void InvalidateInstructionCache(void* address, nuint length);
[DllImport("libgcc_s.so.1", EntryPoint = "__clear_cache")]
private static extern void ClearInstructionCache(void* start, void* end);
}
public class ManagedPairs
{
public const int Calls = 32768;
private static readonly long[] s_values = [1, 2, 3, 4, 5, 6, 7, 8, 9];
private static long s_sink;
[Params(0, 16, 32, 48, 64, 80, 96, 112, 128)]
public int EntrySpOffset { get; set; }
[GlobalSetup]
public void Setup()
{
long odd = Run(true, EntrySpOffset, 64, out nuint oddSp);
long even = Run(false, EntrySpOffset, 64, out nuint evenSp);
if (odd != 45 * 64 || even != 36 * 64)
{
throw new InvalidOperationException("Managed validation failed.");
}
Console.WriteLine($"MANAGED {RuntimeInformation.FrameworkDescription}, page={Environment.SystemPageSize}, " +
$"requestedSP={EntrySpOffset}, oddSP=0x{oddSp:x}, evenSP=0x{evenSp:x}");
}
[Benchmark(Baseline = true, OperationsPerInvoke = Calls)]
public long Odd() => Run(true, EntrySpOffset, Calls, out _);
[Benchmark(OperationsPerInvoke = Calls)]
public long Even() => Run(false, EntrySpOffset, Calls, out _);
[MethodImpl(MethodImplOptions.NoInlining)]
private static void Barrier() => Volatile.Write(ref s_sink, 0);
[MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
public static long NineIntegers(long[] values)
{
long a = values[0], b = values[1], c = values[2], d = values[3];
long e = values[4], f = values[5], g = values[6], h = values[7], i = values[8];
Barrier();
return a + b + c + d + e + f + g + h + i;
}
[MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
public static long EightIntegers(long[] values)
{
long a = values[0], b = values[1], c = values[2], d = values[3];
long e = values[4], f = values[5], g = values[6], h = values[7];
Barrier();
return a + b + c + d + e + f + g + h;
}
[MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.AggressiveOptimization)]
private static unsafe long Run(bool odd, int targetOffset, int count, out nuint stackPointer)
{
nuint pageSize = (nuint)Environment.SystemPageSize;
// A variable-size allocation gives the actual SP, rather than a fixed-frame local's address.
byte* anchor = stackalloc byte[16 + (count & 15)];
anchor[0] = 1;
int padding = (int)(((nuint)anchor - (nuint)targetOffset) & (pageSize - 1));
if (padding == 0)
{
padding = (int)pageSize;
}
byte* buffer = stackalloc byte[padding];
buffer[0] = 1;
stackPointer = (nuint)buffer;
if (stackPointer % pageSize != (nuint)targetOffset)
{
throw new InvalidOperationException("Unexpected localloc stack layout.");
}
long sum = 0;
if (odd)
{
for (int i = 0; i < count; i++)
{
sum += NineIntegers(s_values);
}
}
else
{
for (int i = 0; i < count; i++)
{
sum += EightIntegers(s_values);
}
}
Volatile.Write(ref s_sink, sum + anchor[0] + buffer[0]);
return sum;
}
}Note
Benchmark code and request prepared by GitHub Copilot and submitted at the request of @AndyAyersMS.
Reacted by EgorBot
Activity
EgorBot commentedon Nov 2, 2026
Benchmark job submitted. The results will be posted here once they are ready.
EgorBot commentedon Nov 2, 2026
ubuntu24_azure_cobalt100— Failed@AndyAyersMS
Agent failed with exit code 1
Job | Full logs
AndyAyersMS commentedon Nov 2, 2026
Retry the same validated benchmark on Cobalt 100, using EgorBot's default .NET 11 runtime only. The first attempt failed during restoration of unavailable .NET 10.0.15 reference/host packages, before any benchmark ran. Omitting
--runtimes net10.0avoids requesting that target framework. No runtime source build or PR comparison is required.The code is unchanged: 10 native paired/scalar boundary cases and 18 managed odd/even callee-save cases with checked SP placement. Please confirm actual Cobalt 100 / Neoverse-N2 hardware, OS page size, and the managed prolog disassembly before interpreting results.
@EgorBot ubuntu24_azure_cobalt100 --filter "*" --launchCount 2 --warmupCount 3 --iterationCount 10 --iterationTime 300 --envvars DOTNET_TieredCompilation:0 DOTNET_JitDisasm:*Integers
Note
Benchmark code and retry request prepared by GitHub Copilot; resubmission approved by @AndyAyersMS.
EgorBot commentedon Nov 2, 2026
Benchmark job submitted. The results will be posted here once they are ready.
EgorBot commentedon Nov 2, 2026
Results for
ubuntu24_azure_cobalt100@AndyAyersMS
Full logs
JIT: Align arm64 callee-save register pairs (#136941)