Visitar URL original
harden storage transfers and release 10.1.11 · managedcode/Storage@be5d88b · GitHub
Skip to content

Commit be5d88b

Browse files
KSemenenkoKSemenenko
committed
harden storage transfers and release 10.1.11
1 parent 63e0614 commit be5d88b

94 files changed

Lines changed: 2129 additions & 662 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎AGENTS.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ This file defines how AI agents work in this solution.
5151
## ManagedCode Dependency Ownership And Repair
5252

5353
- `ManagedCode.*` projects and packages are maintained by us. Locate a defective dependency's owning repository alongside this checkout, read its `AGENTS.md`, and repair it there instead of replacing it, copying its implementation, or hiding the defect in a consumer workaround.
54+
- Owner correction 2026-10-13: batch all known confirmed defects in the same owned library/repository within the authorized task before releasing. Inspect the affected scope, collect and fix the defects together, add their focused regressions, self-review the complete change, and run the required checks against the combined batch. One defect may produce one release; several defects in that repository MUST share one release, not a version/release per fix.
55+
- Increment the version once through the canonical mechanism only when the complete repair batch is ready. Push release-triggering changes and publish once per owning repository for that batch; do not trigger intermediate releases while other known fixes are pending. Verify successful GitHub publication and actual NuGet availability, then update each affected consumer once to that published version and run its focused regressions. Preserve unrelated work and all existing correctness, delivery and repository-protection gates.
5456
- Deliver dependency repairs with focused regression tests, required repository checks, a canonical patch-version increment, a scoped commit, and a push to GitHub. These scoped version, commit, push, and release-trigger actions have standing user authorization; preserve unrelated work and never force-push or bypass protections.
5557
- Follow the owning repository's GitHub release workflow to success and verify the new version on the intended NuGet feed before updating a consumer reference and running consumer regressions. Diagnose publication failures in the owning repository; a pushed commit or green build alone is not delivery.
5658
- Do not claim delivery with unpublished local packages or temporary project references. Report the exact source, credentials, permissions, or release-infrastructure blocker when delivery cannot proceed.

‎Directory.Build.props‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
<RepositoryUrl>https://github.com/managedcode/Storage</RepositoryUrl>
3030
<PackageProjectUrl>https://github.com/managedcode/Storage</PackageProjectUrl>
3131
<Product>Managed Code - Storage</Product>
32-
<Version>10.1.10</Version>
33-
<PackageVersion>10.1.10</PackageVersion>
32+
<Version>10.1.11</Version>
33+
<PackageVersion>10.1.11</PackageVersion>
3434

3535
</PropertyGroup>
3636

‎Directory.Packages.props‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
44
</PropertyGroup>
55
<ItemGroup>
6-
<PackageVersion Include="AWSSDK.S3" Version="4.0.104.7" />
6+
<PackageVersion Include="AWSSDK.S3" Version="4.0.104.8" />
77
<PackageVersion Include="Azure.Identity" Version="1.21.0" />
88
<PackageVersion Include="Azure.Storage.Blobs" Version="12.30.1" />
99
<PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.28.0" />
1010
<PackageVersion Include="Cartograph.Catalog" Version="0.1.0-alpha" />
1111
<PackageVersion Include="coverlet.msbuild" Version="10.1.0" />
12-
<PackageVersion Include="CsvHelper" Version="33.1.0" />
12+
<PackageVersion Include="CsvHelper" Version="34.0.0" />
1313
<PackageVersion Include="DotNet.ReproducibleBuilds" Version="2.0.5" />
1414
<PackageVersion Include="Dropbox.Api" Version="7.4.0" />
1515
<PackageVersion Include="Google.Api.Gax" Version="4.15.2" />
@@ -18,8 +18,8 @@
1818
<PackageVersion Include="Google.Apis.Storage.v1" Version="1.77.0.4299" />
1919
<PackageVersion Include="Google.Cloud.Storage.V1" Version="5.0.0" />
2020
<PackageVersion Include="Humanizer.Core" Version="3.0.10" />
21-
<PackageVersion Include="ManagedCode.Communication" Version="10.3.20" />
22-
<PackageVersion Include="ManagedCode.MimeTypes" Version="10.1.5" />
21+
<PackageVersion Include="ManagedCode.Communication" Version="10.3.21" />
22+
<PackageVersion Include="ManagedCode.MimeTypes" Version="10.1.6" />
2323
<PackageVersion Include="Microsoft.AspNetCore.Http" Version="2.3.13" />
2424
<PackageVersion Include="Microsoft.AspNetCore.Http.Connections.Client" Version="10.0.14" />
2525
<PackageVersion Include="Microsoft.AspNetCore.Components.WebAssembly" Version="10.0.14" />
@@ -37,8 +37,8 @@
3737
<PackageVersion Include="Microsoft.JSInterop" Version="10.0.14" />
3838
<PackageVersion Include="Microsoft.Playwright" Version="1.64.1" />
3939
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.11.0" />
40-
<PackageVersion Include="Microsoft.Orleans.Runtime" Version="10.4.0" />
41-
<PackageVersion Include="Microsoft.Orleans.TestingHost" Version="10.4.0" />
40+
<PackageVersion Include="Microsoft.Orleans.Runtime" Version="10.4.2" />
41+
<PackageVersion Include="Microsoft.Orleans.TestingHost" Version="10.4.2" />
4242
<PackageVersion Include="NetArchTest.Rules" Version="1.3.2" />
4343
<PackageVersion Include="Shouldly" Version="4.3.0" />
4444
<PackageVersion Include="SSH.NET" Version="2026.0.0" />

‎Integraions/ManagedCode.Storage.Client/StorageClient.cs‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ public async Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl
148148

149149
var chunkSize = (int)Math.Min(ChunkSize, int.MaxValue);
150150
var totalBytes = file.CanSeek ? file.Length : -1;
151-
var totalChunks = totalBytes > 0 ? (int)Math.Ceiling(totalBytes / (double)ChunkSize) : 0;
151+
var totalChunks = totalBytes > 0 ? checked((int)(1 + (totalBytes - 1) / chunkSize)) : 0;
152152

153153
var buffer = new byte[chunkSize];
154154
var chunkIndex = 1;
@@ -163,14 +163,12 @@ public async Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl
163163
var crcState = Crc32Helper.Begin();
164164

165165
int bytesRead;
166-
while ((bytesRead = await file.ReadAsync(buffer.AsMemory(0, chunkSize), cancellationToken)) > 0)
166+
while ((bytesRead = await file.ReadAtLeastAsync(buffer.AsMemory(), chunkSize,
167+
throwOnEndOfStream: false, cancellationToken: cancellationToken)) > 0)
167168
{
168-
var chunkBytes = new byte[bytesRead];
169-
Buffer.BlockCopy(buffer, 0, chunkBytes, 0, bytesRead);
169+
crcState = Crc32Helper.Update(crcState, buffer.AsSpan(0, bytesRead));
170170

171-
crcState = Crc32Helper.Update(crcState, chunkBytes);
172-
173-
using var memoryStream = new MemoryStream(chunkBytes, writable: false);
171+
using var memoryStream = new MemoryStream(buffer, 0, bytesRead, writable: false);
174172
using var content = new StreamContent(memoryStream);
175173
using var formData = new MultipartFormDataContent();
176174

@@ -274,13 +272,13 @@ public async Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl
274272
{
275273
try
276274
{
277-
var dto = JsonSerializer.Deserialize<ChunkUploadCompleteResponseDto>(valueElement.GetRawText());
278-
if (dto == null)
275+
var dto = valueElement.Deserialize<ChunkUploadCompleteResponseDto>(JsonSerializerOptions.Web);
276+
if (dto?.Checksum is not uint serverChecksum)
279277
{
280-
return Result<uint>.Fail("Chunk upload completion response is empty");
278+
return Result<uint>.Fail("Chunk upload completion response is missing the checksum");
281279
}
282280

283-
checksum = dto.Checksum;
281+
checksum = serverChecksum;
284282
break;
285283
}
286284
catch (JsonException ex)
@@ -296,18 +294,20 @@ public async Task<Result<uint>> UploadLargeFile(Stream file, string uploadApiUrl
296294
}
297295

298296
var computedChecksum = Crc32Helper.Complete(crcState);
299-
var finalChecksum = checksum;
297+
var legacyMissingChecksum = checksum == 0 && valueElement.ValueKind is JsonValueKind.Number or JsonValueKind.String;
298+
if (!legacyMissingChecksum && checksum != computedChecksum)
299+
return Result<uint>.Fail("The server checksum does not match the uploaded content");
300300

301-
if (checksum == 0 && computedChecksum != 0)
302-
{
303-
finalChecksum = computedChecksum;
304-
}
305-
else if (checksum != 0 && checksum != computedChecksum)
301+
if (totalBytes <= 0)
306302
{
307-
finalChecksum = computedChecksum;
303+
onProgressChanged?.Invoke(100d);
304+
var elapsed = started.Elapsed;
305+
var speed = elapsed.TotalSeconds > 0 ? transmitted / elapsed.TotalSeconds : transmitted;
306+
OnProgressStatusChanged?.Invoke(this, new ProgressStatus(
307+
resolvedFileName, 1, transmitted, transmitted, elapsed, TimeSpan.Zero, $"{speed:F2} B/s"));
308308
}
309309

310-
return Result<uint>.Succeed(finalChecksum);
310+
return Result<uint>.Succeed(computedChecksum);
311311
}
312312

313313
public async Task<Result<Stream>> GetFileStream(string fileName, string apiUrl, CancellationToken cancellationToken = default)
@@ -434,6 +434,6 @@ file class ChunkUploadCompleteRequestDto
434434

435435
file class ChunkUploadCompleteResponseDto
436436
{
437-
public uint Checksum { get; set; }
437+
public uint? Checksum { get; set; }
438438
public BlobMetadata? Metadata { get; set; }
439439
}

‎Integraions/ManagedCode.Storage.Orleans/Storage/ManagedCodeGrainStorage.cs‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -146,20 +146,23 @@ public async Task ClearStateAsync<T>(string stateName, GrainId grainId, IGrainSt
146146

147147
private async Task<ManagedCodeStoredGrainState<T>?> TryReadRecordAsync<T>(string storagePath, CancellationToken cancellationToken)
148148
{
149-
var existsResult = await storage.ExistsAsync(storagePath, cancellationToken).ConfigureAwait(false);
150-
ThrowIfFailed(existsResult, $"Failed to probe Orleans grain state '{storagePath}'.");
151-
152-
if (!existsResult.Value)
149+
var opened = await storage.GetStreamAsync(storagePath, cancellationToken).ConfigureAwait(false);
150+
if (opened.IsFailed)
153151
{
154-
return null;
155-
}
152+
if (opened.Problem.StatusCode == 404)
153+
return null;
156154

157-
var downloadResult = await storage.DownloadAsync(storagePath, cancellationToken).ConfigureAwait(false);
158-
ThrowIfFailed(downloadResult, $"Failed to download Orleans grain state '{storagePath}'.");
155+
// Some portable providers report a missing stream as a generic failure.
156+
var exists = await storage.ExistsAsync(storagePath, cancellationToken).ConfigureAwait(false);
157+
ThrowIfFailed(exists, $"Failed to probe Orleans grain state '{storagePath}'.");
158+
if (!exists.Value)
159+
return null;
160+
161+
ThrowIfFailed(opened, $"Failed to open Orleans grain state '{storagePath}'.");
162+
}
159163

160-
using var localFile = downloadResult.Value ?? throw new InvalidOperationException(
161-
$"Storage '{storagePath}' was downloaded without a file payload.");
162-
await using var stream = localFile.OpenReadStream(disposeOwner: false);
164+
await using var stream = opened.Value ?? throw new InvalidOperationException(
165+
$"Storage '{storagePath}' opened without a stream.");
163166
var payload = await BinaryData.FromStreamAsync(stream, cancellationToken).ConfigureAwait(false);
164167

165168
return options.GrainStorageSerializer.Deserialize<ManagedCodeStoredGrainState<T>>(payload);

‎Integraions/ManagedCode.Storage.Server/AGENTS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,4 @@ Parent: `../../AGENTS.md`
5454
## Scoped Coverage Repair Maintainability Exception
5555

5656
- the existing `StorageControllerBase` source file already contain responsibilities above the inherited size limits. This repair keeps existing public contracts stable and limits implementation changes to directly tested defects. Do not expand unrelated behavior under this exception. Further decomposition must preserve provider/transport semantics and be reviewed as a separate change.
57+
- The existing `StorageHubBase` type exceeds the type size limit because it contains both streaming iterators and their event contracts. Transfer registration and cancellation live in a separate registry. A future split of the iterators must preserve hub message shapes and transfer cleanup.

‎Integraions/ManagedCode.Storage.Server/ChunkUpload/ChunkUploadFiles.cs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
using System;
2+
using System.Buffers;
3+
using System.Collections.Generic;
4+
using System.IO;
5+
using System.Threading;
6+
using System.Threading.Tasks;
7+
using ManagedCode.Storage.Core.Helpers;
8+
using ManagedCode.Storage.Server.Models;
9+
10+
namespace ManagedCode.Storage.Server.ChunkUpload;
11+
12+
internal static class ChunkUploadFiles
13+
{
14+
private const int CopyBufferSize = 64 * 1024;
15+
16+
internal static async Task StageChunkAsync(ChunkUploadSession session, FileUploadPayload payload, CancellationToken cancellationToken)
17+
{
18+
var chunkIndex = payload.Payload.ChunkIndex;
19+
var chunkFilePath = Path.Combine(session.WorkingDirectory, $"{chunkIndex:D6}.part");
20+
var stagingPath = chunkFilePath + ".incoming";
21+
try
22+
{
23+
await using (var target = new FileStream(stagingPath, FileMode.Create, FileAccess.Write, FileShare.None, CopyBufferSize, useAsync: true))
24+
await using (var source = payload.File.OpenReadStream())
25+
{
26+
await source.CopyToAsync(target, CopyBufferSize, cancellationToken).ConfigureAwait(false);
27+
}
28+
29+
cancellationToken.ThrowIfCancellationRequested();
30+
File.Move(stagingPath, chunkFilePath, overwrite: true);
31+
session.RegisterChunk(chunkIndex, chunkFilePath);
32+
}
33+
finally
34+
{
35+
File.Delete(stagingPath);
36+
}
37+
}
38+
39+
internal static async Task<uint> MergeChunksAsync(string destinationFile, IReadOnlyCollection<string> chunkFiles,
40+
long? expectedLength, CancellationToken cancellationToken)
41+
{
42+
await using var destination = new FileStream(destinationFile, FileMode.Create, FileAccess.Write, FileShare.None, CopyBufferSize, useAsync: true);
43+
using var owner = MemoryPool<byte>.Shared.Rent(CopyBufferSize);
44+
var buffer = owner.Memory[..CopyBufferSize];
45+
var crc = Crc32Helper.Begin();
46+
long length = 0;
47+
foreach (var chunk in chunkFiles)
48+
{
49+
await using var source = new FileStream(chunk, FileMode.Open, FileAccess.Read, FileShare.Read, CopyBufferSize, useAsync: true);
50+
int read;
51+
while ((read = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) != 0)
52+
{
53+
length = checked(length + read);
54+
if (length > expectedLength)
55+
throw new InvalidDataException("The merged file exceeds its declared length.");
56+
57+
crc = Crc32Helper.Update(crc, buffer.Span[..read]);
58+
await destination.WriteAsync(buffer[..read], cancellationToken).ConfigureAwait(false);
59+
}
60+
}
61+
62+
if (expectedLength.HasValue && length != expectedLength.Value)
63+
throw new InvalidDataException("The merged file is shorter than its declared length.");
64+
65+
return Crc32Helper.Complete(crc);
66+
}
67+
}

0 commit comments

Comments
 (0)