From 306d19f4202ec96078b48cd85d4e1c2782915bbf Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 8 Jul 2026 12:50:09 -0700 Subject: [PATCH 01/32] Add opt-in blob payload auto-purge job to AzureBlobPayloads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large orchestration payloads are externalized to Azure Blob Storage as `blob:v1::` tokens. The DTS backend stores those tokens but cannot delete the backing blobs (it has no storage credentials) — only this SDK can. This adds an opt-in, whole-scheduler singleton durable entity + orchestration job (mirroring src/ExportHistory) that drains payload rows the backend has soft-deleted and deletes their blobs, then acks so the backend can hard-delete the rows. Design: - PayloadStore.DeleteAsync is virtual (default throws NotSupportedException so it is non-breaking for existing external subclasses); BlobPayloadStore overrides it to decode the token and call DeleteIfExistsAsync (idempotent). - BlobPurgeJob (TaskEntity singleton): Create is a no-op when already Active so racing client processes don't disturb the running job; Run starts a fixed-id orchestrator. - BlobPurgeJobOrchestrator (perpetual): fetch a batch of tombstones, delete the blobs with capped parallelism, ack the successful deletions (failed tokens stay tombstoned to retry), idle on a timer when empty, ContinueAsNew periodically. - ExecuteBlobPurgeJobOperationOrchestrator bridges client -> entity. - Two new unary RPCs on TaskHubSidecarService: GetTombstonedPayloads / AckPurgedPayloads (authoritative proto follow-up: microsoft/durabletask-protobuf#76). - LargePayloadStorageOptions gains AutoPurge (opt-in, default false) and PayloadPurgeBatchSize (default 500). - Client-side BlobPurgeJobStarter (IHostedService) ensures the singleton job when AutoPurge is enabled, without blocking host startup. Worker always registers the entity/orchestrators/activities so a client-enabled job has something to run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- Microsoft.DurableTask.sln | 17 +- src/Client/Core/DurableTaskClient.cs | 24 +++ src/Client/Core/PayloadPurgeAckDto.cs | 13 ++ src/Client/Core/TombstonedPayloadDto.cs | 15 ++ src/Client/Grpc/GrpcDurableTaskClient.cs | 43 ++++++ .../Activities/AckPurgedPayloadsActivity.cs | 36 +++++ .../Activities/DeleteExternalBlobActivity.cs | 42 +++++ .../GetTombstonedPayloadsActivity.cs | 32 ++++ .../AutoPurge/Client/BlobPurgeJobStarter.cs | 135 ++++++++++++++++ .../AutoPurge/Constants/BlobPurgeConstants.cs | 28 ++++ .../AutoPurge/Entity/BlobPurgeJob.cs | 84 ++++++++++ .../AzureBlobPayloads/AutoPurge/Logs.cs | 39 +++++ .../Models/BlobPurgeJobCreationOptions.cs | 12 ++ .../AutoPurge/Models/BlobPurgeJobState.cs | 40 +++++ .../AutoPurge/Models/BlobPurgeJobStatus.cs | 26 ++++ .../BlobPurgeJobOrchestrator.cs | 145 ++++++++++++++++++ ...xecuteBlobPurgeJobOperationOrchestrator.cs | 30 ++++ ...ientBuilderExtensions.AzureBlobPayloads.cs | 38 +++++ ...rkerBuilderExtensions.AzureBlobPayloads.cs | 16 +- .../Options/LargePayloadStorageOptions.cs | 14 ++ .../PayloadStore/BlobPayloadStore.cs | 31 ++++ .../PayloadStore/PayloadStore.cs | 16 ++ src/Grpc/orchestrator_service.proto | 47 ++++++ .../AutoPurge/BlobPurgeJobTests.cs | 100 ++++++++++++ .../AzureBlobPayloads.Tests.csproj | 24 +++ .../LargePayloadStorageOptionsTests.cs | 21 +++ .../PayloadStore/BlobPayloadStoreTests.cs | 101 ++++++++++++ 27 files changed, 1166 insertions(+), 3 deletions(-) create mode 100644 src/Client/Core/PayloadPurgeAckDto.cs create mode 100644 src/Client/Core/TombstonedPayloadDto.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs create mode 100644 test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs create mode 100644 test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj create mode 100644 test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs create mode 100644 test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 3c6d4539..bf471507 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -1,4 +1,4 @@ - + Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.3.32901.215 @@ -145,6 +145,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureManaged", "AzureManage EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Grpc", "Grpc", "{3B8F957E-7773-4C0C-ACD7-91A1591D9312}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureBlobPayloads.Tests", "test\AzureBlobPayloads.Tests\AzureBlobPayloads.Tests.csproj", "{531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -839,6 +841,18 @@ Global {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x64.Build.0 = Release|Any CPU {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x86.ActiveCfg = Release|Any CPU {C1995163-1DCE-405D-BE82-8B4B2584893E}.Release|x86.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|Any CPU.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x64.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x64.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x86.ActiveCfg = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Debug|x86.Build.0 = Debug|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|Any CPU.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|Any CPU.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x64.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x64.Build.0 = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x86.ActiveCfg = Release|Any CPU + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -911,6 +925,7 @@ Global {C1995163-1DCE-405D-BE82-8B4B2584893E} = {9686B8F9-2644-6C9B-E567-55B0471E4584} {53193780-CD18-2643-6953-C26F59EAEDF5} = {5B448FF6-EC42-491D-A22E-1DC8B618E6D5} {3B8F957E-7773-4C0C-ACD7-91A1591D9312} = {5B448FF6-EC42-491D-A22E-1DC8B618E6D5} + {531B29A0-B2AD-43E2-8F65-0BA85C82C4B5} = {E5637F81-2FB9-4CD7-900D-455363B142A7} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71} diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index 03303800..208e6441 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -549,6 +549,30 @@ public virtual Task> ListInstanceIdsAsync( $"{this.GetType()} does not support listing orchestration instance IDs filtered by completed time."); } + /// + /// Gets a batch of tombstoned (soft-deleted) externalized payloads whose backing blobs should be deleted + /// by a credentialed caller before the backend hard-deletes the rows. + /// + /// The maximum number of tombstoned payloads to request. + /// The cancellation token. + /// The batch of tombstoned payloads whose blobs should be deleted. + /// Thrown if this implementation does not support the operation. + public virtual Task> GetTombstonedPayloadsAsync( + int limit, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); + + /// + /// Acknowledges tombstoned payloads whose backing blobs have been deleted so the backend can hard-delete + /// the corresponding rows. + /// + /// The payloads whose blobs have been deleted. + /// The cancellation token. + /// A task that completes when the acknowledgement has been sent. + /// Thrown if this implementation does not support the operation. + public virtual Task AckPurgedPayloadsAsync( + IEnumerable acks, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); + // TODO: Create task hub // TODO: Delete task hub diff --git a/src/Client/Core/PayloadPurgeAckDto.cs b/src/Client/Core/PayloadPurgeAckDto.cs new file mode 100644 index 00000000..01db0f21 --- /dev/null +++ b/src/Client/Core/PayloadPurgeAckDto.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable acknowledgement that the worker has deleted the blob for a tombstoned payload, so the +/// backend can hard-delete the soft-deleted row. Mirrors the PayloadPurgeAck protobuf message. +/// +/// The backend partition that owns the payload row. +/// The orchestration instance key the payload belongs to. +/// The backend identifier of the soft-deleted payload row. +public sealed record PayloadPurgeAckDto(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayloadDto.cs b/src/Client/Core/TombstonedPayloadDto.cs new file mode 100644 index 00000000..69308d47 --- /dev/null +++ b/src/Client/Core/TombstonedPayloadDto.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable representation of a tombstoned payload the backend has soft-deleted and whose blob the +/// worker should delete. Mirrors the TombstonedPayload protobuf message but is safe to pass through +/// the orchestration/activity boundary. +/// +/// The backend partition that owns the payload row. +/// The orchestration instance key the payload belongs to. +/// The backend identifier of the soft-deleted payload row. +/// The externalized payload token whose backing blob should be deleted. +public sealed record TombstonedPayloadDto(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 23350d4c..822f7238 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -624,6 +624,49 @@ public override async Task> GetOrchestrationHistoryAsync( } } + /// + public override async Task> GetTombstonedPayloadsAsync( + int limit, CancellationToken cancellation = default) + { + P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( + new P.GetTombstonedPayloadsRequest { Limit = limit }, + cancellationToken: cancellation); + + List result = new(response.Payloads.Count); + foreach (P.TombstonedPayload payload in response.Payloads) + { + result.Add(new TombstonedPayloadDto( + payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); + } + + return result; + } + + /// + public override async Task AckPurgedPayloadsAsync( + IEnumerable acks, CancellationToken cancellation = default) + { + Check.NotNull(acks); + + P.AckPurgedPayloadsRequest request = new(); + foreach (PayloadPurgeAckDto ack in acks) + { + request.Acks.Add(new P.PayloadPurgeAck + { + PartitionId = ack.PartitionId, + InstanceKey = ack.InstanceKey, + PayloadId = ack.PayloadId, + }); + } + + if (request.Acks.Count == 0) + { + return; + } + + await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + } + static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) { Func>? recreator = options.Internal.ChannelRecreator; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs new file mode 100644 index 00000000..637fa5ec --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that acknowledges to the backend the payloads whose blobs the worker has deleted, so the backend +/// can hard-delete the soft-deleted rows. +/// +/// The Durable Task client used to acknowledge purged payloads to the backend. +/// The logger instance. +[DurableTask] +public class AckPurgedPayloadsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity, object?> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync(TaskActivityContext context, List input) + { + if (input is null || input.Count == 0) + { + return null; + } + + await this.client.AckPurgedPayloadsAsync(input, CancellationToken.None); + this.logger.BlobPurgeAckedPayloads(input.Count); + return null; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs new file mode 100644 index 00000000..0018cb7c --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so +/// re-delivered tokens and concurrent workers are safe. On failure the payload is left tombstoned so a later +/// purge cycle can retry it. +/// +/// The payload store used to delete blobs. +/// The logger instance. +[DurableTask] +public class DeleteExternalBlobActivity( + PayloadStore store, + ILogger logger) + : TaskActivity +{ + readonly PayloadStore store = Check.NotNull(store); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync(TaskActivityContext context, string input) + { + Check.NotNullOrEmpty(input, nameof(input)); + + try + { + await this.store.DeleteAsync(input, CancellationToken.None); + return true; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Leave the payload tombstoned so the backend re-streams it on a later cycle; a single bad token + // must not fail the whole batch. + this.logger.BlobPurgeDeleteFailed(ex, input); + return false; + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs new file mode 100644 index 00000000..6086efe2 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that fetches a batch of tombstoned payloads from the backend for the auto-purge job to delete. +/// +/// The Durable Task client used to query the backend for tombstoned payloads. +/// The logger instance. +[DurableTask] +public class GetTombstonedPayloadsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task> RunAsync(TaskActivityContext context, int input) + { + int limit = input > 0 ? input : 500; + List payloads = + await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); + this.logger.BlobPurgeFetchedTombstones(payloads.Count); + return payloads; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs new file mode 100644 index 00000000..9652c1e3 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists when auto-purge +/// is enabled via . It never blocks host startup: it runs +/// on a background task and retries until the backend is reachable. The job is a whole-scheduler singleton, +/// so racing client processes simply no-op. +/// +sealed class BlobPurgeJobStarter : IHostedService +{ + static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); + + readonly DurableTaskClient client; + readonly IOptionsMonitor options; + readonly string builderName; + readonly ILogger logger; + readonly EntityInstanceId entityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + CancellationTokenSource? cts; + Task? ensureTask; + + public BlobPurgeJobStarter( + DurableTaskClient client, + IOptionsMonitor options, + string builderName, + ILogger logger) + { + this.client = Check.NotNull(client); + this.options = Check.NotNull(options); + this.builderName = Check.NotNull(builderName); + this.logger = Check.NotNull(logger); + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + LargePayloadStorageOptions opts = this.options.Get(this.builderName); + if (!opts.AutoPurge) + { + this.logger.BlobPurgeDisabled(); + return Task.CompletedTask; + } + + int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : 500; + + // Do not block host startup; ensure the job on a background task with basic retry until the backend + // is reachable. + this.cts = new CancellationTokenSource(); + this.ensureTask = Task.Run(() => this.EnsureJobAsync(batchSize, this.cts.Token), CancellationToken.None); + return Task.CompletedTask; + } + + /// + public async Task StopAsync(CancellationToken cancellationToken) + { + this.cts?.Cancel(); + + Task? pending = this.ensureTask; + if (pending is not null) + { + // The ensure loop observes cancellation and returns promptly; swallow any faulted/cancelled result. + await Task.WhenAny(pending, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false); + } + } + + async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + if (await this.IsJobActiveAsync(cancellationToken)) + { + return; + } + + BlobPurgeJobOperationRequest request = new( + this.entityId, + nameof(BlobPurgeJob.Create), + new BlobPurgeJobCreationOptions(batchSize)); + + await this.client.ScheduleNewOrchestrationInstanceAsync( + new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), + request, + cancellationToken); + + this.logger.BlobPurgeJobEnsured(); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + this.logger.BlobPurgeStarterRetry(ex); + try + { + await Task.Delay(RetryDelay, cancellationToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + } + + async Task IsJobActiveAsync(CancellationToken cancellationToken) + { + try + { + EntityMetadata? metadata = + await this.client.Entities.GetEntityAsync( + this.entityId, cancellation: cancellationToken); + return metadata is not null && metadata.State.Status == BlobPurgeJobStatus.Active; + } + catch (NotSupportedException) + { + // The entity-query API is unavailable on this client; fall back to scheduling the idempotent + // Create, which no-ops if the job is already active. + return false; + } + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs new file mode 100644 index 00000000..eafee991 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Constants used throughout the blob payload auto-purge functionality. +/// +static class BlobPurgeConstants +{ + /// + /// The fixed, process-global job ID for the singleton blob payload auto-purge job. A single job drains + /// tombstoned payloads for the whole scheduler, so the ID is hard-coded rather than caller-supplied. + /// + public const string JobId = "__dt_blob_payload_autopurge__"; + + /// + /// The prefix used for generating blob purge job orchestrator instance IDs. Format: "BlobPurgeJob-{jobId}". + /// + public const string OrchestratorInstanceIdPrefix = "BlobPurgeJob-"; + + /// + /// Generates an orchestrator instance ID for a given blob purge job ID. + /// + /// The blob purge job ID. + /// The orchestrator instance ID. + public static string GetOrchestratorInstanceId(string jobId) => $"{OrchestratorInstanceIdPrefix}{jobId}"; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs new file mode 100644 index 00000000..8f08b867 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Durable entity that manages the lifecycle of the singleton blob payload auto-purge job. +/// +/// The logger instance. +class BlobPurgeJob(ILogger logger) : TaskEntity +{ + /// + /// Creates (or reactivates) the auto-purge job. Because the job is a whole-scheduler singleton, this is + /// intentionally a no-op when the job is already so that extra + /// client processes racing to create it do not disturb the running job. + /// + /// The entity context. + /// The job creation options. + public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creationOptions) + { + Check.NotNull(creationOptions, nameof(creationOptions)); + + if (this.State.Status == BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobAlreadyRunning(context.Id.Key); + return; + } + + this.State.Status = BlobPurgeJobStatus.Active; + this.State.PurgeBatchSize = creationOptions.PurgeBatchSize > 0 ? creationOptions.PurgeBatchSize : 500; + this.State.CreatedAt ??= DateTimeOffset.UtcNow; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + this.State.LastError = null; + + logger.BlobPurgeJobCreated(context.Id.Key); + + // Signal Run to start the perpetual purge orchestrator. + context.SignalEntity(context.Id, nameof(this.Run)); + } + + /// + /// Starts the purge orchestrator if the job is active. Uses a fixed orchestrator instance ID so only one + /// orchestrator ever runs for the singleton job. + /// + /// The entity context. + public void Run(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + return; + } + + string instanceId = BlobPurgeConstants.GetOrchestratorInstanceId(context.Id.Key); + StartOrchestrationOptions startOrchestrationOptions = new(instanceId); + + context.ScheduleNewOrchestration( + new TaskName(nameof(BlobPurgeJobOrchestrator)), + new BlobPurgeJobRunRequest(context.Id, this.State.PurgeBatchSize), + startOrchestrationOptions); + + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + /// + /// Records progress after a purge cycle completes. + /// + /// The entity context. + /// The number of blobs purged in the cycle. + public void RecordPurged(TaskEntityContext context, long purgedCount) + { + this.State.PurgedCount += purgedCount; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } + + /// + /// Gets the current state of the auto-purge job. + /// + /// The entity context. + /// The current job state. + public BlobPurgeJobState Get(TaskEntityContext context) => this.State; +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs new file mode 100644 index 00000000..6e4ebed8 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Log messages for the Azure Blob externalized-payload auto-purge job. +/// +static partial class Logs +{ + [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] + public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already running; ignoring the create request.")] + public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); + + [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] + public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); + + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Failed to delete externalized payload blob for token '{token}'; leaving it tombstoned for a later purge cycle.")] + public static partial void BlobPurgeDeleteFailed(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] + public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); + + [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] + public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); + + [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled; the singleton purge job will not be started.")] + public static partial void BlobPurgeDisabled(this ILogger logger); + + [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] + public static partial void BlobPurgeJobEnsured(this ILogger logger); + + [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] + public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs new file mode 100644 index 00000000..e49e423c --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Options used to create the singleton blob payload auto-purge job. +/// +/// +/// The maximum number of tombstoned payloads to request from the backend per cycle. +/// +public sealed record BlobPurgeJobCreationOptions(int PurgeBatchSize); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs new file mode 100644 index 00000000..8bd4cdd5 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// State for the singleton blob payload auto-purge job, stored in the entity. +/// +public sealed class BlobPurgeJobState +{ + /// + /// Gets or sets the current status of the auto-purge job. + /// + public BlobPurgeJobStatus Status { get; set; } + + /// + /// Gets or sets the time when the job was first created. + /// + public DateTimeOffset? CreatedAt { get; set; } + + /// + /// Gets or sets the time when the job state was last modified. + /// + public DateTimeOffset? LastModifiedAt { get; set; } + + /// + /// Gets or sets the total number of payload blobs the job has purged. + /// + public long PurgedCount { get; set; } + + /// + /// Gets or sets the last error message, if any. + /// + public string? LastError { get; set; } + + /// + /// Gets or sets the maximum number of tombstoned payloads requested from the backend per cycle. + /// + public int PurgeBatchSize { get; set; } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs new file mode 100644 index 00000000..917b8d1b --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Represents the current status of the singleton blob payload auto-purge job. +/// +public enum BlobPurgeJobStatus +{ + /// + /// The job is not running. This is the default status of a freshly initialized entity, so it is kept + /// as the zero value to avoid a brand-new entity accidentally appearing active. + /// + Stopped, + + /// + /// The job is active and draining tombstoned payloads from the backend. + /// + Active, + + /// + /// The job has failed. + /// + Failed, +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs new file mode 100644 index 00000000..2b7ba6f2 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Orchestrator input describing the purge job to run. +/// +/// The entity ID of the owning . +/// The maximum number of tombstoned payloads to request per cycle. +/// The number of cycles processed since the last continue-as-new. +public sealed record BlobPurgeJobRunRequest( + EntityInstanceId JobEntityId, int PurgeBatchSize, int ProcessedCycles = 0); + +/// +/// Perpetual orchestrator that drains tombstoned payloads from the backend, deletes their blobs with capped +/// parallelism, and acknowledges the successful deletions so the backend can hard-delete the rows. It idles +/// on a timer when there is nothing to purge and continues-as-new periodically to keep its history small. +/// +[DurableTask] +public class BlobPurgeJobOrchestrator : TaskOrchestrator +{ + const int ContinueAsNewFrequency = 5; + const int MaxParallelDeletes = 32; + const int DefaultPurgeBatchSize = 500; + static readonly TimeSpan IdleDelay = TimeSpan.FromMinutes(1); + + // Retry policy for the purge activities: 3 attempts with exponential backoff (15s, 30s, capped at 60s). + static readonly RetryPolicy PurgeActivityRetryPolicy = new( + maxNumberOfAttempts: 3, + firstRetryInterval: TimeSpan.FromSeconds(15), + backoffCoefficient: 2.0, + maxRetryInterval: TimeSpan.FromSeconds(60)); + + /// + public override async Task RunAsync(TaskOrchestrationContext context, BlobPurgeJobRunRequest input) + { + ILogger logger = context.CreateReplaySafeLogger(); + string jobId = input.JobEntityId.Key; + + int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : DefaultPurgeBatchSize; + int processedCycles = input.ProcessedCycles; + + while (true) + { + processedCycles++; + if (processedCycles > ContinueAsNewFrequency) + { + context.ContinueAsNew(new BlobPurgeJobRunRequest(input.JobEntityId, batchSize, ProcessedCycles: 0)); + return null!; + } + + // Stop cleanly if the job has been stopped or removed. + BlobPurgeJobState? state = await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.Get), null); + + if (state is null || state.Status != BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); + return null; + } + + List tombstones = await context.CallActivityAsync>( + nameof(GetTombstonedPayloadsActivity), + batchSize, + new TaskOptions(PurgeActivityRetryPolicy)); + + if (tombstones is null || tombstones.Count == 0) + { + // Nothing to purge right now: block on a timer (push-free idle) then check again. + await context.CreateTimer(IdleDelay, default); + continue; + } + + List deleted = await this.DeleteBatchAsync(context, tombstones); + + if (deleted.Count > 0) + { + await context.CallActivityAsync( + nameof(AckPurgedPayloadsActivity), + deleted, + new TaskOptions(PurgeActivityRetryPolicy)); + + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)deleted.Count); + } + } + } + + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) + { + List deleted = new(tombstones.Count); + List> tasks = new(); + + foreach (TombstonedPayloadDto tombstone in tombstones) + { + tasks.Add(this.DeleteOneAsync(context, tombstone)); + + if (tasks.Count >= MaxParallelDeletes) + { + await DrainAsync(tasks, deleted); + tasks.Clear(); + } + } + + if (tasks.Count > 0) + { + await DrainAsync(tasks, deleted); + } + + return deleted; + } + + static async Task DrainAsync(List> tasks, List deleted) + { + DeleteOutcome[] outcomes = await Task.WhenAll(tasks); + foreach (DeleteOutcome outcome in outcomes) + { + // Only acknowledge blobs that were actually deleted; failed tokens stay tombstoned to retry. + if (outcome.Deleted) + { + deleted.Add(outcome.Ack); + } + } + } + + async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayloadDto tombstone) + { + bool deleted = await context.CallActivityAsync( + nameof(DeleteExternalBlobActivity), + tombstone.Token, + new TaskOptions(PurgeActivityRetryPolicy)); + + return new DeleteOutcome( + deleted, + new PayloadPurgeAckDto(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + } + + readonly record struct DeleteOutcome(bool Deleted, PayloadPurgeAckDto Ack); +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs new file mode 100644 index 00000000..e8f0c66b --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/ExecuteBlobPurgeJobOperationOrchestrator.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Entities; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Orchestrator that executes a single operation on a blob purge job entity and returns the result. Used as +/// a client-to-entity bridge so clients can drive the entity through the orchestration surface. +/// +[DurableTask] +public class ExecuteBlobPurgeJobOperationOrchestrator + : TaskOrchestrator +{ + /// + public override async Task RunAsync( + TaskOrchestrationContext context, BlobPurgeJobOperationRequest input) + { + return await context.Entities.CallEntityAsync(input.EntityId, input.OperationName, input.Input); + } +} + +/// +/// Request for executing a blob purge job entity operation. +/// +/// The ID of the entity to execute the operation on. +/// The name of the operation to execute. +/// Optional input for the operation. +public record BlobPurgeJobOperationRequest(EntityInstanceId EntityId, string OperationName, object? Input = null); diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0817bcea..2557c6da 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -2,11 +2,14 @@ // Licensed under the MIT License. using Grpc.Core.Interceptors; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.Grpc; using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace Microsoft.DurableTask; @@ -16,6 +19,29 @@ namespace Microsoft.DurableTask; /// public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads { + /// + /// Enables externalized payload storage using Azure Blob Storage for the specified client builder. + /// + /// The builder to configure. + /// The callback to configure the storage options. + /// The original builder, for call chaining. + public static IDurableTaskClientBuilder UseExternalizedPayloads( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + return UseExternalizedPayloadsCore(builder); + } + /// /// Enables externalized payload storage using a pre-configured shared payload store. /// This overload helps ensure client and worker use the same configuration. @@ -56,6 +82,18 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); + RegisterBlobPurgeJobStarter(builder); + return builder; } + + static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) + { + string builderName = builder.Name; + builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( + sp.GetRequiredService(), + sp.GetRequiredService>(), + builderName, + sp.GetRequiredService>())); + } } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index b690d288..42ac0f0a 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -2,8 +2,7 @@ // Licensed under the MIT License. using Grpc.Core.Interceptors; -using Grpc.Net.Client; -using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; @@ -82,6 +81,19 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB opt.Capabilities.Add(P.WorkerCapability.LargePayloads); }); + // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are + // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to + // execute here. The purge activities fetch/ack via the injected DurableTaskClient. + builder.AddTasks(r => + { + r.AddEntity(); + r.AddOrchestrator(); + r.AddOrchestrator(); + r.AddActivity(); + r.AddActivity(); + r.AddActivity(); + }); + return builder; } } diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 6abcbdf2..f53c9e72 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -115,4 +115,18 @@ public int ThresholdBytes /// Defaults to true for reduced storage and bandwidth. /// public bool CompressionEnabled { get; set; } = true; + + /// + /// Gets or sets a value indicating whether the client should start the singleton blob payload auto-purge + /// job. When enabled, the job periodically drains payload rows the backend has soft-deleted and deletes + /// the corresponding blobs from customer storage (the backend has no storage credentials of its own). + /// Defaults to false (opt-in). + /// + public bool AutoPurge { get; set; } + + /// + /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend + /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. + /// + public int PayloadPurgeBatchSize { get; set; } = 500; } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index e01d2e74..1934c5d4 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -67,6 +67,18 @@ public BlobPayloadStore(LargePayloadStorageOptions options) this.containerClient = serviceClient.GetBlobContainerClient(options.ContainerName); } + /// + /// Initializes a new instance of the class using a caller-supplied + /// container client. Intended for unit testing so a mocked can be injected. + /// + /// The blob container client to use. + /// The options for the blob payload store. + internal BlobPayloadStore(BlobContainerClient containerClient, LargePayloadStorageOptions options) + { + this.containerClient = containerClient ?? throw new ArgumentNullException(nameof(containerClient)); + this.options = options ?? throw new ArgumentNullException(nameof(options)); + } + /// public override async Task UploadAsync(string payLoad, CancellationToken cancellationToken) { @@ -145,6 +157,25 @@ public override async Task DownloadAsync(string token, CancellationToken } } + /// + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + { + (string container, string name) = DecodeToken(token); + if (!string.Equals(container, this.containerClient.Name, StringComparison.Ordinal)) + { + throw new ArgumentException("Token container does not match configured container.", nameof(token)); + } + + BlobClient blob = this.containerClient.GetBlobClient(name); + + // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is + // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. + await blob.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + conditions: null, + cancellationToken); + } + /// public override bool IsKnownPayloadToken(string value) { diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index b0fe6f80..0ae5ccb6 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -24,6 +24,22 @@ public abstract class PayloadStore /// Payload string. public abstract Task DownloadAsync(string token, CancellationToken cancellationToken); + /// + /// Deletes the payload referenced by the token. Implementations that support deletion must be + /// idempotent: deleting a payload that no longer exists is a no-op and must not throw. + /// + /// + /// The default implementation throws . Stores that externalize + /// payloads to deletable storage (for example Azure Blob Storage) should override it. It is declared + /// virtual rather than abstract so that adding it does not break existing external subclasses. + /// + /// The opaque reference token. + /// Cancellation token. + /// A task that completes when the payload has been deleted (or was already absent). + public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException( + $"This {nameof(PayloadStore)} implementation does not support deleting payloads."); + /// /// Returns true if the specified value appears to be a token understood by this store. /// Implementations should not throw for unknown tokens. diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 3d9194ac..af065941 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -823,6 +823,53 @@ service TaskHubSidecarService { // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". // Note that a maximum of 500 orchestrations can be terminated at a time using this method. rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); + + // Returns a batch of blob-externalized payload tombstones the backend has soft-deleted so the worker + // can delete the corresponding blobs from customer storage (the backend has no storage credentials of + // its own). The worker deletes each blob and then calls AckPurgedPayloads so the backend can + // hard-delete the soft-deleted rows. This is an opt-in, worker-driven pull model. + rpc GetTombstonedPayloads(GetTombstonedPayloadsRequest) returns (GetTombstonedPayloadsResponse); + + // Acknowledges that the worker has deleted the blobs for the identified payloads so the backend can + // hard-delete the soft-deleted rows. + rpc AckPurgedPayloads(AckPurgedPayloadsRequest) returns (AckPurgedPayloadsResponse); +} + +// Server -> client. Identifies a blob-externalized payload that the backend has soft-deleted and whose +// blob the worker should delete from customer storage. +message TombstonedPayload { + int32 partitionId = 1; + int64 instanceKey = 2; + int64 payloadId = 3; + // The externalized payload token (e.g. "blob:v1::") whose backing blob should be deleted. + string token = 4; +} + +// Client -> server. Acknowledges that the worker has deleted the blob for the identified payload so the +// backend can hard-delete the soft-deleted row. +message PayloadPurgeAck { + int32 partitionId = 1; + int64 instanceKey = 2; + int64 payloadId = 3; +} + +// Client -> server. Requests up to `limit` tombstoned payloads for the worker to delete. +message GetTombstonedPayloadsRequest { + int32 limit = 1; +} + +// Server -> client. Carries the batch of tombstoned payloads for the worker to delete. +message GetTombstonedPayloadsResponse { + repeated TombstonedPayload payloads = 1; +} + +// Client -> server. Acknowledges a batch of payloads whose blobs the worker has deleted. +message AckPurgedPayloadsRequest { + repeated PayloadPurgeAck acks = 1; +} + +// Server -> client. Empty response acknowledging the acks were recorded. +message AckPurgedPayloadsResponse { } message GetWorkItemsRequest { diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs new file mode 100644 index 00000000..126afdf8 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Entities.Tests; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobTests +{ + readonly BlobPurgeJob job = new(new TestLogger()); + + [Fact] + public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() + { + // Arrange + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + new BlobPurgeJobCreationOptions(250)); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(250); + state.CreatedAt.Should().NotBeNull(); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Create_WhenAlreadyActive_IsNoOp() + { + // Arrange + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + new BlobPurgeJobCreationOptions(999)); + + // Act + await this.job.RunAsync(operation); + + // Assert - status stays Active and the original batch size is retained, proving the create no-op'd. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(100); + } + + [Fact] + public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() + { + // Arrange + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(null), + new BlobPurgeJobCreationOptions(0)); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(500); + } + + [Fact] + public async Task Get_ReturnsCurrentState() + { + // Arrange + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 42, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Get), + new TestEntityState(existing), + null); + + // Act + object? result = await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType(result); + state.Status.Should().Be(BlobPurgeJobStatus.Active); + state.PurgeBatchSize.Should().Be(42); + } +} diff --git a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj new file mode 100644 index 00000000..77078cbb --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj @@ -0,0 +1,24 @@ + + + + net10.0 + enable + enable + false + true + + Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests + Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests + + + + + + + + + + + + diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs new file mode 100644 index 00000000..317b1e1e --- /dev/null +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.Options; + +public class LargePayloadStorageOptionsTests +{ + [Fact] + public void Defaults_AutoPurgeDisabled_AndBatchSize500() + { + // Arrange & Act + LargePayloadStorageOptions options = new(); + + // Assert + options.AutoPurge.Should().BeFalse(); + options.PayloadPurgeBatchSize.Should().Be(500); + } +} diff --git a/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs b/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs new file mode 100644 index 00000000..e7fe1448 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreTests.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests; + +public class BlobPayloadStoreTests +{ + const string ContainerName = "payloads"; + + static Mock CreateContainer(Mock blob, string expectedBlobName) + { + Mock container = new(); + container.Setup(c => c.Name).Returns(ContainerName); + container.Setup(c => c.GetBlobClient(expectedBlobName)).Returns(blob.Object); + return container; + } + + static Mock CreateBlob(bool existed) + { + Mock blob = new(); + blob + .Setup(b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(existed, Mock.Of())); + return blob; + } + + [Fact] + public async Task DeleteAsync_ValidToken_DeletesBackingBlobIncludingSnapshots() + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act + await store.DeleteAsync($"blob:v1:{ContainerName}:abc123", CancellationToken.None); + + // Assert + container.Verify(c => c.GetBlobClient("abc123"), Times.Once); + blob.Verify( + b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() + { + // Arrange + Mock blob = CreateBlob(existed: false); + Mock container = CreateContainer(blob, "missing"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act (a missing blob must be a no-op, not an error) + await store.DeleteAsync($"blob:v1:{ContainerName}:missing", CancellationToken.None); + + // Assert + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Once); + } + + [Fact] + public async Task DeleteAsync_ContainerMismatch_ThrowsAndDoesNotDelete() + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act & Assert + await Assert.ThrowsAsync( + () => store.DeleteAsync("blob:v1:other-container:abc123", CancellationToken.None)); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); + } + + [Theory] + [InlineData("not-a-token")] + [InlineData("blob:v1:only-container")] + [InlineData("blob:v1::blobname")] + public async Task DeleteAsync_InvalidToken_ThrowsArgumentException(string token) + { + // Arrange + Mock blob = CreateBlob(existed: true); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(container.Object, new LargePayloadStorageOptions()); + + // Act & Assert + await Assert.ThrowsAsync(() => store.DeleteAsync(token, CancellationToken.None)); + } +} From 60f6637bb5983914d0c4de592bf5e30383e693a2 Mon Sep 17 00:00:00 2001 From: wangbill Date: Mon, 13 Jul 2026 18:39:26 -0700 Subject: [PATCH 02/32] Address PR #758 review feedback: naming, self-heal, poison-ack, starter simplification - Drop the `Dto` suffix now that the payload records are first-class public types in `Microsoft.DurableTask.Client` (`TombstonedPayload`, `PayloadPurgeAck`). - Collapse the magic `500` batch-size literal into a single `BlobPurgeConstants.DefaultBatchSize` used everywhere. - Rename `BlobPurgeJobStatus.Stopped` -> `Pending` (still the zero value) and remove the dead `Failed` member (nothing ever set it; the job self-heals). - Make the perpetual orchestrator self-heal: wrap each cycle in try/catch so a transient backend/entity/activity failure logs, backs off, and continues instead of failing the orchestration and killing the eternal loop. - Ack poison tokens: `DeleteExternalBlobActivity` now returns a three-way `BlobDeleteResult` (Deleted/Discarded/Retry). Malformed tokens are discarded and acked so the backend can clear the stuck row instead of re-streaming it forever; transient failures stay tombstoned to retry. - Replace the single-value `BlobPurgeJobCreationOptions` record with a plain `int` on `BlobPurgeJob.Create`. - Guard the client fetch RPC: `GetTombstonedPayloadsAsync` throws `ArgumentOutOfRangeException` unless `0 < limit < 1000`. - Simplify `BlobPurgeJobStarter` to a fixed-instance-id fire-once: drop the entity-active pre-check and schedule the Create bridge once with a fixed instance id, retrying only until the backend is reachable. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Core/DurableTaskClient.cs | 4 +- ...yloadPurgeAckDto.cs => PayloadPurgeAck.cs} | 2 +- ...onedPayloadDto.cs => TombstonedPayload.cs} | 2 +- src/Client/Grpc/GrpcDurableTaskClient.cs | 16 ++- .../Activities/AckPurgedPayloadsActivity.cs | 4 +- .../Activities/DeleteExternalBlobActivity.cs | 23 ++-- .../GetTombstonedPayloadsActivity.cs | 8 +- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 33 +----- .../AutoPurge/Constants/BlobPurgeConstants.cs | 12 ++ .../AutoPurge/Entity/BlobPurgeJob.cs | 10 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 6 + .../AutoPurge/Models/BlobDeleteResult.cs | 27 +++++ .../Models/BlobPurgeJobCreationOptions.cs | 12 -- .../AutoPurge/Models/BlobPurgeJobStatus.cs | 11 +- .../BlobPurgeJobOrchestrator.cs | 109 ++++++++++-------- .../Options/LargePayloadStorageOptions.cs | 3 +- .../AutoPurge/BlobPurgeJobTests.cs | 6 +- 17 files changed, 160 insertions(+), 128 deletions(-) rename src/Client/Core/{PayloadPurgeAckDto.cs => PayloadPurgeAck.cs} (87%) rename src/Client/Core/{TombstonedPayloadDto.cs => TombstonedPayload.cs} (87%) create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index 208e6441..bc31915c 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -557,7 +557,7 @@ public virtual Task> ListInstanceIdsAsync( /// The cancellation token. /// The batch of tombstoned payloads whose blobs should be deleted. /// Thrown if this implementation does not support the operation. - public virtual Task> GetTombstonedPayloadsAsync( + public virtual Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); @@ -570,7 +570,7 @@ public virtual Task> GetTombstonedPayloadsAsync( /// A task that completes when the acknowledgement has been sent. /// Thrown if this implementation does not support the operation. public virtual Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + IEnumerable acks, CancellationToken cancellation = default) => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); // TODO: Create task hub diff --git a/src/Client/Core/PayloadPurgeAckDto.cs b/src/Client/Core/PayloadPurgeAck.cs similarity index 87% rename from src/Client/Core/PayloadPurgeAckDto.cs rename to src/Client/Core/PayloadPurgeAck.cs index 01db0f21..6e7788aa 100644 --- a/src/Client/Core/PayloadPurgeAckDto.cs +++ b/src/Client/Core/PayloadPurgeAck.cs @@ -10,4 +10,4 @@ namespace Microsoft.DurableTask.Client; /// The backend partition that owns the payload row. /// The orchestration instance key the payload belongs to. /// The backend identifier of the soft-deleted payload row. -public sealed record PayloadPurgeAckDto(int PartitionId, long InstanceKey, long PayloadId); +public sealed record PayloadPurgeAck(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayloadDto.cs b/src/Client/Core/TombstonedPayload.cs similarity index 87% rename from src/Client/Core/TombstonedPayloadDto.cs rename to src/Client/Core/TombstonedPayload.cs index 69308d47..bb3f6896 100644 --- a/src/Client/Core/TombstonedPayloadDto.cs +++ b/src/Client/Core/TombstonedPayload.cs @@ -12,4 +12,4 @@ namespace Microsoft.DurableTask.Client; /// The orchestration instance key the payload belongs to. /// The backend identifier of the soft-deleted payload row. /// The externalized payload token whose backing blob should be deleted. -public sealed record TombstonedPayloadDto(int PartitionId, long InstanceKey, long PayloadId, string Token); +public sealed record TombstonedPayload(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 822f7238..8be31fd5 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -625,17 +625,23 @@ public override async Task> GetOrchestrationHistoryAsync( } /// - public override async Task> GetTombstonedPayloadsAsync( + public override async Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) { + if (limit <= 0 || limit >= 1000) + { + throw new ArgumentOutOfRangeException( + nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); + } + P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( new P.GetTombstonedPayloadsRequest { Limit = limit }, cancellationToken: cancellation); - List result = new(response.Payloads.Count); + List result = new(response.Payloads.Count); foreach (P.TombstonedPayload payload in response.Payloads) { - result.Add(new TombstonedPayloadDto( + result.Add(new TombstonedPayload( payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); } @@ -644,12 +650,12 @@ public override async Task> GetTombstonedPayloadsAsyn /// public override async Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + IEnumerable acks, CancellationToken cancellation = default) { Check.NotNull(acks); P.AckPurgedPayloadsRequest request = new(); - foreach (PayloadPurgeAckDto ack in acks) + foreach (PayloadPurgeAck ack in acks) { request.Acks.Add(new P.PayloadPurgeAck { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs index 637fa5ec..7ef6dbc8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs @@ -16,13 +16,13 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class AckPurgedPayloadsActivity( DurableTaskClient client, ILogger logger) - : TaskActivity, object?> + : TaskActivity, object?> { readonly DurableTaskClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, List input) + public override async Task RunAsync(TaskActivityContext context, List input) { if (input is null || input.Count == 0) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 0018cb7c..f070f964 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -7,8 +7,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. On failure the payload is left tombstoned so a later -/// purge cycle can retry it. +/// re-delivered tokens and concurrent workers are safe. Malformed (poison) tokens are discarded so they get +/// acknowledged instead of retried forever; transient failures leave the payload tombstoned to retry. /// /// The payload store used to delete blobs. /// The logger instance. @@ -16,27 +16,34 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class DeleteExternalBlobActivity( PayloadStore store, ILogger logger) - : TaskActivity + : TaskActivity { readonly PayloadStore store = Check.NotNull(store); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, string input) + public override async Task RunAsync(TaskActivityContext context, string input) { Check.NotNullOrEmpty(input, nameof(input)); try { await this.store.DeleteAsync(input, CancellationToken.None); - return true; + return BlobDeleteResult.Deleted; + } + catch (ArgumentException ex) + { + // The token is malformed or points at a different container; it can never succeed. Discard it so + // the backend clears the row instead of re-streaming the same poison token every cycle. + this.logger.BlobPurgeDeleteDiscarded(ex, input); + return BlobDeleteResult.Discarded; } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - // Leave the payload tombstoned so the backend re-streams it on a later cycle; a single bad token - // must not fail the whole batch. + // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single + // bad token must not fail the whole batch. this.logger.BlobPurgeDeleteFailed(ex, input); - return false; + return BlobDeleteResult.Retry; } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs index 6086efe2..2b1bad7d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -15,16 +15,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class GetTombstonedPayloadsActivity( DurableTaskClient client, ILogger logger) - : TaskActivity> + : TaskActivity> { readonly DurableTaskClient client = Check.NotNull(client); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task> RunAsync(TaskActivityContext context, int input) + public override async Task> RunAsync(TaskActivityContext context, int input) { - int limit = input > 0 ? input : 500; - List payloads = + int limit = input > 0 ? input : BlobPurgeConstants.DefaultBatchSize; + List payloads = await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); this.logger.BlobPurgeFetchedTombstones(payloads.Count); return payloads; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 9652c1e3..17d11ab0 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Client; -using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -51,7 +50,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : 500; + int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. @@ -79,19 +78,16 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) { try { - if (await this.IsJobActiveAsync(cancellationToken)) - { - return; - } - + // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is + // active) and the orchestrator's fixed instance id, so just schedule the bridge once with a + // fixed instance id. Retry only if the backend is unreachable at startup. BlobPurgeJobOperationRequest request = new( - this.entityId, - nameof(BlobPurgeJob.Create), - new BlobPurgeJobCreationOptions(batchSize)); + this.entityId, nameof(BlobPurgeJob.Create), batchSize); await this.client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, + new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), cancellationToken); this.logger.BlobPurgeJobEnsured(); @@ -115,21 +111,4 @@ await this.client.ScheduleNewOrchestrationInstanceAsync( } } } - - async Task IsJobActiveAsync(CancellationToken cancellationToken) - { - try - { - EntityMetadata? metadata = - await this.client.Entities.GetEntityAsync( - this.entityId, cancellation: cancellationToken); - return metadata is not null && metadata.State.Status == BlobPurgeJobStatus.Active; - } - catch (NotSupportedException) - { - // The entity-query API is unavailable on this client; fall back to scheduling the idempotent - // Create, which no-ops if the job is already active. - return false; - } - } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index eafee991..3697f199 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -14,6 +14,18 @@ static class BlobPurgeConstants /// public const string JobId = "__dt_blob_payload_autopurge__"; + /// + /// The default number of tombstoned payloads the auto-purge job requests from the backend per cycle, + /// used whenever a configured batch size is missing or non-positive. + /// + public const int DefaultBatchSize = 500; + + /// + /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the + /// singleton job. A fixed ID keeps racing client processes from creating duplicate bridge orchestrations. + /// + public const string StarterInstanceId = "BlobPurgeJobStarter-" + JobId; + /// /// The prefix used for generating blob purge job orchestrator instance IDs. Format: "BlobPurgeJob-{jobId}". /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 8f08b867..c2103fa6 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -18,11 +18,11 @@ class BlobPurgeJob(ILogger logger) : TaskEntity /// client processes racing to create it do not disturb the running job. /// /// The entity context. - /// The job creation options. - public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creationOptions) + /// + /// The maximum number of tombstoned payloads to request from the backend per cycle. + /// + public void Create(TaskEntityContext context, int purgeBatchSize) { - Check.NotNull(creationOptions, nameof(creationOptions)); - if (this.State.Status == BlobPurgeJobStatus.Active) { logger.BlobPurgeJobAlreadyRunning(context.Id.Key); @@ -30,7 +30,7 @@ public void Create(TaskEntityContext context, BlobPurgeJobCreationOptions creati } this.State.Status = BlobPurgeJobStatus.Active; - this.State.PurgeBatchSize = creationOptions.PurgeBatchSize > 0 ? creationOptions.PurgeBatchSize : 500; + this.State.PurgeBatchSize = purgeBatchSize > 0 ? purgeBatchSize : BlobPurgeConstants.DefaultBatchSize; this.State.CreatedAt ??= DateTimeOffset.UtcNow; this.State.LastModifiedAt = DateTimeOffset.UtcNow; this.State.LastError = null; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 6e4ebed8..3b7ccc98 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -36,4 +36,10 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); + + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Discarding poison externalized payload token '{token}'; it can never be deleted, acknowledging it so the backend can clear the row.")] + public static partial void BlobPurgeDeleteDiscarded(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] + public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs new file mode 100644 index 00000000..d3386be5 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. +/// +public enum BlobDeleteResult +{ + /// + /// The blob was deleted, or was already gone. The payload should be acknowledged so the backend can + /// hard-delete the row. + /// + Deleted, + + /// + /// The token is permanently invalid (poison) and can never be deleted. The payload should still be + /// acknowledged so the backend clears the stuck row instead of re-streaming the same token forever. + /// + Discarded, + + /// + /// A transient failure occurred. The payload is left tombstoned so a later purge cycle can retry it. + /// + Retry, +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs deleted file mode 100644 index e49e423c..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobCreationOptions.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Options used to create the singleton blob payload auto-purge job. -/// -/// -/// The maximum number of tombstoned payloads to request from the backend per cycle. -/// -public sealed record BlobPurgeJobCreationOptions(int PurgeBatchSize); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs index 917b8d1b..97c37e66 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -9,18 +9,13 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public enum BlobPurgeJobStatus { /// - /// The job is not running. This is the default status of a freshly initialized entity, so it is kept - /// as the zero value to avoid a brand-new entity accidentally appearing active. + /// The job has not been started yet. This is the default status of a freshly initialized entity, so it is + /// kept as the zero value to avoid a brand-new entity accidentally appearing active. /// - Stopped, + Pending, /// /// The job is active and draining tombstoned payloads from the backend. /// Active, - - /// - /// The job has failed. - /// - Failed, } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 2b7ba6f2..bf3db1d4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -26,8 +26,8 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator(); string jobId = input.JobEntityId.Key; - int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : DefaultPurgeBatchSize; + int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; int processedCycles = input.ProcessedCycles; while (true) @@ -54,92 +54,103 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator( - input.JobEntityId, nameof(BlobPurgeJob.Get), null); - - if (state is null || state.Status != BlobPurgeJobStatus.Active) + try { - logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); - return null; - } - - List tombstones = await context.CallActivityAsync>( - nameof(GetTombstonedPayloadsActivity), - batchSize, - new TaskOptions(PurgeActivityRetryPolicy)); + // Stop cleanly if the job has been stopped or removed. + BlobPurgeJobState? state = await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.Get), null); + + if (state is null || state.Status != BlobPurgeJobStatus.Active) + { + logger.BlobPurgeJobOrchestratorStopping(jobId, state?.Status.ToString() ?? "null"); + return null; + } + + List tombstones = await context.CallActivityAsync>( + nameof(GetTombstonedPayloadsActivity), + batchSize, + new TaskOptions(PurgeActivityRetryPolicy)); - if (tombstones is null || tombstones.Count == 0) - { - // Nothing to purge right now: block on a timer (push-free idle) then check again. - await context.CreateTimer(IdleDelay, default); - continue; + if (tombstones is null || tombstones.Count == 0) + { + // Nothing to purge right now: block on a timer (push-free idle) then check again. + await context.CreateTimer(IdleDelay, default); + continue; + } + + List acks = await this.DeleteBatchAsync(context, tombstones); + + if (acks.Count > 0) + { + await context.CallActivityAsync( + nameof(AckPurgedPayloadsActivity), + acks, + new TaskOptions(PurgeActivityRetryPolicy)); + + await context.Entities.CallEntityAsync( + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); + } } - - List deleted = await this.DeleteBatchAsync(context, tombstones); - - if (deleted.Count > 0) + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - await context.CallActivityAsync( - nameof(AckPurgedPayloadsActivity), - deleted, - new TaskOptions(PurgeActivityRetryPolicy)); - - await context.Entities.CallEntityAsync( - input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)deleted.Count); + // A single bad cycle (transient backend/entity/activity failure) must not kill the perpetual + // loop. Log, back off, then continue so the job self-heals and keeps draining. + logger.BlobPurgeCycleFailed(ex, jobId); + await context.CreateTimer(ErrorBackoff, default); + continue; } } } - async Task> DeleteBatchAsync( - TaskOrchestrationContext context, List tombstones) + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) { - List deleted = new(tombstones.Count); + List acks = new(tombstones.Count); List> tasks = new(); - foreach (TombstonedPayloadDto tombstone in tombstones) + foreach (TombstonedPayload tombstone in tombstones) { tasks.Add(this.DeleteOneAsync(context, tombstone)); if (tasks.Count >= MaxParallelDeletes) { - await DrainAsync(tasks, deleted); + await DrainAsync(tasks, acks); tasks.Clear(); } } if (tasks.Count > 0) { - await DrainAsync(tasks, deleted); + await DrainAsync(tasks, acks); } - return deleted; + return acks; } - static async Task DrainAsync(List> tasks, List deleted) + static async Task DrainAsync(List> tasks, List acks) { DeleteOutcome[] outcomes = await Task.WhenAll(tasks); foreach (DeleteOutcome outcome in outcomes) { - // Only acknowledge blobs that were actually deleted; failed tokens stay tombstoned to retry. - if (outcome.Deleted) + // Acknowledge blobs that were deleted (or already gone) and poison tokens that can never succeed + // so the backend can hard-delete their rows; transient failures stay tombstoned to retry. + if (outcome.ShouldAck) { - deleted.Add(outcome.Ack); + acks.Add(outcome.Ack); } } } - async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayloadDto tombstone) + async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayload tombstone) { - bool deleted = await context.CallActivityAsync( + BlobDeleteResult result = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), - tombstone.Token, - new TaskOptions(PurgeActivityRetryPolicy)); + tombstone.Token); return new DeleteOutcome( - deleted, - new PayloadPurgeAckDto(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + result != BlobDeleteResult.Retry, + new PayloadPurgeAck(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); } - readonly record struct DeleteOutcome(bool Deleted, PayloadPurgeAckDto Ack); + readonly record struct DeleteOutcome(bool ShouldAck, PayloadPurgeAck Ack); } diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index f53c9e72..582e3326 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using Azure.Core; +using Microsoft.DurableTask.AzureBlobPayloads; // Intentionally no DataAnnotations to avoid extra package requirements in minimal hosts. namespace Microsoft.DurableTask; @@ -128,5 +129,5 @@ public int ThresholdBytes /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. /// - public int PayloadPurgeBatchSize { get; set; } = 500; + public int PayloadPurgeBatchSize { get; set; } = BlobPurgeConstants.DefaultBatchSize; } diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 126afdf8..f3a727d6 100644 --- a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -19,7 +19,7 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), - new BlobPurgeJobCreationOptions(250)); + 250); // Act await this.job.RunAsync(operation); @@ -45,7 +45,7 @@ public async Task Create_WhenAlreadyActive_IsNoOp() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(existing), - new BlobPurgeJobCreationOptions(999)); + 999); // Act await this.job.RunAsync(operation); @@ -64,7 +64,7 @@ public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), - new BlobPurgeJobCreationOptions(0)); + 0); // Act await this.job.RunAsync(operation); From 149c63a9f06ba101bccc701680a7e9fe39d2edfe Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 10:44:23 -0700 Subject: [PATCH 03/32] Refine BlobPurgeJobStarter: pre-check bridge status before rescheduling Avoid needlessly re-running a Completed bridge orchestration on every host restart (fixed id + no dedupe means the backend would purge+replace a terminal instance). Check the existing bridge via GetInstanceAsync and only (re)schedule when it is absent or ended Failed/Terminated, so a failed setup still self-heals. Handle the schedule race with OrchestrationAlreadyExistsException. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 17d11ab0..33a9adec 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using DurableTask.Core.Exceptions; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; @@ -79,8 +80,24 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) try { // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is - // active) and the orchestrator's fixed instance id, so just schedule the bridge once with a - // fixed instance id. Retry only if the backend is unreachable at startup. + // active) and the orchestrator's fixed instance id. The bridge orchestration's only job is to + // apply the entity's Create once, under a fixed instance id. Before (re)scheduling it, check the + // existing bridge: if it already Completed - or is still alive (Running/Pending/Suspended) - the + // job is set up, so do not reschedule. (Re-running a Completed bridge is wasteful: with a fixed + // id and no dedupe policy the backend would purge and replace the terminal instance on every + // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated + // state that may never have applied Create - which lets a failed setup self-heal. + OrchestrationMetadata? existing = await this.client.GetInstanceAsync( + BlobPurgeConstants.StarterInstanceId, cancellationToken); + + bool needsSchedule = existing is null + or { RuntimeStatus: OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated }; + if (!needsSchedule) + { + this.logger.BlobPurgeJobEnsured(); + return; + } + BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); @@ -93,6 +110,13 @@ await this.client.ScheduleNewOrchestrationInstanceAsync( this.logger.BlobPurgeJobEnsured(); return; } + catch (OrchestrationAlreadyExistsException) + { + // Race: another client scheduled the bridge between our status check and schedule call. That is + // fine - the singleton is already kicked off; treat it as ensured and stop. + this.logger.BlobPurgeJobEnsured(); + return; + } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { return; From 4d52005c80e35518c7cef3feba2c4f1e5de50b2d Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 11:19:09 -0700 Subject: [PATCH 04/32] Classify RequestFailedException 400 as permanent in DeleteExternalBlobActivity The Azure Storage SDK already retries transient failures internally (connection errors + HTTP 408/429/5xx with backoff), so an escaped exception means those retries were exhausted. Treating every escaped exception as Retry mis-classified permanent service rejections (e.g. Status 400 InvalidResourceName from a malformed decoded blob name) as transient, causing an infinite re-drain of a poison token. Add a RequestFailedException Status 400 -> Discarded branch (ack so the backend clears the row); keep 403/408/429/5xx/timeouts/cancellation as Retry. Document the doc-verified exception model on the activity and add focused tests for the 400 -> Discarded and non-400 -> Retry outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Activities/DeleteExternalBlobActivity.cs | 36 +++++++++++- .../DeleteExternalBlobActivityTests.cs | 58 +++++++++++++++++++ 2 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index f070f964..6452857c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -1,15 +1,39 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Azure; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. Malformed (poison) tokens are discarded so they get -/// acknowledged instead of retried forever; transient failures leave the payload tombstoned to retry. +/// re-delivered tokens and concurrent workers are safe. /// +/// +/// Outcome classification, verified against the Azure.Storage.Blobs / Azure.Core exception model (not +/// assumed): +/// +/// +/// The Azure SDK already retries transient failures internally (connection errors plus HTTP +/// 408/429/500/502/503/504, with exponential backoff), so any exception that escapes +/// means those built-in retries were already exhausted. +/// +/// +/// Permanent failures are discarded (acked so the backend clears the row): an +/// from the store's own token decode / container-mismatch check (thrown client-side before any network call), +/// and a with 400 (for +/// example InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules). Retrying +/// either can never succeed. +/// +/// +/// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: +/// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator +/// credential fix rather than dropping data), and timeouts / cancellation. A blob is never dropped on an +/// uncertain error, and a single bad token never fails the whole batch. +/// +/// +/// /// The payload store used to delete blobs. /// The logger instance. [DurableTask] @@ -38,6 +62,14 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (RequestFailedException ex) when (ex.Status == 400) + { + // Service rejected the request as permanently invalid (e.g. InvalidUri / InvalidResourceName - the + // decoded blob name violates Azure naming rules). Retrying can never succeed, so discard it like a + // poison token: ack so the backend clears the row instead of re-streaming it forever. + this.logger.BlobPurgeDeleteDiscarded(ex, input); + return BlobDeleteResult.Discarded; + } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs new file mode 100644 index 00000000..6ca963cb --- /dev/null +++ b/test/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure; +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class DeleteExternalBlobActivityTests +{ + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken() + { + // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. + StubPayloadStore store = new(new RequestFailedException(400, "InvalidResourceName")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:bad name"); + + // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. + result.Should().Be(BlobDeleteResult.Discarded); + } + + [Fact] + public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedForRetry() + { + // Arrange - a Status 503 that escaped the SDK's internal retries is treated as transient. + StubPayloadStore store = new(new RequestFailedException(503, "ServerBusy")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + + // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. + result.Should().Be(BlobDeleteResult.Retry); + } + + sealed class StubPayloadStore : PayloadStore + { + readonly Exception? deleteError; + + public StubPayloadStore(Exception? deleteError) => this.deleteError = deleteError; + + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.deleteError is null ? Task.CompletedTask : throw this.deleteError; + + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => true; + } +} From 780d7430ae286f533c87c9b01f01f98cdb482a26 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 13:25:12 -0700 Subject: [PATCH 05/32] Reuse shared PayloadStore and register purge starter conditionally on AutoPurge Address review #B (store duplication) and #C (conditional-DI of the starter): - TryAddSingleton the PayloadStore so we reuse an already-registered shared store instead of creating a second, redundant one. - Register the auto-purge starter only when the configure overload opted into AutoPurge (peek the flag via a probe options instance); UseExternalizedPayloadsCore no longer registers it unconditionally, so the no-arg overload never starts it. - The starter no longer self-checks the flag in StartAsync (registration already gates it); remove the now-unused BlobPurgeDisabled log and refresh its xmldoc. - Add DI-registration tests proving the starter is registered iff AutoPurge is on. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 15 ++--- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 - ...ientBuilderExtensions.AzureBlobPayloads.cs | 22 +++++-- .../AzureBlobPayloads.Tests.csproj | 1 + .../UseExternalizedPayloadsTests.cs | 62 +++++++++++++++++++ 5 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 33a9adec..23176ef8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,10 +11,11 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists when auto-purge -/// is enabled via . It never blocks host startup: it runs -/// on a background task and retries until the backend is reachable. The job is a whole-scheduler singleton, -/// so racing client processes simply no-op. +/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered +/// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), +/// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and +/// retries until the backend is reachable. The job is a whole-scheduler singleton, so racing client processes +/// simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService { @@ -45,12 +46,6 @@ public BlobPurgeJobStarter( public Task StartAsync(CancellationToken cancellationToken) { LargePayloadStorageOptions opts = this.options.Get(this.builderName); - if (!opts.AutoPurge) - { - this.logger.BlobPurgeDisabled(); - return Task.CompletedTask; - } - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 3b7ccc98..b217f53e 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -28,9 +28,6 @@ static partial class Logs [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); - [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled; the singleton purge job will not be started.")] - public static partial void BlobPurgeDisabled(this ILogger logger); - [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 2557c6da..78e4d47f 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,6 +8,7 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -33,13 +34,28 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( Check.NotNull(configure); builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => + + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => { LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); return new BlobPayloadStore(opts); }); - return UseExternalizedPayloadsCore(builder); + UseExternalizedPayloadsCore(builder); + + // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the + // flag now by running the configure delegate against a probe (options configurators are pure setters). + LargePayloadStorageOptions probe = new(); + configure(probe); + if (probe.AutoPurge) + { + RegisterBlobPurgeJobStarter(builder); + } + + return builder; } /// @@ -82,8 +98,6 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); - RegisterBlobPurgeJobStarter(builder); - return builder; } diff --git a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj index 77078cbb..b650cb59 100644 --- a/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj +++ b/test/AzureBlobPayloads.Tests/AzureBlobPayloads.Tests.csproj @@ -14,6 +14,7 @@ + diff --git a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs new file mode 100644 index 00000000..b3343078 --- /dev/null +++ b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Moq; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.DependencyInjection; + +public class UseExternalizedPayloadsTests +{ + [Fact] + public void UseExternalizedPayloads_WithAutoPurgeEnabled_RegistersHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act + builder.Object.UseExternalizedPayloads(options => options.AutoPurge = true); + + // Assert - the purge-job starter is the only IHostedService this path registers. + services.Should().ContainSingle(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - auto-purge left at its default (false). + builder.Object.UseExternalizedPayloads(options => { }); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } + + [Fact] + public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStarter() + { + // Arrange + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - the shared-store overload never enables auto-purge. + builder.Object.UseExternalizedPayloads(); + + // Assert + services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + } +} From 3a2215cd9796268ef56064f4603f1a3964fdc17f Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 14:07:32 -0700 Subject: [PATCH 06/32] Register fallback PayloadStore in shared Core for both client and worker The store's consumer is the interceptor wired up in UseExternalizedPayloadsCore, so Core is the single fallback registration site. Move the TryAddSingleton out of each configure overload and into Core, symmetrically for the client and worker extensions. TryAdd keeps reusing a shared store (AddExternalizedPayloadStore or the sibling builder) and never creates a redundant one. The worker overload switches from AddSingleton to the shared TryAddSingleton in Core as part of the move. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...lientBuilderExtensions.AzureBlobPayloads.cs | 18 +++++++++--------- ...orkerBuilderExtensions.AzureBlobPayloads.cs | 15 ++++++++++----- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 78e4d47f..0c485208 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -35,15 +35,6 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( builder.Services.Configure(builder.Name, configure); - // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or - // the worker builder in the same process); only register our own as a fallback so we never create a - // second, redundant PayloadStore. - builder.Services.TryAddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - UseExternalizedPayloadsCore(builder); // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the @@ -73,6 +64,15 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index 42ac0f0a..c63db137 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -6,6 +6,7 @@ using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using P = Microsoft.DurableTask.Protobuf; @@ -30,11 +31,6 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( Check.NotNull(configure); builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); return UseExternalizedPayloadsCore(builder); } @@ -54,6 +50,15 @@ public static IDurableTaskWorkerBuilder UseExternalizedPayloads( static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the client builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC worker builder.Services .AddOptions(builder.Name) From 47651dc0e93d17a67dc145f14d028df019782590 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 14 Jul 2026 14:39:58 -0700 Subject: [PATCH 07/32] Stop self-registering PayloadStore on the client; consume the shared store Restore the original design: the client never registers a PayloadStore, it only consumes a shared/external one (via AddExternalizedPayloadStore or an in-process worker). Only the worker self-registers a fallback store. Remove the client Core's TryAddSingleton block (and its now-unused Microsoft.Extensions.DependencyInjection.Extensions using); the client Core PostConfigure still resolves PayloadStore from the shared/worker registration. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- ...bleTaskClientBuilderExtensions.AzureBlobPayloads.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0c485208..f811d6da 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,7 +8,6 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -64,15 +63,6 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { - // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or - // the worker builder in the same process); only register our own as a fallback so we never create a - // second, redundant PayloadStore. - builder.Services.TryAddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) From 7397fa64e71b88150087b551580944dce0e8cc2f Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 10:02:11 -0700 Subject: [PATCH 08/32] Validate PayloadPurgeBatchSize once at specification (fail fast on out-of-range); drop redundant downstream coercions Validate the batch size at its single point of specification - LargePayloadStorageOptions.PayloadPurgeBatchSize - and fail fast when out of range, mirroring the existing ThresholdBytes setter. Valid range is 1..999 inclusive, matching the gRPC GetTombstonedPayloadsAsync contract (which rejects limits >= 1000); add BlobPurgeConstants.MaxBatchSize = 999 to express the upper bound. Now that the value is guaranteed valid at specification, remove the redundant '> 0 ? x : DefaultBatchSize' coercions from every downstream consumer (starter, entity, orchestrator, activity); they simply use the value directly. Tests: add setter validation tests (throws for 0, -1, 1000, 1001; accepts 1, 500, 999) and repurpose the entity's non-positive test to assert the entity now stores the batch size verbatim (no coercion). Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../GetTombstonedPayloadsActivity.cs | 2 +- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 2 +- .../AutoPurge/Constants/BlobPurgeConstants.cs | 8 ++++- .../AutoPurge/Entity/BlobPurgeJob.cs | 2 +- .../BlobPurgeJobOrchestrator.cs | 2 +- .../Options/LargePayloadStorageOptions.cs | 20 +++++++++-- .../AutoPurge/BlobPurgeJobTests.cs | 8 +++-- .../LargePayloadStorageOptionsTests.cs | 33 +++++++++++++++++++ 8 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs index 2b1bad7d..838b293a 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs @@ -23,7 +23,7 @@ public class GetTombstonedPayloadsActivity( /// public override async Task> RunAsync(TaskActivityContext context, int input) { - int limit = input > 0 ? input : BlobPurgeConstants.DefaultBatchSize; + int limit = input; List payloads = await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); this.logger.BlobPurgeFetchedTombstones(payloads.Count); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 23176ef8..bc3e18a5 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -46,7 +46,7 @@ public BlobPurgeJobStarter( public Task StartAsync(CancellationToken cancellationToken) { LargePayloadStorageOptions opts = this.options.Get(this.builderName); - int batchSize = opts.PayloadPurgeBatchSize > 0 ? opts.PayloadPurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index 3697f199..48174f52 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -16,10 +16,16 @@ static class BlobPurgeConstants /// /// The default number of tombstoned payloads the auto-purge job requests from the backend per cycle, - /// used whenever a configured batch size is missing or non-positive. + /// used whenever a batch size is not explicitly configured. /// public const int DefaultBatchSize = 500; + /// + /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC + /// GetTombstonedPayloadsAsync contract, which rejects limits >= 1000. + /// + public const int MaxBatchSize = 999; + /// /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the /// singleton job. A fixed ID keeps racing client processes from creating duplicate bridge orchestrations. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index c2103fa6..89391890 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -30,7 +30,7 @@ public void Create(TaskEntityContext context, int purgeBatchSize) } this.State.Status = BlobPurgeJobStatus.Active; - this.State.PurgeBatchSize = purgeBatchSize > 0 ? purgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + this.State.PurgeBatchSize = purgeBatchSize; this.State.CreatedAt ??= DateTimeOffset.UtcNow; this.State.LastModifiedAt = DateTimeOffset.UtcNow; this.State.LastError = null; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index bf3db1d4..8ba4f707 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -42,7 +42,7 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator(); string jobId = input.JobEntityId.Key; - int batchSize = input.PurgeBatchSize > 0 ? input.PurgeBatchSize : BlobPurgeConstants.DefaultBatchSize; + int batchSize = input.PurgeBatchSize; int processedCycles = input.ProcessedCycles; while (true) diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 582e3326..cbd63835 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -28,6 +28,7 @@ namespace Microsoft.DurableTask; public sealed class LargePayloadStorageOptions { int thresholdBytes = 256 * 1024; + int payloadPurgeBatchSize = BlobPurgeConstants.DefaultBatchSize; /// /// Initializes a new instance of the class. @@ -127,7 +128,22 @@ public int ThresholdBytes /// /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend - /// per cycle. Defaults to 500. Values less than or equal to zero are treated as the default. + /// per cycle. Must be between 1 and 999 (inclusive); values outside this range throw + /// . Defaults to 500. /// - public int PayloadPurgeBatchSize { get; set; } = BlobPurgeConstants.DefaultBatchSize; + public int PayloadPurgeBatchSize + { + get => this.payloadPurgeBatchSize; + set + { + if (value < 1 || value > BlobPurgeConstants.MaxBatchSize) + { + throw new ArgumentOutOfRangeException( + nameof(this.PayloadPurgeBatchSize), value, + $"PayloadPurgeBatchSize must be between 1 and {BlobPurgeConstants.MaxBatchSize} (inclusive)."); + } + + this.payloadPurgeBatchSize = value; + } + } } diff --git a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index f3a727d6..2117299c 100644 --- a/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -58,9 +58,11 @@ public async Task Create_WhenAlreadyActive_IsNoOp() } [Fact] - public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() + public async Task Create_StoresBatchSizeVerbatim_WithoutCoercion() { - // Arrange + // Arrange - the batch size is validated once at specification (LargePayloadStorageOptions), so the + // entity trusts its input and performs no coercion of its own. A zero here is stored as-is, proving + // the previous non-positive-to-default fallback was removed. TestEntityOperation operation = new( nameof(BlobPurgeJob.Create), new TestEntityState(null), @@ -72,7 +74,7 @@ public async Task Create_WithNonPositiveBatchSize_FallsBackToDefault() // Assert BlobPurgeJobState state = Assert.IsType( operation.State.GetState(typeof(BlobPurgeJobState))); - state.PurgeBatchSize.Should().Be(500); + state.PurgeBatchSize.Should().Be(0); } [Fact] diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs index 317b1e1e..6a4a95ed 100644 --- a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -18,4 +18,37 @@ public void Defaults_AutoPurgeDisabled_AndBatchSize500() options.AutoPurge.Should().BeFalse(); options.PayloadPurgeBatchSize.Should().Be(500); } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(1000)] + [InlineData(1001)] + public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) + { + // Arrange + LargePayloadStorageOptions options = new(); + + // Act + Action act = () => options.PayloadPurgeBatchSize = value; + + // Assert + act.Should().Throw(); + } + + [Theory] + [InlineData(1)] + [InlineData(500)] + [InlineData(999)] + public void PayloadPurgeBatchSize_InRange_IsAccepted(int value) + { + // Arrange + LargePayloadStorageOptions options = new(); + + // Act + options.PayloadPurgeBatchSize = value; + + // Assert + options.PayloadPurgeBatchSize.Should().Be(value); + } } From 4afeb8a93aa05cb46a3098c4f0d54656c583c938 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 10:03:51 -0700 Subject: [PATCH 09/32] Translate gRPC Cancelled to OperationCanceledException in GetTombstonedPayloadsAsync/AckPurgedPayloadsAsync Both new methods previously let RpcException(StatusCode.Cancelled) escape, unlike every other method in this client. Wrap each method's gRPC call in a try/catch that translates Cancelled to OperationCanceledException, mirroring the existing pattern used by GetOrchestrationHistoryAsync and the other client methods. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Grpc/GrpcDurableTaskClient.cs | 25 ++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 8be31fd5..ca9b51ae 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -634,9 +634,18 @@ public override async Task> GetTombstonedPayloadsAsync( nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); } - P.GetTombstonedPayloadsResponse response = await this.sidecarClient.GetTombstonedPayloadsAsync( - new P.GetTombstonedPayloadsRequest { Limit = limit }, - cancellationToken: cancellation); + P.GetTombstonedPayloadsResponse response; + try + { + response = await this.sidecarClient.GetTombstonedPayloadsAsync( + new P.GetTombstonedPayloadsRequest { Limit = limit }, + cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.GetTombstonedPayloadsAsync)} operation was canceled.", e, cancellation); + } List result = new(response.Payloads.Count); foreach (P.TombstonedPayload payload in response.Payloads) @@ -670,7 +679,15 @@ public override async Task AckPurgedPayloadsAsync( return; } - await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + try + { + await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.AckPurgedPayloadsAsync)} operation was canceled.", e, cancellation); + } } static AsyncDisposable GetCallInvoker(GrpcDurableTaskClientOptions options, ILogger logger, out CallInvoker callInvoker) From e74f6330ab6de3fe6accdfab1892a7e6752e5a9a Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 15 Jul 2026 11:05:17 -0700 Subject: [PATCH 10/32] Raise auto-purge MaxBatchSize to 1000 (inclusive); relax gRPC GetTombstonedPayloads limit check to allow 1000 1000 is a valid, accepted batch size end-to-end: the proto documents no numeric range and the backend only clamps via min(limit, configuredCap) without rejecting >= 1000, so the whole range is contained in this SDK. Raise BlobPurgeConstants.MaxBatchSize from 999 to 1000 (the options setter already validates against it) and relax the gRPC GetTombstonedPayloadsAsync guard from 'limit >= 1000' to 'limit > 1000'. Update the two PayloadPurgeBatchSize theories so 1000 is asserted accepted and 1001 still throws. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Grpc/GrpcDurableTaskClient.cs | 4 ++-- .../AutoPurge/Constants/BlobPurgeConstants.cs | 4 ++-- .../AzureBlobPayloads/Options/LargePayloadStorageOptions.cs | 2 +- .../Options/LargePayloadStorageOptionsTests.cs | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index ca9b51ae..6e79b6b2 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -628,10 +628,10 @@ public override async Task> GetOrchestrationHistoryAsync( public override async Task> GetTombstonedPayloadsAsync( int limit, CancellationToken cancellation = default) { - if (limit <= 0 || limit >= 1000) + if (limit <= 0 || limit > 1000) { throw new ArgumentOutOfRangeException( - nameof(limit), limit, "Limit must be greater than 0 and less than 1000."); + nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); } P.GetTombstonedPayloadsResponse response; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index 48174f52..ca4cce32 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -22,9 +22,9 @@ static class BlobPurgeConstants /// /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC - /// GetTombstonedPayloadsAsync contract, which rejects limits >= 1000. + /// GetTombstonedPayloadsAsync contract, which rejects limits greater than 1000. /// - public const int MaxBatchSize = 999; + public const int MaxBatchSize = 1000; /// /// The fixed instance ID of the client-to-entity bridge orchestration the starter schedules to ensure the diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index cbd63835..0edaee90 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -128,7 +128,7 @@ public int ThresholdBytes /// /// Gets or sets the maximum number of tombstoned payloads the auto-purge job requests from the backend - /// per cycle. Must be between 1 and 999 (inclusive); values outside this range throw + /// per cycle. Must be between 1 and 1000 (inclusive); values outside this range throw /// . Defaults to 500. /// public int PayloadPurgeBatchSize diff --git a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs index 6a4a95ed..d608a936 100644 --- a/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs +++ b/test/AzureBlobPayloads.Tests/Options/LargePayloadStorageOptionsTests.cs @@ -22,7 +22,6 @@ public void Defaults_AutoPurgeDisabled_AndBatchSize500() [Theory] [InlineData(0)] [InlineData(-1)] - [InlineData(1000)] [InlineData(1001)] public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) { @@ -40,6 +39,7 @@ public void PayloadPurgeBatchSize_OutOfRange_Throws(int value) [InlineData(1)] [InlineData(500)] [InlineData(999)] + [InlineData(1000)] public void PayloadPurgeBatchSize_InRange_IsAccepted(int value) { // Arrange From 50ae944676378bd3da37591fb16073aadce33ff0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Thu, 30 Jul 2026 10:54:51 -0700 Subject: [PATCH 11/32] Register PayloadStore in the client builder extension (symmetry with worker; fixes client-only DI failure) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- ...ientBuilderExtensions.AzureBlobPayloads.cs | 10 +++++++++ .../UseExternalizedPayloadsTests.cs | 21 +++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index f811d6da..0c485208 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -8,6 +8,7 @@ using Microsoft.DurableTask.Converters; using Microsoft.DurableTask.Worker.Grpc.Internal; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -63,6 +64,15 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientBuilder builder) { + // Reuse the shared payload store when one is already registered (e.g. via AddExternalizedPayloadStore or + // the worker builder in the same process); only register our own as a fallback so we never create a + // second, redundant PayloadStore. + builder.Services.TryAddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + // Wrap the gRPC CallInvoker with our interceptor when using the gRPC client builder.Services .AddOptions(builder.Name) diff --git a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index b3343078..2ee439ef 100644 --- a/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -59,4 +59,25 @@ public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStar // Assert services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); } + + [Fact] + public void UseExternalizedPayloads_ClientOnly_RegistersResolvablePayloadStore() + { + // Arrange - a client-only host with no worker and no explicit AddExternalizedPayloadStore. This is the + // exact shape that previously failed: the core method declared a PostConfigure dependency on + // PayloadStore without ever registering it, so options resolution threw at runtime. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + // Act - UseDevelopmentStorage=true is a valid connection string that BlobServiceClient accepts with no + // network I/O, so the store constructs offline. Build the provider and actually resolve PayloadStore. + builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the store resolves without throwing and is the blob-backed implementation. + PayloadStore store = provider.GetRequiredService(); + store.Should().BeOfType(); + } } From a5ed2989f8e9c7d17647bd37e9f6bdad5b888e64 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 12:59:34 -0700 Subject: [PATCH 12/32] Resolve v2 tokens in DeleteAsync and discard payloads in unreachable accounts The blob delete activity now discards (acks) a payload whose v2 token points at a storage account the configured credential cannot reach: BlobPayloadStore throws PayloadStorageException for that cross-account case, and since the backend batch is a cursor-less SELECT TOP (N), retrying such a permanently unreachable row would re-stream it every cycle and block the pipeline head-of-line. It is discarded and logged at error level (EventId 821) instead. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 24 +++++++++++++++---- .../AzureBlobPayloads/AutoPurge/Logs.cs | 3 +++ .../DeleteExternalBlobActivityTests.cs | 16 +++++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 6452857c..76490237 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -20,11 +20,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// means those built-in retries were already exhausted. /// /// -/// Permanent failures are discarded (acked so the backend clears the row): an -/// from the store's own token decode / container-mismatch check (thrown client-side before any network call), -/// and a with 400 (for -/// example InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules). Retrying -/// either can never succeed. +/// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: +/// an from the store's token decode - a genuinely malformed token, or a legacy +/// v1 token whose container does not match the configured store (self-describing v2 tokens are not conflated +/// into this: a different account/container is either handled or surfaced as the store exception below); a +/// with 400 (for example +/// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a +/// when a v2 token points at a storage account the configured credential +/// cannot reach (connection-string / account-key auth is account-specific). The backend batch is cursor-less, +/// so an undroppable row would otherwise re-stream every cycle and block the pipeline head-of-line; the +/// account-unreachable case is logged at error level so an operator can reconcile it. /// /// /// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: @@ -70,6 +75,15 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (PayloadStorageException ex) + { + // The payload lives in a storage account this worker's credential cannot reach (connection-string / + // account-key auth is account-specific). Retrying can never succeed and the batch is cursor-less, so a + // permanently unreachable row would re-stream every cycle and block later rows. Discard it and log + // loudly so an operator can reconfigure identity auth and delete the blob out of band. + this.logger.BlobPurgeDeleteOrphanedUnreachableAccount(ex, input); + return BlobDeleteResult.Discarded; + } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index b217f53e..7ef2e16b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -39,4 +39,7 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); + + [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' was orphaned: its blob lives in a storage account the configured credential cannot reach (cross-account deletes require identity/AAD auth; connection-string / account-key auth is account-specific). Discarding it so the purge pipeline is not blocked; reconfigure identity auth with access to that account and delete the blob out of band.")] + public static partial void BlobPurgeDeleteOrphanedUnreachableAccount(this ILogger logger, Exception exception, string token); } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 6ca963cb..5e3004cb 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -38,6 +38,22 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedF result.Should().Be(BlobDeleteResult.Retry); } + [Fact] + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnblockPipeline() + { + // Arrange - the payload lives in a storage account the configured credential cannot reach. Retrying can + // never succeed and the backend batch is cursor-less, so a permanently unreachable row would re-stream + // every cycle and block later rows; it must be discarded (acked), not retried. + StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + + // Assert - discarded so the pipeline head-of-line is not blocked by an undeletable payload. + result.Should().Be(BlobDeleteResult.Discarded); + } + sealed class StubPayloadStore : PayloadStore { readonly Exception? deleteError; From fff06b02f81e593804003aed4218659c4ad75876 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 13:08:05 -0700 Subject: [PATCH 13/32] Align unreachable-account log, exception wording and v2 delete test names with review spec Rename the cross-account orphan log to BlobPurgeDeleteUnreachable (EventId 821, Error) with the reviewer-specified message, mirror that exact wording in the DeleteExternalBlobActivity PayloadStorageException catch, reword the BlobPayloadStore cross-account message ("cannot delete in another account"), and rename the three v2 DeleteAsync tests to the reviewer-specified names. No behavior change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 11 ++++++----- src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs | 4 ++-- .../PayloadStore/BlobPayloadStore.cs | 2 +- .../PayloadStore/BlobPayloadStoreDeleteTests.cs | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 76490237..2935cf68 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -77,11 +77,12 @@ public override async Task RunAsync(TaskActivityContext contex } catch (PayloadStorageException ex) { - // The payload lives in a storage account this worker's credential cannot reach (connection-string / - // account-key auth is account-specific). Retrying can never succeed and the batch is cursor-less, so a - // permanently unreachable row would re-stream every cycle and block later rows. Discard it and log - // loudly so an operator can reconfigure identity auth and delete the blob out of band. - this.logger.BlobPurgeDeleteOrphanedUnreachableAccount(ex, input); + // The token is well-formed but points at a storage account this worker's credential cannot reach + // (cross-account without AAD). Retrying can never succeed from this process, and because the backend + // streams tombstones with an uncursored TOP(N) query, leaving it un-acked would re-serve the same + // token every cycle and permanently block the purge pipeline. Discard it so the row is cleared, and + // log at Error so an operator can reclaim the orphaned blob out-of-band. + this.logger.BlobPurgeDeleteUnreachable(ex, input); return BlobDeleteResult.Discarded; } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 7ef2e16b..5238496d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -40,6 +40,6 @@ static partial class Logs [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); - [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' was orphaned: its blob lives in a storage account the configured credential cannot reach (cross-account deletes require identity/AAD auth; connection-string / account-key auth is account-specific). Discarding it so the purge pipeline is not blocked; reconfigure identity auth with access to that account and delete the blob out of band.")] - public static partial void BlobPurgeDeleteOrphanedUnreachableAccount(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] + public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 305c2185..7a9cbacb 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -231,7 +231,7 @@ public override async Task DeleteAsync(string token, CancellationToken cancellat $"The externalized payload lives in a different storage account ('{decoded.ContainerUri}') than the " + $"currently-configured payload store ('{this.containerClient.Uri}'). Cross-account payload deletes " + "require identity (AAD) authentication with access to both accounts; connection-string / " + - "account-key credentials are account-specific and cannot delete from another account."); + "account-key credentials are account-specific and cannot delete in another account."); } // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs index bdf238e6..7ccab24a 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs @@ -105,7 +105,7 @@ public async Task DeleteAsync_InvalidToken_ThrowsArgumentException(string token) } [Fact] - public async Task DeleteAsync_V2TokenSameAccountAndContainer_DeletesViaConfiguredContainerClient() + public async Task DeleteAsync_V2TokenSameContainer_DeletesViaConfiguredClient() { // Arrange - a self-describing v2 token whose account+container match the configured store. The store // recognizes it via IsConfiguredContainer and deletes through the existing container client (which works @@ -125,7 +125,7 @@ public async Task DeleteAsync_V2TokenSameAccountAndContainer_DeletesViaConfigure } [Fact] - public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_UsesCrossAccountClientNotConfiguredContainer() + public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_DoesNotUseConfiguredContainer() { // Arrange - a v2 token pointing at a DIFFERENT account than the configured store, with identity auth // available. The store must build a BlobClient bound to the token's own account using the credential and @@ -151,7 +151,7 @@ public async Task DeleteAsync_V2TokenDifferentAccountWithCredential_UsesCrossAcc } [Fact] - public async Task DeleteAsync_V2TokenDifferentAccountWithoutCredential_ThrowsPayloadStorageException() + public async Task DeleteAsync_V2TokenDifferentAccountWithoutCredential_ThrowsPayloadStorageExceptionAndDoesNotDelete() { // Arrange - the configured store uses a connection string (account-key auth, no TokenCredential) and the // token points at a different account. Account keys are account-specific, so the delete cannot cross From 6e4f5d08568df270bf7c89ce00fd7172b9df4027 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 14:16:49 -0700 Subject: [PATCH 14/32] Gate blob auto-purge on v2 tokens and deleting stores Decision 1: the auto-purge job declines to act on legacy v1 tokens. DeleteExternalBlobActivity.RunAsync discards v1 tokens up front - logged at error as the operator recovery pointer - without calling the store, because a v1 token identifies no storage account and a delete cannot be verified. BlobPayloadStore.TokenPrefixV1 is made internal so the activity can reference it; BlobPayloadStore.DeleteAsync is unchanged and still honors v1 for direct callers. Decision 2: do not start the purge job when the registered store cannot delete. BlobPurgeJobStarter now takes the resolved PayloadStore and no-ops at startup (logged at error, does not throw) when it is not a BlobPayloadStore. Defense in depth: the activity catches NotSupportedException and returns Retry to keep the payload tombstoned rather than acking a row whose blob was never deleted. Adds Logs 822/823/824, XML doc updates, a per-task-hub singleton phrasing fix, and focused tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Activities/DeleteExternalBlobActivity.cs | 42 ++++++++-- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 16 +++- .../AutoPurge/Entity/BlobPurgeJob.cs | 2 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 9 +++ ...ientBuilderExtensions.AzureBlobPayloads.cs | 1 + .../Options/LargePayloadStorageOptions.cs | 6 ++ .../PayloadStore/BlobPayloadStore.cs | 2 +- .../AutoPurge/BlobPurgeJobStarterTests.cs | 80 +++++++++++++++++++ .../DeleteExternalBlobActivityTests.cs | 52 +++++++++++- 9 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 2935cf68..12438d47 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -20,10 +20,16 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// means those built-in retries were already exhausted. /// /// +/// Legacy blob:v1: tokens are discarded without a delete attempt: a v1 token carries only a container +/// name, not the storage account, so a delete against the currently-configured account cannot be verified - if +/// the store has been repointed the delete would report success while the real blob survives elsewhere. The +/// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked +/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. +/// +/// /// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: -/// an from the store's token decode - a genuinely malformed token, or a legacy -/// v1 token whose container does not match the configured store (self-describing v2 tokens are not conflated -/// into this: a different account/container is either handled or surfaced as the store exception below); a +/// an from the store's token decode - a genuinely malformed or unrecognized +/// token (v1 tokens are gated out above and never reach the store); a /// with 400 (for example /// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a /// when a v2 token points at a storage account the configured credential @@ -34,8 +40,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: /// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator -/// credential fix rather than dropping data), and timeouts / cancellation. A blob is never dropped on an -/// uncertain error, and a single bad token never fails the whole batch. +/// credential fix rather than dropping data), timeouts / cancellation, and a +/// from a misconfigured store that cannot delete at all (retried, not dropped, so the work is recoverable once +/// a deleting store is registered). A blob is never dropped on an uncertain error, and a single bad token never +/// fails the whole batch. /// /// /// @@ -55,6 +63,20 @@ public override async Task RunAsync(TaskActivityContext contex { Check.NotNullOrEmpty(input, nameof(input)); + if (input.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) + { + // Auto-purge deliberately does not act on legacy v1 tokens. A v1 token carries only a container + // *name* - not the storage account - so a delete against the currently-configured account cannot be + // verified: if the store has since been repointed, DeleteIfExistsAsync returns false and the purge + // would silently report success while the real blob survives in the old account. Rather than delete + // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" + // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by + // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the + // pipeline. The full token is logged at error level so it remains a recoverable pointer. + this.logger.BlobPurgeDeleteV1TokenUnsupported(input); + return BlobDeleteResult.Discarded; + } + try { await this.store.DeleteAsync(input, CancellationToken.None); @@ -75,6 +97,16 @@ public override async Task RunAsync(TaskActivityContext contex this.logger.BlobPurgeDeleteDiscarded(ex, input); return BlobDeleteResult.Discarded; } + catch (NotSupportedException ex) + { + // The registered store does not implement deletion (PayloadStore.DeleteAsync is virtual and its base + // implementation throws). This is a misconfiguration, not poison data: every payload would fail the + // same way, so acking would hard-delete the backend's entire record of what still needs cleanup while + // every blob survives. Keep the payload tombstoned so the work is recoverable once an operator + // registers a store that can delete. + this.logger.BlobPurgeDeleteNotSupported(ex, input); + return BlobDeleteResult.Retry; + } catch (PayloadStorageException ex) { // The token is well-formed but points at a storage account this worker's credential cannot reach diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index bc3e18a5..ed8ac7d7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -14,7 +14,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered /// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), /// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and -/// retries until the backend is reachable. The job is a whole-scheduler singleton, so racing client processes +/// retries until the backend is reachable. The job is a per-task-hub singleton, so racing client processes /// simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService @@ -22,6 +22,7 @@ sealed class BlobPurgeJobStarter : IHostedService static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); readonly DurableTaskClient client; + readonly PayloadStore store; readonly IOptionsMonitor options; readonly string builderName; readonly ILogger logger; @@ -32,11 +33,13 @@ sealed class BlobPurgeJobStarter : IHostedService public BlobPurgeJobStarter( DurableTaskClient client, + PayloadStore store, IOptionsMonitor options, string builderName, ILogger logger) { this.client = Check.NotNull(client); + this.store = Check.NotNull(store); this.options = Check.NotNull(options); this.builderName = Check.NotNull(builderName); this.logger = Check.NotNull(logger); @@ -45,6 +48,17 @@ public BlobPurgeJobStarter( /// public Task StartAsync(CancellationToken cancellationToken) { + // Auto-purge deletes blobs through the store, but PayloadStore.DeleteAsync is virtual and its base + // implementation throws NotSupportedException. A store that cannot delete would fail every single + // payload, so refuse to start the job rather than spin against the backend - and rather than ack rows + // whose blobs were never deleted, which would destroy the backend's record of what still needs cleanup. + // This is a configuration error and is surfaced at startup, where it is cheapest to notice. + if (this.store is not BlobPayloadStore) + { + this.logger.BlobPurgeStoreCannotDelete(this.store.GetType().FullName); + return Task.CompletedTask; + } + LargePayloadStorageOptions opts = this.options.Get(this.builderName); int batchSize = opts.PayloadPurgeBatchSize; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 89391890..2d55a979 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,7 +13,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a whole-scheduler singleton, this is + /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this is /// intentionally a no-op when the job is already so that extra /// client processes racing to create it do not disturb the running job. /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 5238496d..a84a2ff8 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -42,4 +42,13 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); + + [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Externalized payload token '{token}' uses the legacy v1 format, which auto-purge does not delete because it does not identify the storage account. The backing blob will NOT be deleted and is now orphaned; the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] + public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); + + [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] + public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); + + [LoggerMessage(EventId = 824, Level = LogLevel.Error, Message = "The registered PayloadStore does not support deleting payloads, so externalized payload token '{token}' cannot be purged. Leaving it tombstoned; register an Azure Blob payload store on the worker or disable AutoPurge.")] + public static partial void BlobPurgeDeleteNotSupported(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index 0c485208..e924cb3e 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -106,6 +106,7 @@ static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) string builderName = builder.Name; builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService>(), builderName, sp.GetRequiredService>())); diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 0edaee90..9339c997 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -124,6 +124,12 @@ public int ThresholdBytes /// the corresponding blobs from customer storage (the backend has no storage credentials of its own). /// Defaults to false (opt-in). /// + /// + /// Auto-purge only deletes blobs referenced by self-describing blob:v2: tokens. Payloads that were + /// externalized by SDK versions that wrote legacy blob:v1: tokens are acknowledged and logged at + /// error level, but their blobs are not deleted: a v1 token does not identify the storage account, so the + /// delete cannot be verified. + /// public bool AutoPurge { get; set; } /// diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 7a9cbacb..ca73812f 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -23,7 +23,7 @@ namespace Microsoft.DurableTask; Justification = "SemaphoreSlim does not allocate a disposable resource unless AvailableWaitHandle is accessed.")] public sealed class BlobPayloadStore : PayloadStore { - const string TokenPrefixV1 = "blob:v1:"; + internal const string TokenPrefixV1 = "blob:v1:"; const string TokenPrefixV2 = "blob:v2:"; const string ContentEncodingGzip = "gzip"; const int MaxRetryAttempts = 8; diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs new file mode 100644 index 00000000..e8617b2b --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Options; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobStarterTests +{ + [Fact] + public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient() + { + // Arrange - the registered store is not the blob store, so its DeleteAsync is unsupported. Every payload + // would fail, so the starter must refuse to run rather than spin against the backend. + Mock client = new("test"); + BlobPurgeJobStarter starter = new( + client.Object, + new NonDeletingPayloadStore(), + OptionsFor(new LargePayloadStorageOptions { AutoPurge = true }), + "test", + new TestLogger()); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - the startup gate short-circuits before scheduling anything, so the client is never queried. + client.Verify( + c => c.GetInstanceAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() + { + // Arrange - the real blob store. UseDevelopmentStorage=true is a valid connection string that constructs + // the store offline (no network I/O), so resolving it at host start is safe. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock client = new("test"); + TestLogger logger = new(); + BlobPurgeJobStarter starter = new( + client.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + "test", + logger); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - the store-cannot-delete gate did NOT fire; the starter proceeded to its background ensure + // path (which runs on a background task, so timing is not asserted). + logger.Logs.Should().NotContain(entry => entry.Message.Contains("is not an Azure Blob payload store")); + + // Cleanup - cancel the background ensure task so it does not outlive the test. + await starter.StopAsync(CancellationToken.None); + } + + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) + { + Mock> monitor = new(); + monitor.Setup(m => m.Get(It.IsAny())).Returns(options); + return monitor.Object; + } + + sealed class NonDeletingPayloadStore : PayloadStore + { + // DeleteAsync is intentionally NOT overridden: the base PayloadStore.DeleteAsync throws + // NotSupportedException, which is exactly the "store cannot delete" configuration the starter refuses. + public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override Task DownloadAsync(string token, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + public override bool IsKnownPayloadToken(string value) => false; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 5e3004cb..93989325 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -18,7 +18,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken( DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:bad name"); + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/bad name"); // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. result.Should().Be(BlobDeleteResult.Discarded); @@ -32,7 +32,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedF DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. result.Should().Be(BlobDeleteResult.Retry); @@ -54,6 +54,54 @@ public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnb result.Should().Be(BlobDeleteResult.Discarded); } + [Fact] + public async Task RunAsync_V1Token_DiscardsWithoutCallingStore() + { + // Arrange - auto-purge policy: a legacy v1 token identifies no storage account, so it is dropped before + // the store is ever consulted. + Mock store = new(); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + + // Assert - discarded by the gate, and the store's DeleteAsync was never invoked. + result.Should().Be(BlobDeleteResult.Discarded); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public async Task RunAsync_V2Token_CallsStore() + { + // Arrange - a self-describing v2 token is not gated and must reach the store. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + + // Assert - the store deleted the blob (proves the gate is v1-only and did not break the happy path). + result.Should().Be(BlobDeleteResult.Deleted); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombstone() + { + // Arrange - a store that cannot delete (the base PayloadStore.DeleteAsync throws NotSupportedException). + StubPayloadStore store = new(new NotSupportedException()); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + + // Assert - retried (tombstone preserved), never discarded: acking would destroy the backend's cleanup + // ledger while the blob survives. + result.Should().Be(BlobDeleteResult.Retry); + result.Should().NotBe(BlobDeleteResult.Discarded); + } + sealed class StubPayloadStore : PayloadStore { readonly Exception? deleteError; From 65e9cbb1c5ca1c773fcba0e17e02f7f61605cfc4 Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 14:53:05 -0700 Subject: [PATCH 15/32] Fix auto-purge starter registration and client resolution Address three reviewer findings on the blob payload auto-purge starter: - Finding 1: register BlobPurgeJobStarter unconditionally from UseExternalizedPayloadsCore (both overloads) instead of probing the configure delegate at registration time. The probe double-invoked user code and missed every enable path except the inline delegate (services.Configure, config binding, PostConfigure, parameterless overload). The AutoPurge decision now happens in StartAsync, once options are fully resolved, and no-ops silently when disabled. - Finding 2: inject IDurableTaskClientProvider and resolve the client by builder name lazily in StartAsync (after both gates) so named clients get their own client and no DurableTaskClient is constructed at host start for apps that externalize payloads without auto-purge. - Finding 3: pass the purge retry policy to the delete activity call in BlobPurgeJobOrchestrator.DeleteOneAsync, matching the other two calls. Tests: update starter tests for the new ctor, add a disabled-path test asserting no client resolution or logging, and add DI regression tests covering the services.Configure enable path and single configure invocation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 44 +++++++++----- .../BlobPurgeJobOrchestrator.cs | 3 +- ...ientBuilderExtensions.AzureBlobPayloads.cs | 21 +++---- .../AutoPurge/BlobPurgeJobStarterTests.cs | 54 +++++++++++++----- .../UseExternalizedPayloadsTests.cs | 57 ++++++++++++++++--- 5 files changed, 131 insertions(+), 48 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index ed8ac7d7..b6f10d9c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,17 +11,18 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the singleton blob payload auto-purge job exists. It is registered -/// only when auto-purge is enabled at registration time (see the UseExternalizedPayloads configure overload), -/// so it does not re-check the flag here. It never blocks host startup: it runs on a background task and -/// retries until the backend is reachable. The job is a per-task-hub singleton, so racing client processes -/// simply no-op. +/// Client-side hosted service that ensures the per-task-hub singleton blob payload auto-purge job exists. It is +/// registered unconditionally by UseExternalizedPayloads and decides what to do at startup, once options are +/// fully resolved: it no-ops silently when auto-purge is disabled, and no-ops with an error log when the +/// registered store cannot delete. It never blocks host startup - the ensure work runs on a background task +/// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client +/// processes simply no-op. /// sealed class BlobPurgeJobStarter : IHostedService { static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); - readonly DurableTaskClient client; + readonly IDurableTaskClientProvider clientProvider; readonly PayloadStore store; readonly IOptionsMonitor options; readonly string builderName; @@ -32,13 +33,13 @@ sealed class BlobPurgeJobStarter : IHostedService Task? ensureTask; public BlobPurgeJobStarter( - DurableTaskClient client, + IDurableTaskClientProvider clientProvider, PayloadStore store, IOptionsMonitor options, string builderName, ILogger logger) { - this.client = Check.NotNull(client); + this.clientProvider = Check.NotNull(clientProvider); this.store = Check.NotNull(store); this.options = Check.NotNull(options); this.builderName = Check.NotNull(builderName); @@ -48,6 +49,19 @@ public BlobPurgeJobStarter( /// public Task StartAsync(CancellationToken cancellationToken) { + LargePayloadStorageOptions opts = this.options.Get(this.builderName); + + // Not opted in. The starter is registered unconditionally by UseExternalizedPayloads because whether + // auto-purge is enabled can only be known once options are fully resolved - the flag can be set by the + // inline configure delegate, services.Configure, configuration binding or PostConfigure. Deciding at + // registration time (by running the delegate against a probe instance) both invoked user code twice and + // missed every enable path except the inline delegate. This is the normal path for apps that externalize + // payloads without auto-purge, so it returns silently without logging. + if (!opts.AutoPurge) + { + return Task.CompletedTask; + } + // Auto-purge deletes blobs through the store, but PayloadStore.DeleteAsync is virtual and its base // implementation throws NotSupportedException. A store that cannot delete would fail every single // payload, so refuse to start the job rather than spin against the backend - and rather than ack rows @@ -59,13 +73,17 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - LargePayloadStorageOptions opts = this.options.Get(this.builderName); + // Resolve the client by builder name rather than by type: a named client builder must get its own + // client, and resolving lazily here - after the AutoPurge gate - avoids constructing a DurableTaskClient + // at host start for apps that externalize payloads without auto-purge. + DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. this.cts = new CancellationTokenSource(); - this.ensureTask = Task.Run(() => this.EnsureJobAsync(batchSize, this.cts.Token), CancellationToken.None); + this.ensureTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -82,7 +100,7 @@ public async Task StopAsync(CancellationToken cancellationToken) } } - async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) + async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) { @@ -96,7 +114,7 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) // id and no dedupe policy the backend would purge and replace the terminal instance on every // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated // state that may never have applied Create - which lets a failed setup self-heal. - OrchestrationMetadata? existing = await this.client.GetInstanceAsync( + OrchestrationMetadata? existing = await client.GetInstanceAsync( BlobPurgeConstants.StarterInstanceId, cancellationToken); bool needsSchedule = existing is null @@ -110,7 +128,7 @@ async Task EnsureJobAsync(int batchSize, CancellationToken cancellationToken) BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); - await this.client.ScheduleNewOrchestrationInstanceAsync( + await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 8ba4f707..6523001b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -145,7 +145,8 @@ async Task DeleteOneAsync(TaskOrchestrationContext context, Tombs { BlobDeleteResult result = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), - tombstone.Token); + tombstone.Token, + new TaskOptions(PurgeActivityRetryPolicy)); return new DeleteOutcome( result != BlobDeleteResult.Retry, diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index e924cb3e..f54d499b 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -35,18 +35,7 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( builder.Services.Configure(builder.Name, configure); - UseExternalizedPayloadsCore(builder); - - // Conditional DI: register the auto-purge starter only when the caller opted into auto-purge. Peek the - // flag now by running the configure delegate against a probe (options configurators are pure setters). - LargePayloadStorageOptions probe = new(); - configure(probe); - if (probe.AutoPurge) - { - RegisterBlobPurgeJobStarter(builder); - } - - return builder; + return UseExternalizedPayloadsCore(builder); } /// @@ -98,6 +87,12 @@ static IDurableTaskClientBuilder UseExternalizedPayloadsCore(IDurableTaskClientB } }); + // Always register the auto-purge starter. Whether auto-purge is actually enabled can only be known once + // options are fully resolved - the flag can be set by the inline configure delegate, services.Configure, + // configuration binding or PostConfigure, none of which are visible here at registration time - so the + // starter is registered unconditionally and no-ops in StartAsync when AutoPurge is disabled. + RegisterBlobPurgeJobStarter(builder); + return builder; } @@ -105,7 +100,7 @@ static void RegisterBlobPurgeJobStarter(IDurableTaskClientBuilder builder) { string builderName = builder.Name; builder.Services.AddSingleton(sp => new BlobPurgeJobStarter( - sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetRequiredService(), sp.GetRequiredService>(), builderName, diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index e8617b2b..f79c79e6 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -15,10 +15,11 @@ public class BlobPurgeJobStarterTests public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient() { // Arrange - the registered store is not the blob store, so its DeleteAsync is unsupported. Every payload - // would fail, so the starter must refuse to run rather than spin against the backend. - Mock client = new("test"); + // would fail, so the starter must refuse to run rather than spin against the backend. AutoPurge is on so + // the run gets past the opt-in gate and reaches the store-capability gate under test. + Mock provider = new(); BlobPurgeJobStarter starter = new( - client.Object, + provider.Object, new NonDeletingPayloadStore(), OptionsFor(new LargePayloadStorageOptions { AutoPurge = true }), "test", @@ -27,37 +28,64 @@ public async Task StartAsync_WhenStoreCannotDelete_DoesNotStartJobOrTouchClient( // Act await starter.StartAsync(CancellationToken.None); - // Assert - the startup gate short-circuits before scheduling anything, so the client is never queried. - client.Verify( - c => c.GetInstanceAsync(It.IsAny(), It.IsAny()), Times.Never); + // Assert - the store gate short-circuits before the client is resolved, so the provider is never asked + // for a client. + provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); } [Fact] public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() { - // Arrange - the real blob store. UseDevelopmentStorage=true is a valid connection string that constructs - // the store offline (no network I/O), so resolving it at host start is safe. + // Arrange - the real blob store with auto-purge enabled. UseDevelopmentStorage=true is a valid connection + // string that constructs the store offline (no network I/O), so resolving it at host start is safe. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); - Mock client = new("test"); + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(new Mock("test").Object); TestLogger logger = new(); BlobPurgeJobStarter starter = new( - client.Object, + provider.Object, store, - OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), "test", logger); // Act await starter.StartAsync(CancellationToken.None); - // Assert - the store-cannot-delete gate did NOT fire; the starter proceeded to its background ensure - // path (which runs on a background task, so timing is not asserted). + // Assert - neither gate fired: no store-cannot-delete log, and the starter proceeded to resolve the + // client and start its background ensure path (which runs on a background task, so timing is not + // asserted). logger.Logs.Should().NotContain(entry => entry.Message.Contains("is not an Azure Blob payload store")); + provider.Verify(p => p.GetClient(It.IsAny()), Times.Once); // Cleanup - cancel the background ensure task so it does not outlive the test. await starter.StopAsync(CancellationToken.None); } + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() + { + // Arrange - auto-purge is off. Even with a delete-capable blob store, the starter must no-op silently: + // it is registered unconditionally, so the not-opted-in path is the common case and must not log or + // resolve a client. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock provider = new(); + TestLogger logger = new(); + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), + "test", + logger); + + // Act + await starter.StartAsync(CancellationToken.None); + + // Assert - returned before resolving a client and without logging anything at all. + provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); + logger.Logs.Should().BeEmpty(); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 2ee439ef..05013de4 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -2,9 +2,13 @@ // Licensed under the MIT License. using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using Moq; using Xunit; @@ -29,7 +33,7 @@ public void UseExternalizedPayloads_WithAutoPurgeEnabled_RegistersHostedPurgeSta } [Fact] - public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedPurgeStarter() + public void UseExternalizedPayloads_WithAutoPurgeDisabled_StillRegistersHostedPurgeStarter() { // Arrange ServiceCollection services = new(); @@ -40,24 +44,61 @@ public void UseExternalizedPayloads_WithAutoPurgeDisabled_DoesNotRegisterHostedP // Act - auto-purge left at its default (false). builder.Object.UseExternalizedPayloads(options => { }); - // Assert - services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + // Assert - the starter is registered unconditionally now; whether auto-purge is enabled can only be + // known once options are fully resolved, so the no-op moved into BlobPurgeJobStarter.StartAsync. Do not + // "fix" this back to NotContain - the registration is intentional. + services.Should().ContainSingle(d => d.ServiceType == typeof(IHostedService)); } [Fact] - public void UseExternalizedPayloads_NoArgOverload_DoesNotRegisterHostedPurgeStarter() + public void UseExternalizedPayloads_AutoPurgeViaServicesConfigure_RegistersResolvableStarter() { - // Arrange + // Arrange - the enable path that silently no-op'd before: AutoPurge set through services.Configure (not + // the inline delegate) plus the parameterless overload. The old probe-at-registration only saw the + // inline delegate, so the starter was never registered. It must now be registered and, more importantly, + // resolvable from the built provider. ServiceCollection services = new(); + services.AddSingleton>(NullLogger.Instance); + services.AddSingleton(Mock.Of()); + services.Configure(o => + { + o.AutoPurge = true; + o.ConnectionString = "UseDevelopmentStorage=true"; + }); + Mock builder = new(); builder.Setup(b => b.Services).Returns(services); builder.Setup(b => b.Name).Returns(string.Empty); - // Act - the shared-store overload never enables auto-purge. + // Act - the parameterless overload; AutoPurge comes from services.Configure above. builder.Object.UseExternalizedPayloads(); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert - the starter resolves as a hosted service (construction pulls PayloadStore, the client + // provider, options and logger), proving the whole enable path is wired end to end. + provider.GetServices().OfType().Should().ContainSingle(); + } + + [Fact] + public void UseExternalizedPayloads_ConfigureDelegate_InvokedExactlyOnce() + { + // Arrange - a delegate that counts its invocations. The old probe-at-registration ran configure a second + // time against a throwaway options instance; this locks in that user code runs exactly once, when the + // named options are first materialized. + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + + int invocations = 0; + + // Act + builder.Object.UseExternalizedPayloads(options => invocations++); + using ServiceProvider provider = services.BuildServiceProvider(); + provider.GetRequiredService>().Get(string.Empty); - // Assert - services.Should().NotContain(d => d.ServiceType == typeof(IHostedService)); + // Assert - configure ran once (at options materialization), not a second time at registration. + invocations.Should().Be(1); } [Fact] From 8f436df07a535fdc1c5f0336f5a6fd7926227e5f Mon Sep 17 00:00:00 2001 From: wangbill Date: Fri, 31 Jul 2026 15:02:24 -0700 Subject: [PATCH 16/32] Back off on zero-ack purge cycles and document TokenPrefixV1 Item A: add the SA1600 doc comment to the internal TokenPrefixV1 const in BlobPayloadStore.cs, restoring the multi-TFM warning baseline to 201. Item B: add an else branch to BlobPurgeJobOrchestrator.RunAsync for the case where a batch produced no acks. DeleteExternalBlobActivity reports storage failures as a BlobDeleteResult.Retry return value rather than an exception, so the activity retry policy never engages and there is no backoff on that path. Combined with the backend's uncursored TOP(N) tombstone query, an immediate continue would refetch the identical rows and re-attempt the identical deletes in a tight loop for the duration of a storage outage. Back off on ErrorBackoff (the existing one-minute timer) before the next cycle. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../Orchestrations/BlobPurgeJobOrchestrator.cs | 10 ++++++++++ .../AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 6523001b..50758e60 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -90,6 +90,16 @@ await context.CallActivityAsync( await context.Entities.CallEntityAsync( input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); } + else + { + // Nothing in this batch could be acknowledged: every delete returned Retry (e.g. a storage + // outage or throttling). Deletes report failure as a return value rather than an exception, + // so no retry policy or backoff applies on that path. The backend serves tombstones with an + // uncursored TOP(N) query, so continuing immediately would refetch the identical rows and + // re-attempt the identical deletes in a tight loop for as long as the outage lasts. Back off + // before trying again. + await context.CreateTimer(ErrorBackoff, default); + } } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index ca73812f..d67f967c 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -23,6 +23,10 @@ namespace Microsoft.DurableTask; Justification = "SemaphoreSlim does not allocate a disposable resource unless AvailableWaitHandle is accessed.")] public sealed class BlobPayloadStore : PayloadStore { + /// + /// The prefix of legacy v1 payload tokens, which identify the container by name only and not the storage + /// account. Auto-purge uses this to detect and skip v1 tokens. + /// internal const string TokenPrefixV1 = "blob:v1:"; const string TokenPrefixV2 = "blob:v2:"; const string ContentEncodingGzip = "gzip"; From 81284e025cd9336bc946e0819fc5253bbc33c287 Mon Sep 17 00:00:00 2001 From: wangbill Date: Sun, 2 Aug 2026 12:33:47 -0700 Subject: [PATCH 17/32] docs(AzureBlobPayloads): reframe v1-token handling as a defensive guard The DTS backend now hard-deletes legacy blob:v1: payload rows instead of tombstoning them, so the SDK will normally never be handed a v1 token. Update the prose that described v1-tombstoning as the expected path. - LargePayloadStorageOptions.AutoPurge (public API): v1-backed blobs are not reclaimed and remain in storage exactly as before auto-purge existed (not a new leak); the backend removes their rows normally. - Logs.cs EventId 822: reworded to read as unexpected (older backend, or a row tombstoned before the fix); keeps Error level, EventId 822, and the full token. - DeleteExternalBlobActivity item + inline gate comment: note the v1 branch is now a defensive guard for backend version skew, not the expected path. Prose only; no behavioural change. The only non-comment edit is the 822 message string. Multi-TFM warning baseline unchanged (201); AzureBlobPayloads tests 42/42. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 44c27836-c49c-45fd-ae2b-3309f9c3f0f0 --- .../AutoPurge/Activities/DeleteExternalBlobActivity.cs | 8 ++++++-- src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs | 2 +- .../Options/LargePayloadStorageOptions.cs | 10 ++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 12438d47..e90c5543 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -24,7 +24,9 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// name, not the storage account, so a delete against the currently-configured account cannot be verified - if /// the store has been repointed the delete would report success while the real blob survives elsewhere. The /// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked -/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. +/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. A current backend +/// hard-deletes v1 rows instead of tombstoning them, so this branch is a defensive guard for an older backend +/// build or a row tombstoned before that fix, not the expected path. /// /// /// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: @@ -72,7 +74,9 @@ public override async Task RunAsync(TaskActivityContext contex // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the - // pipeline. The full token is logged at error level so it remains a recoverable pointer. + // pipeline. The full token is logged at error level so it remains a recoverable pointer. A current + // backend hard-deletes v1 rows instead of tombstoning them, so reaching this branch means an older + // backend build or a row tombstoned before that fix - it is a defensive guard, not the expected path. this.logger.BlobPurgeDeleteV1TokenUnsupported(input); return BlobDeleteResult.Discarded; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index a84a2ff8..f2989b9f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -43,7 +43,7 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); - [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Externalized payload token '{token}' uses the legacy v1 format, which auto-purge does not delete because it does not identify the storage account. The backing blob will NOT be deleted and is now orphaned; the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] + [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Received a legacy v1 externalized payload token '{token}' from the backend, which is unexpected: a current backend hard-deletes v1 payload rows instead of tombstoning them, so this indicates either an older backend build or a row that was tombstoned before that fix. Auto-purge does not delete v1 tokens because a v1 token identifies the container by name only and not the storage account, so the delete cannot be verified; the backing blob is NOT deleted and the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 9339c997..2b0b807a 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -125,10 +125,12 @@ public int ThresholdBytes /// Defaults to false (opt-in). /// /// - /// Auto-purge only deletes blobs referenced by self-describing blob:v2: tokens. Payloads that were - /// externalized by SDK versions that wrote legacy blob:v1: tokens are acknowledged and logged at - /// error level, but their blobs are not deleted: a v1 token does not identify the storage account, so the - /// delete cannot be verified. + /// Auto-purge reclaims only blobs referenced by self-describing blob:v2: tokens. Payloads written by + /// SDK versions that emitted legacy blob:v1: tokens are not reclaimed, because a v1 token identifies + /// the container by name only and not the storage account, so the delete cannot be verified. Their backing + /// blobs remain in storage exactly as they did before auto-purge existed - this is not a new leak introduced + /// by auto-purge - and the backend removes their rows normally. Upgrading to an SDK version that writes v2 + /// tokens makes newly-externalized payloads eligible. /// public bool AutoPurge { get; set; } From 1fcdb1061f219ae254b7c47a63f9054f36e96c60 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:15:25 -0700 Subject: [PATCH 18/32] Reshape large-payload auto-purge to the finalized design Replaces the earlier draft shape with the finalized contract and dispositions. Wire contract (orchestrator_service.proto): drops GetTombstonedPayloads / AckPurgedPayloads and their messages, and splices in the canonical block verbatim -- GetLargePayloadTombstones / ReportLargePayloadPurgeResults, the LargePayloadPurgeDisposition and LargePayloadPurgeReason enums, and large_payload_auto_purge_enabled = 12 on GetWorkItemsRequest. Dispositions: removes Discarded entirely. It was success-shaped and destroyed evidence. Every attempt now returns a disposition plus a stable reason code. v1 tokens and malformed v2 bodies are Quarantined; unknown version prefixes, unreachable accounts, and authorization failures are Retry. Splitting the store's three ArgumentException cases is done with a token prefix gate in the activity rather than new exception types, which required widening TokenPrefixV2 to internal. Ownership marker: uploads now stamp managed_by=dts as blob metadata on both the compressed and uncompressed write paths, and the delete path reads the blob's properties and passes that read's ETag as an If-Match on the delete, so check-and-delete is atomic without a lease. A blob without the marker is left untouched and reported DELETED / BLOB_NOT_STORE_OWNED. Metadata rather than index tags: tags are unsupported on ADLS Gen2 and would require Storage Blob Data Owner. Orchestration: the batch is now always reported, since the backend owns retry scheduling and needs to hear about failures to defer a row. Backoff is preserved for the all-retry case so a storage outage cannot become a tight refetch loop. Progress counts only Deleted rows; counting quarantined rows as purged would reintroduce the success-shaped reporting this change removes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/Client/Core/DurableTaskClient.cs | 26 +-- .../Core/LargePayloadPurgeDisposition.cs | 33 +++ src/Client/Core/LargePayloadPurgeReason.cs | 91 ++++++++ src/Client/Core/LargePayloadPurgeResult.cs | 34 +++ src/Client/Core/LargePayloadTombstone.cs | 23 ++ src/Client/Core/PayloadPurgeAck.cs | 13 -- src/Client/Core/TombstonedPayload.cs | 15 -- src/Client/Grpc/GrpcDurableTaskClient.cs | 53 +++-- .../Activities/AckPurgedPayloadsActivity.cs | 36 --- .../Activities/DeleteExternalBlobActivity.cs | 213 +++++++++++------- .../GetLargePayloadTombstonesActivity.cs | 33 +++ .../GetTombstonedPayloadsActivity.cs | 32 --- .../ReportLargePayloadPurgeResultsActivity.cs | 39 ++++ .../AutoPurge/Constants/BlobPurgeConstants.cs | 2 +- .../AzureBlobPayloads/AutoPurge/Logs.cs | 22 +- .../AutoPurge/Models/BlobDeleteResult.cs | 27 --- .../AutoPurge/Models/BlobPurgeOutcome.cs | 22 ++ .../BlobPurgeJobOrchestrator.cs | 122 ++++++---- ...rkerBuilderExtensions.AzureBlobPayloads.cs | 11 +- .../PayloadStore/BlobPayloadStore.cs | 103 ++++++++- .../PayloadStore/PayloadDeleteOutcome.cs | 28 +++ .../PayloadStore/PayloadStore.cs | 10 +- src/Grpc/orchestrator_service.proto | 156 ++++++++++--- .../Grpc/GrpcDurableTaskWorker.Processor.cs | 4 + .../Grpc/GrpcDurableTaskWorkerOptions.cs | 7 + .../LargePayloadPurgeEnumParityTests.cs | 47 ++++ .../DeleteExternalBlobActivityTests.cs | 185 ++++++++++++--- .../BlobPayloadStoreTests.cs | 42 ++++ .../UseExternalizedPayloadsTests.cs | 37 ++- .../BlobPayloadStoreDeleteTests.cs | 83 ++++++- 30 files changed, 1165 insertions(+), 384 deletions(-) create mode 100644 src/Client/Core/LargePayloadPurgeDisposition.cs create mode 100644 src/Client/Core/LargePayloadPurgeReason.cs create mode 100644 src/Client/Core/LargePayloadPurgeResult.cs create mode 100644 src/Client/Core/LargePayloadTombstone.cs delete mode 100644 src/Client/Core/PayloadPurgeAck.cs delete mode 100644 src/Client/Core/TombstonedPayload.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs delete mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs create mode 100644 src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs create mode 100644 src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs create mode 100644 test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs diff --git a/src/Client/Core/DurableTaskClient.cs b/src/Client/Core/DurableTaskClient.cs index bc31915c..c903574a 100644 --- a/src/Client/Core/DurableTaskClient.cs +++ b/src/Client/Core/DurableTaskClient.cs @@ -550,28 +550,28 @@ public virtual Task> ListInstanceIdsAsync( } /// - /// Gets a batch of tombstoned (soft-deleted) externalized payloads whose backing blobs should be deleted - /// by a credentialed caller before the backend hard-deletes the rows. + /// Gets a batch of due large-payload tombstones whose backing blobs a credentialed caller must delete. /// - /// The maximum number of tombstoned payloads to request. + /// The maximum number of tombstones to request. /// The cancellation token. - /// The batch of tombstoned payloads whose blobs should be deleted. + /// The batch of tombstones whose blobs should be deleted. /// Thrown if this implementation does not support the operation. - public virtual Task> GetTombstonedPayloadsAsync( + public virtual Task> GetLargePayloadTombstonesAsync( int limit, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support retrieving tombstoned payloads."); + => throw new NotSupportedException($"{this.GetType()} does not support retrieving large-payload tombstones."); /// - /// Acknowledges tombstoned payloads whose backing blobs have been deleted so the backend can hard-delete - /// the corresponding rows. + /// Reports the outcome of each attempted large-payload blob deletion so the backend can resolve, + /// reschedule, or quarantine the corresponding tombstones. Every attempted row is reported, not just the + /// successful ones; the backend owns retry scheduling. /// - /// The payloads whose blobs have been deleted. + /// The per-row outcomes of the attempted deletions. /// The cancellation token. - /// A task that completes when the acknowledgement has been sent. + /// A task that completes when the outcomes have been recorded. /// Thrown if this implementation does not support the operation. - public virtual Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) - => throw new NotSupportedException($"{this.GetType()} does not support acknowledging purged payloads."); + public virtual Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default) + => throw new NotSupportedException($"{this.GetType()} does not support reporting large-payload purge results."); // TODO: Create task hub diff --git a/src/Client/Core/LargePayloadPurgeDisposition.cs b/src/Client/Core/LargePayloadPurgeDisposition.cs new file mode 100644 index 00000000..bfdedca2 --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeDisposition.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// The outcome of a single large-payload blob deletion attempt. The split is by whether a failure can +/// self-heal. Mirrors the LargePayloadPurgeDisposition protobuf enum. +/// +public enum LargePayloadPurgeDisposition +{ + /// + /// No disposition was specified. + /// + Unspecified = 0, + + /// + /// Terminal success. The blob was deleted, was already absent, or was deliberately left in place because + /// it is not owned by the payload store. The backend deletes the tombstone in all three cases. + /// + Deleted = 1, + + /// + /// The failure may self-heal, so the row stays pending and the backend sets the next attempt. + /// + Retry = 2, + + /// + /// A deterministic failure or protocol violation that retrying can never fix. The backend preserves the + /// evidence, alerts, and stops automatic retries. + /// + Quarantined = 3, +} diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs new file mode 100644 index 00000000..4be5190d --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// A stable, bounded reason for a . Never carries a token or raw +/// exception text, because tokens expose the storage account, container, and blob path. Mirrors the +/// LargePayloadPurgeReason protobuf enum. +/// +public enum LargePayloadPurgeReason +{ + /// + /// No reason was specified. + /// + Unspecified = 0, + + /// + /// The blob was deleted by this attempt. Reported with . + /// + BlobDeleted = 1, + + /// + /// The blob was already absent. Deletion is idempotent, so this is a success. Reported with + /// . + /// + BlobAlreadyAbsent = 2, + + /// + /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an + /// expected outcome, not a defect: the token text merely matched the v2 grammar. The tombstone is still + /// resolved because the blob is not the store's to delete. Reported with + /// . + /// + BlobNotStoreOwned = 3, + + /// + /// Network failure, timeout, storage outage, throttling, or a 5xx response. Reported with + /// . + /// + TransientStorageFailure = 10, + + /// + /// The registered payload store does not implement deletion. Every payload would fail the same way, so the + /// work is kept recoverable until an operator registers a store that can delete. Reported with + /// . + /// + StoreCannotDelete = 11, + + /// + /// The token is well formed but points at a storage account this worker's credential cannot reach. + /// Recoverable after a configuration or credential change. Reported with + /// . + /// + StorageAccountUnreachable = 12, + + /// + /// The token uses a recognized-but-newer version prefix this worker does not understand. Recoverable after + /// an SDK upgrade, so it earns a long defer rather than quarantine. Reported with + /// . + /// + UnsupportedTokenVersion = 13, + + /// + /// Authorization failed in a way that may be transient or reconfigurable (401/403). Reported with + /// . + /// + StorageAuthorizationFailed = 14, + + /// + /// The token carries a known version prefix but its body does not parse. Because the SDK and backend + /// control both sides of the protocol, this indicates a producer, corruption, or compatibility bug. + /// Reported with . + /// + MalformedToken = 20, + + /// + /// Storage rejected a request generated from a well-formed token as permanently invalid (HTTP 400, for + /// example InvalidUri / InvalidResourceName). Retrying can never succeed. Reported with + /// . + /// + InvalidStorageRequest = 21, + + /// + /// A legacy v1 token reached the worker. This is an invariant violation, because the backend excludes v1 + /// at insertion time. It cannot be safely deleted (no storage account in the token) and cannot be fixed by + /// retrying, so the evidence is preserved instead. Reported with + /// . + /// + LegacyV1Token = 22, +} diff --git a/src/Client/Core/LargePayloadPurgeResult.cs b/src/Client/Core/LargePayloadPurgeResult.cs new file mode 100644 index 00000000..f63f45fb --- /dev/null +++ b/src/Client/Core/LargePayloadPurgeResult.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable outcome of exactly one attempted large-payload blob deletion. Mirrors the +/// LargePayloadPurgeResult protobuf message but is safe to pass through the orchestration/activity +/// boundary. The backend owns retry scheduling: it deletes rows reported as +/// , reschedules +/// with a reason-appropriate next attempt, and moves +/// rows out of the active fetch. +/// +/// The backend partition that owns the tombstoned row. +/// The orchestration instance key the payload belonged to. +/// The backend identifier of the tombstoned payload row. +/// +/// The revision echoed unmodified from the fetched ; used by the backend +/// as a compare-and-swap guard. +/// +/// The disposition of the deletion attempt. +/// The stable reason code explaining the disposition. +/// +/// An optional bounded, sanitized storage status or error code for diagnostics (for example +/// BlobNotFound or 409). Never contains a token or raw exception text. +/// +public sealed record LargePayloadPurgeResult( + int PartitionId, + long InstanceKey, + long PayloadId, + long Revision, + LargePayloadPurgeDisposition Disposition, + LargePayloadPurgeReason Reason, + string? StorageErrorCode = null); diff --git a/src/Client/Core/LargePayloadTombstone.cs b/src/Client/Core/LargePayloadTombstone.cs new file mode 100644 index 00000000..c11a40db --- /dev/null +++ b/src/Client/Core/LargePayloadTombstone.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Client; + +/// +/// Serializable representation of a tombstoned large-payload row whose external blob a credentialed caller +/// must delete. Mirrors the LargePayloadTombstone protobuf message but is safe to pass through the +/// orchestration/activity boundary. +/// +/// The backend partition that owns the tombstoned row. +/// The orchestration instance key the payload belonged to. +/// The backend identifier of the tombstoned payload row. +/// +/// The self-describing blob:v2:{fullBlobUrl} payload token whose backing blob should be deleted. +/// +/// +/// An optimistic-concurrency guard echoed back unmodified in the corresponding +/// so the backend can reject duplicate or stale reports without taking +/// a per-row lease. +/// +public sealed record LargePayloadTombstone( + int PartitionId, long InstanceKey, long PayloadId, string Token, long Revision); diff --git a/src/Client/Core/PayloadPurgeAck.cs b/src/Client/Core/PayloadPurgeAck.cs deleted file mode 100644 index 6e7788aa..00000000 --- a/src/Client/Core/PayloadPurgeAck.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Serializable acknowledgement that the worker has deleted the blob for a tombstoned payload, so the -/// backend can hard-delete the soft-deleted row. Mirrors the PayloadPurgeAck protobuf message. -/// -/// The backend partition that owns the payload row. -/// The orchestration instance key the payload belongs to. -/// The backend identifier of the soft-deleted payload row. -public sealed record PayloadPurgeAck(int PartitionId, long InstanceKey, long PayloadId); diff --git a/src/Client/Core/TombstonedPayload.cs b/src/Client/Core/TombstonedPayload.cs deleted file mode 100644 index bb3f6896..00000000 --- a/src/Client/Core/TombstonedPayload.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Serializable representation of a tombstoned payload the backend has soft-deleted and whose blob the -/// worker should delete. Mirrors the TombstonedPayload protobuf message but is safe to pass through -/// the orchestration/activity boundary. -/// -/// The backend partition that owns the payload row. -/// The orchestration instance key the payload belongs to. -/// The backend identifier of the soft-deleted payload row. -/// The externalized payload token whose backing blob should be deleted. -public sealed record TombstonedPayload(int PartitionId, long InstanceKey, long PayloadId, string Token); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 6e79b6b2..adaa2116 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -625,7 +625,7 @@ public override async Task> GetOrchestrationHistoryAsync( } /// - public override async Task> GetTombstonedPayloadsAsync( + public override async Task> GetLargePayloadTombstonesAsync( int limit, CancellationToken cancellation = default) { if (limit <= 0 || limit > 1000) @@ -634,59 +634,70 @@ public override async Task> GetTombstonedPayloadsAsync( nameof(limit), limit, "Limit must be greater than 0 and less than or equal to 1000."); } - P.GetTombstonedPayloadsResponse response; + P.GetLargePayloadTombstonesResponse response; try { - response = await this.sidecarClient.GetTombstonedPayloadsAsync( - new P.GetTombstonedPayloadsRequest { Limit = limit }, + response = await this.sidecarClient.GetLargePayloadTombstonesAsync( + new P.GetLargePayloadTombstonesRequest { Limit = limit }, cancellationToken: cancellation); } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( - $"The {nameof(this.GetTombstonedPayloadsAsync)} operation was canceled.", e, cancellation); + $"The {nameof(this.GetLargePayloadTombstonesAsync)} operation was canceled.", e, cancellation); } - List result = new(response.Payloads.Count); - foreach (P.TombstonedPayload payload in response.Payloads) + List result = new(response.Tombstones.Count); + foreach (P.LargePayloadTombstone tombstone in response.Tombstones) { - result.Add(new TombstonedPayload( - payload.PartitionId, payload.InstanceKey, payload.PayloadId, payload.Token)); + result.Add(new LargePayloadTombstone( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Token, + tombstone.Revision)); } return result; } /// - public override async Task AckPurgedPayloadsAsync( - IEnumerable acks, CancellationToken cancellation = default) + public override async Task ReportLargePayloadPurgeResultsAsync( + IEnumerable results, CancellationToken cancellation = default) { - Check.NotNull(acks); + Check.NotNull(results); - P.AckPurgedPayloadsRequest request = new(); - foreach (PayloadPurgeAck ack in acks) + P.ReportLargePayloadPurgeResultsRequest request = new(); + foreach (LargePayloadPurgeResult result in results) { - request.Acks.Add(new P.PayloadPurgeAck + request.Results.Add(new P.LargePayloadPurgeResult { - PartitionId = ack.PartitionId, - InstanceKey = ack.InstanceKey, - PayloadId = ack.PayloadId, + PartitionId = result.PartitionId, + InstanceKey = result.InstanceKey, + PayloadId = result.PayloadId, + Revision = result.Revision, + + // The managed enums declare the same numeric values as their protobuf counterparts, so the + // disposition and reason map across by value. + Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, + Reason = (P.LargePayloadPurgeReason)result.Reason, + StorageErrorCode = result.StorageErrorCode ?? string.Empty, }); } - if (request.Acks.Count == 0) + if (request.Results.Count == 0) { return; } try { - await this.sidecarClient.AckPurgedPayloadsAsync(request, cancellationToken: cancellation); + await this.sidecarClient.ReportLargePayloadPurgeResultsAsync(request, cancellationToken: cancellation); } catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) { throw new OperationCanceledException( - $"The {nameof(this.AckPurgedPayloadsAsync)} operation was canceled.", e, cancellation); + $"The {nameof(this.ReportLargePayloadPurgeResultsAsync)} operation was canceled.", e, cancellation); } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs deleted file mode 100644 index 7ef6dbc8..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/AckPurgedPayloadsActivity.cs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Activity that acknowledges to the backend the payloads whose blobs the worker has deleted, so the backend -/// can hard-delete the soft-deleted rows. -/// -/// The Durable Task client used to acknowledge purged payloads to the backend. -/// The logger instance. -[DurableTask] -public class AckPurgedPayloadsActivity( - DurableTaskClient client, - ILogger logger) - : TaskActivity, object?> -{ - readonly DurableTaskClient client = Check.NotNull(client); - readonly ILogger logger = Check.NotNull(logger); - - /// - public override async Task RunAsync(TaskActivityContext context, List input) - { - if (input is null || input.Count == 0) - { - return null; - } - - await this.client.AckPurgedPayloadsAsync(input, CancellationToken.None); - this.logger.BlobPurgeAckedPayloads(input.Count); - return null; - } -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index e90c5543..e811f46f 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -1,53 +1,45 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Globalization; +using System.Net; using Azure; +using Microsoft.DurableTask.Client; using Microsoft.Extensions.Logging; namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Activity that deletes a single externalized payload blob given its token. Deletion is idempotent, so -/// re-delivered tokens and concurrent workers are safe. +/// Activity that deletes a single externalized payload blob given its token, and classifies the attempt as +/// , , or +/// with a stable reason code. Deletion is idempotent, +/// so re-delivered tokens and concurrent workers are safe. /// /// -/// Outcome classification, verified against the Azure.Storage.Blobs / Azure.Core exception model (not -/// assumed): +/// The split between retry and quarantine is whether the failure can self-heal, verified against the +/// Azure.Storage.Blobs / Azure.Core exception model (not assumed): /// /// /// The Azure SDK already retries transient failures internally (connection errors plus HTTP /// 408/429/500/502/503/504, with exponential backoff), so any exception that escapes -/// means those built-in retries were already exhausted. +/// means those built-in retries were already exhausted. It is still +/// classified as retryable, because the backend - not this activity - owns retry scheduling and can defer the +/// row past a storage outage. /// /// -/// Legacy blob:v1: tokens are discarded without a delete attempt: a v1 token carries only a container -/// name, not the storage account, so a delete against the currently-configured account cannot be verified - if -/// the store has been repointed the delete would report success while the real blob survives elsewhere. The -/// backend ack protocol has no "skip" status (an un-acked row is re-served every cycle), so the token is acked -/// to keep the pipeline moving and logged at error level as the operator's recovery pointer. A current backend -/// hard-deletes v1 rows instead of tombstoning them, so this branch is a defensive guard for an older backend -/// build or a row tombstoned before that fix, not the expected path. +/// Quarantine is reserved for deterministic failures and protocol violations that retrying can never fix: a +/// known version prefix whose body does not parse, a request storage rejected as permanently invalid, and a +/// legacy v1 token. Quarantine preserves the row and its token as durable evidence, so a permanent failure +/// neither blocks the queue nor destroys the only record of the blob. /// /// -/// Permanent failures are discarded (acked so the backend clears the row) because retrying can never succeed: -/// an from the store's token decode - a genuinely malformed or unrecognized -/// token (v1 tokens are gated out above and never reach the store); a -/// with 400 (for example -/// InvalidUri / InvalidResourceName when the decoded blob name violates Azure naming rules); and a -/// when a v2 token points at a storage account the configured credential -/// cannot reach (connection-string / account-key auth is account-specific). The backend batch is cursor-less, -/// so an undroppable row would otherwise re-stream every cycle and block the pipeline head-of-line; the -/// account-unreachable case is logged at error level so an operator can reconcile it. -/// -/// -/// Everything else is treated as transient and leaves the payload tombstoned to retry on a later cycle: -/// throttling / 5xx that outlived the SDK's retries, 403 authorization failures (which need an operator -/// credential fix rather than dropping data), timeouts / cancellation, and a -/// from a misconfigured store that cannot delete at all (retried, not dropped, so the work is recoverable once -/// a deleting store is registered). A blob is never dropped on an uncertain error, and a single bad token never -/// fails the whole batch. +/// A blob is never deleted on an uncertain error, and a single bad token never fails the whole batch: a +/// failure is returned as a disposition rather than thrown. /// /// +/// Per design §7 no log here carries the token or raw exception text - a token exposes the storage account, +/// container, and blob path. Diagnostics are the stable reason enum plus a bounded, sanitized storage error +/// code; the token itself is preserved on the backend's quarantined row. /// /// The payload store used to delete blobs. /// The logger instance. @@ -55,78 +47,145 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public class DeleteExternalBlobActivity( PayloadStore store, ILogger logger) - : TaskActivity + : TaskActivity { readonly PayloadStore store = Check.NotNull(store); readonly ILogger logger = Check.NotNull(logger); /// - public override async Task RunAsync(TaskActivityContext context, string input) + public override async Task RunAsync(TaskActivityContext context, string input) { Check.NotNullOrEmpty(input, nameof(input)); - if (input.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) + BlobPurgeOutcome outcome = await this.DeleteAsync(input); + + switch (outcome.Disposition) + { + case LargePayloadPurgeDisposition.Quarantined: + this.logger.BlobPurgeDeleteQuarantined(outcome.Reason.ToString(), outcome.StorageErrorCode); + break; + case LargePayloadPurgeDisposition.Retry: + this.logger.BlobPurgeDeleteRetryable(outcome.Reason.ToString(), outcome.StorageErrorCode); + break; + case LargePayloadPurgeDisposition.Deleted + when outcome.Reason == LargePayloadPurgeReason.BlobNotStoreOwned: + this.logger.BlobPurgeBlobNotStoreOwned(); + break; + } + + return outcome; + } + + /// + /// Extracts a bounded, sanitized storage error code for diagnostics. The service's own error code (for + /// example BlobNotFound) is a fixed vocabulary and the numeric status is the fallback, so neither + /// can carry a token or raw exception text. + /// + static string SanitizeErrorCode(RequestFailedException exception) => + string.IsNullOrEmpty(exception.ErrorCode) + ? exception.Status.ToString(CultureInfo.InvariantCulture) + : exception.ErrorCode; + + async Task DeleteAsync(string token) + { + // Classify the token's version prefix before consulting the store. The store reports every token it + // cannot decode as the same ArgumentException, but the three cases have opposite dispositions, so they + // are separated here, where the prefix is still visible. + if (token.StartsWith(BlobPayloadStore.TokenPrefixV1, StringComparison.Ordinal)) { - // Auto-purge deliberately does not act on legacy v1 tokens. A v1 token carries only a container - // *name* - not the storage account - so a delete against the currently-configured account cannot be - // verified: if the store has since been repointed, DeleteIfExistsAsync returns false and the purge - // would silently report success while the real blob survives in the old account. Rather than delete - // on an unverifiable pointer, the token is discarded. The backend ack protocol carries no "skip" - // status (PayloadPurgeAck is just partition/instance/payload id, and an un-acked row is re-served by - // an uncursored TOP(N) query every cycle), so declining without acking would permanently block the - // pipeline. The full token is logged at error level so it remains a recoverable pointer. A current - // backend hard-deletes v1 rows instead of tombstoning them, so reaching this branch means an older - // backend build or a row tombstoned before that fix - it is a defensive guard, not the expected path. - this.logger.BlobPurgeDeleteV1TokenUnsupported(input); - return BlobDeleteResult.Discarded; + // A v1 token carries a container *name* but not the storage account, so a delete against the + // currently-configured account cannot be verified: if the store has since been repointed, + // DeleteIfExists returns false and the purge would falsely report success while the real blob + // survives in the old account. Retrying cannot fix that, and a success-shaped discard would destroy + // the only durable record of the blob, so the row is quarantined instead - the backend preserves + // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so + // reaching this branch is an invariant violation rather than an expected path. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.LegacyV1Token); + } + + if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) + { + // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That + // recovers after an upgrade, so it earns a deferral rather than quarantine. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.UnsupportedTokenVersion); } try { - await this.store.DeleteAsync(input, CancellationToken.None); - return BlobDeleteResult.Deleted; + PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, CancellationToken.None); + return outcome switch + { + PayloadDeleteOutcome.Deleted => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobDeleted), + PayloadDeleteOutcome.AlreadyAbsent => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobAlreadyAbsent), + + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it + // would fill the quarantine set with non-defects. The tombstone is still resolved, because a + // blob the store does not own is not the store's to delete. + _ => new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobNotStoreOwned), + }; + } + catch (ArgumentException) + { + // The prefix gate above proves this is a v2 token, so the only remaining decode failure is a v2 + // body that does not parse. The SDK and backend control both sides of the protocol, so that + // indicates a producer, corruption, or compatibility bug; retrying can never fix it. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.MalformedToken); + } + catch (NotSupportedException) + { + // The registered store does not implement deletion. Every payload would fail the same way, so the + // work is kept recoverable until an operator registers a store that can delete. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StoreCannotDelete); } - catch (ArgumentException ex) + catch (PayloadStorageException) { - // The token is malformed or points at a different container; it can never succeed. Discard it so - // the backend clears the row instead of re-streaming the same poison token every cycle. - this.logger.BlobPurgeDeleteDiscarded(ex, input); - return BlobDeleteResult.Discarded; + // The token is well formed but points at a storage account this worker's credential cannot reach + // (account-key auth is account-specific). Recoverable after a configuration or credential change, + // so it is deferred rather than discarded. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageAccountUnreachable); } - catch (RequestFailedException ex) when (ex.Status == 400) + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { - // Service rejected the request as permanently invalid (e.g. InvalidUri / InvalidResourceName - the - // decoded blob name violates Azure naming rules). Retrying can never succeed, so discard it like a - // poison token: ack so the backend clears the row instead of re-streaming it forever. - this.logger.BlobPurgeDeleteDiscarded(ex, input); - return BlobDeleteResult.Discarded; + // Storage rejected a request generated from a well-formed token as permanently invalid (for + // example InvalidUri / InvalidResourceName). Retrying can never succeed. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Quarantined, + LargePayloadPurgeReason.InvalidStorageRequest, + SanitizeErrorCode(ex)); } - catch (NotSupportedException ex) + catch (RequestFailedException ex) when ( + ex.Status == (int)HttpStatusCode.Unauthorized || ex.Status == (int)HttpStatusCode.Forbidden) { - // The registered store does not implement deletion (PayloadStore.DeleteAsync is virtual and its base - // implementation throws). This is a misconfiguration, not poison data: every payload would fail the - // same way, so acking would hard-delete the backend's entire record of what still needs cleanup while - // every blob survives. Keep the payload tombstoned so the work is recoverable once an operator - // registers a store that can delete. - this.logger.BlobPurgeDeleteNotSupported(ex, input); - return BlobDeleteResult.Retry; + // Authorization can be transient or fixed by reconfiguration, so it stays recoverable rather than + // dropping data an operator can still reclaim. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, + LargePayloadPurgeReason.StorageAuthorizationFailed, + SanitizeErrorCode(ex)); } - catch (PayloadStorageException ex) + catch (RequestFailedException ex) { - // The token is well-formed but points at a storage account this worker's credential cannot reach - // (cross-account without AAD). Retrying can never succeed from this process, and because the backend - // streams tombstones with an uncursored TOP(N) query, leaving it un-acked would re-serve the same - // token every cycle and permanently block the purge pipeline. Discard it so the row is cleared, and - // log at Error so an operator can reclaim the orphaned blob out-of-band. - this.logger.BlobPurgeDeleteUnreachable(ex, input); - return BlobDeleteResult.Discarded; + // Throttling, 5xx, and anything else the service reported, including a failed If-Match on the + // ownership check: transient by default. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, + LargePayloadPurgeReason.TransientStorageFailure, + SanitizeErrorCode(ex)); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { - // Transient failure: leave the payload tombstoned so a later purge cycle can retry it. A single - // bad token must not fail the whole batch. - this.logger.BlobPurgeDeleteFailed(ex, input); - return BlobDeleteResult.Retry; + // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. + return new BlobPurgeOutcome( + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TransientStorageFailure); } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs new file mode 100644 index 00000000..273989ea --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetLargePayloadTombstonesActivity.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that fetches a bounded batch of due large-payload tombstones from the backend for the auto-purge +/// job to delete. +/// +/// The Durable Task client used to query the backend for tombstones. +/// The logger instance. +[DurableTask] +public class GetLargePayloadTombstonesActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task> RunAsync(TaskActivityContext context, int input) + { + int limit = input; + List tombstones = + await this.client.GetLargePayloadTombstonesAsync(limit, CancellationToken.None); + this.logger.BlobPurgeFetchedTombstones(tombstones.Count); + return tombstones; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs deleted file mode 100644 index 838b293a..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/GetTombstonedPayloadsActivity.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using Microsoft.DurableTask.Client; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// Activity that fetches a batch of tombstoned payloads from the backend for the auto-purge job to delete. -/// -/// The Durable Task client used to query the backend for tombstoned payloads. -/// The logger instance. -[DurableTask] -public class GetTombstonedPayloadsActivity( - DurableTaskClient client, - ILogger logger) - : TaskActivity> -{ - readonly DurableTaskClient client = Check.NotNull(client); - readonly ILogger logger = Check.NotNull(logger); - - /// - public override async Task> RunAsync(TaskActivityContext context, int input) - { - int limit = input; - List payloads = - await this.client.GetTombstonedPayloadsAsync(limit, CancellationToken.None); - this.logger.BlobPurgeFetchedTombstones(payloads.Count); - return payloads; - } -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs new file mode 100644 index 00000000..7003b7c9 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/ReportLargePayloadPurgeResultsActivity.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// Activity that reports the outcome of every attempted blob deletion to the backend, so it can delete the +/// resolved tombstones, reschedule the retryable ones, and quarantine the rest. Every attempted row is +/// reported, not only the successful ones: the backend owns retry scheduling, so a row it hears nothing about +/// would simply be re-served unchanged on the next cycle. +/// +/// The Durable Task client used to report purge results to the backend. +/// The logger instance. +[DurableTask] +public class ReportLargePayloadPurgeResultsActivity( + DurableTaskClient client, + ILogger logger) + : TaskActivity, object?> +{ + readonly DurableTaskClient client = Check.NotNull(client); + readonly ILogger logger = Check.NotNull(logger); + + /// + public override async Task RunAsync( + TaskActivityContext context, List input) + { + if (input is null || input.Count == 0) + { + return null; + } + + await this.client.ReportLargePayloadPurgeResultsAsync(input, CancellationToken.None); + this.logger.BlobPurgeReportedResults(input.Count); + return null; + } +} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs index ca4cce32..0138da35 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Constants/BlobPurgeConstants.cs @@ -22,7 +22,7 @@ static class BlobPurgeConstants /// /// The maximum batch size the auto-purge job may request per cycle. Mirrors the gRPC - /// GetTombstonedPayloadsAsync contract, which rejects limits greater than 1000. + /// GetLargePayloadTombstones contract, which rejects limits greater than 1000. /// public const int MaxBatchSize = 1000; diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index f2989b9f..c20cc5d4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -19,14 +19,14 @@ static partial class Logs [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); - [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Failed to delete externalized payload blob for token '{token}'; leaving it tombstoned for a later purge cycle.")] - public static partial void BlobPurgeDeleteFailed(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; reason '{reason}', storage error code '{storageErrorCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it.")] + public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string reason, string? storageErrorCode); [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); - [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge acknowledged {count} purged payload(s) to the backend.")] - public static partial void BlobPurgeAckedPayloads(this ILogger logger, int count); + [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge reported {count} purge result(s) to the backend.")] + public static partial void BlobPurgeReportedResults(this ILogger logger, int count); [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); @@ -34,21 +34,15 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); - [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Discarding poison externalized payload token '{token}'; it can never be deleted, acknowledging it so the backend can clear the row.")] - public static partial void BlobPurgeDeleteDiscarded(this ILogger logger, Exception exception, string token); + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; reason '{reason}', storage error code '{storageErrorCode}'. The backend reschedules the tombstone for a later attempt.")] + public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string reason, string? storageErrorCode); [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); - [LoggerMessage(EventId = 821, Level = LogLevel.Error, Message = "Externalized payload token '{token}' points at a storage account the configured credential cannot reach; the blob cannot be deleted by this worker and will be orphaned. Acknowledging it so the backend can clear the row - reclaim the blob manually or reconfigure the payload store with identity (AAD) authentication that can access both accounts.")] - public static partial void BlobPurgeDeleteUnreachable(this ILogger logger, Exception exception, string token); - - [LoggerMessage(EventId = 822, Level = LogLevel.Error, Message = "Received a legacy v1 externalized payload token '{token}' from the backend, which is unexpected: a current backend hard-deletes v1 payload rows instead of tombstoning them, so this indicates either an older backend build or a row that was tombstoned before that fix. Auto-purge does not delete v1 tokens because a v1 token identifies the container by name only and not the storage account, so the delete cannot be verified; the backing blob is NOT deleted and the row is acknowledged so the purge pipeline is not blocked. Reclaim the blob using the container and blob name in the token above, and upgrade to an SDK version that writes self-describing v2 tokens.")] - public static partial void BlobPurgeDeleteV1TokenUnsupported(this ILogger logger, string token); + [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] + public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); - - [LoggerMessage(EventId = 824, Level = LogLevel.Error, Message = "The registered PayloadStore does not support deleting payloads, so externalized payload token '{token}' cannot be purged. Leaving it tombstoned; register an Azure Blob payload store on the worker or disable AutoPurge.")] - public static partial void BlobPurgeDeleteNotSupported(this ILogger logger, Exception exception, string token); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs deleted file mode 100644 index d3386be5..00000000 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobDeleteResult.cs +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.AzureBlobPayloads; - -/// -/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. -/// -public enum BlobDeleteResult -{ - /// - /// The blob was deleted, or was already gone. The payload should be acknowledged so the backend can - /// hard-delete the row. - /// - Deleted, - - /// - /// The token is permanently invalid (poison) and can never be deleted. The payload should still be - /// acknowledged so the backend clears the stuck row instead of re-streaming the same token forever. - /// - Discarded, - - /// - /// A transient failure occurred. The payload is left tombstoned so a later purge cycle can retry it. - /// - Retry, -} diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs new file mode 100644 index 00000000..e5c563ec --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; + +namespace Microsoft.DurableTask.AzureBlobPayloads; + +/// +/// The outcome of attempting to delete a single externalized payload blob during an auto-purge cycle. The +/// orchestrator combines it with the tombstone's identity and revision to build the reported +/// . +/// +/// Whether the row is resolved, should be retried, or must be quarantined. +/// The stable reason code explaining the disposition. +/// +/// An optional bounded, sanitized storage status or error code for diagnostics. Never a token or raw +/// exception text. +/// +public sealed record BlobPurgeOutcome( + LargePayloadPurgeDisposition Disposition, + LargePayloadPurgeReason Reason, + string? StorageErrorCode = null); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 50758e60..7d1730c6 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -17,9 +17,10 @@ public sealed record BlobPurgeJobRunRequest( EntityInstanceId JobEntityId, int PurgeBatchSize, int ProcessedCycles = 0); /// -/// Perpetual orchestrator that drains tombstoned payloads from the backend, deletes their blobs with capped -/// parallelism, and acknowledges the successful deletions so the backend can hard-delete the rows. It idles -/// on a timer when there is nothing to purge and continues-as-new periodically to keep its history small. +/// Perpetual orchestrator that drains due large-payload tombstones from the backend, deletes their blobs with +/// capped parallelism, and reports every outcome so the backend can resolve, reschedule, or quarantine each +/// row. It idles on a timer when there is nothing to purge and continues-as-new periodically to keep its +/// history small. /// [DurableTask] public class BlobPurgeJobOrchestrator : TaskOrchestrator @@ -66,8 +67,8 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator tombstones = await context.CallActivityAsync>( - nameof(GetTombstonedPayloadsActivity), + List tombstones = await context.CallActivityAsync>( + nameof(GetLargePayloadTombstonesActivity), batchSize, new TaskOptions(PurgeActivityRetryPolicy)); @@ -78,26 +79,34 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator acks = await this.DeleteBatchAsync(context, tombstones); + List results = await this.DeleteBatchAsync(context, tombstones); - if (acks.Count > 0) - { - await context.CallActivityAsync( - nameof(AckPurgedPayloadsActivity), - acks, - new TaskOptions(PurgeActivityRetryPolicy)); + // Every attempted row produces a result, including the retryable ones: the backend owns retry + // scheduling, so it needs to hear about a failure to defer the row. Reporting unconditionally + // is what keeps a failing row from being re-served unchanged on the very next cycle. + await context.CallActivityAsync( + nameof(ReportLargePayloadPurgeResultsActivity), + results, + new TaskOptions(PurgeActivityRetryPolicy)); + + // Two different questions, deliberately not conflated. Progress counts only payloads that were + // actually purged; the backoff decision asks whether ANY row left the retry queue, because a + // quarantined row also stops being re-served even though nothing was reclaimed. + int purged = CountDisposition(results, LargePayloadPurgeDisposition.Deleted); + int resolved = results.Count - CountDisposition(results, LargePayloadPurgeDisposition.Retry); + if (purged > 0) + { await context.Entities.CallEntityAsync( - input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)acks.Count); + input.JobEntityId, nameof(BlobPurgeJob.RecordPurged), (long)purged); } - else + + if (resolved == 0) { - // Nothing in this batch could be acknowledged: every delete returned Retry (e.g. a storage - // outage or throttling). Deletes report failure as a return value rather than an exception, - // so no retry policy or backoff applies on that path. The backend serves tombstones with an - // uncursored TOP(N) query, so continuing immediately would refetch the identical rows and - // re-attempt the identical deletes in a tight loop for as long as the outage lasts. Back off - // before trying again. + // The whole batch came back retryable (e.g. a storage outage or throttling). Deletes report + // failure as a return value rather than an exception, so no activity retry policy applies on + // that path. Continuing immediately would refetch and re-attempt in a tight loop for as long + // as the outage lasts, so back off before the next cycle. await context.CreateTimer(ErrorBackoff, default); } } @@ -112,56 +121,71 @@ await context.Entities.CallEntityAsync( } } - async Task> DeleteBatchAsync( - TaskOrchestrationContext context, List tombstones) + static int CountDisposition( + List results, LargePayloadPurgeDisposition disposition) + { + int count = 0; + foreach (LargePayloadPurgeResult result in results) + { + if (result.Disposition == disposition) + { + count++; + } + } + + return count; + } + + static async Task DrainAsync( + List> tasks, List results) + { + LargePayloadPurgeResult[] completed = await Task.WhenAll(tasks); + results.AddRange(completed); + } + + async Task> DeleteBatchAsync( + TaskOrchestrationContext context, List tombstones) { - List acks = new(tombstones.Count); - List> tasks = new(); + List results = new(tombstones.Count); + List> tasks = new(); - foreach (TombstonedPayload tombstone in tombstones) + foreach (LargePayloadTombstone tombstone in tombstones) { tasks.Add(this.DeleteOneAsync(context, tombstone)); if (tasks.Count >= MaxParallelDeletes) { - await DrainAsync(tasks, acks); + await DrainAsync(tasks, results); tasks.Clear(); } } if (tasks.Count > 0) { - await DrainAsync(tasks, acks); + await DrainAsync(tasks, results); } - return acks; - } - - static async Task DrainAsync(List> tasks, List acks) - { - DeleteOutcome[] outcomes = await Task.WhenAll(tasks); - foreach (DeleteOutcome outcome in outcomes) - { - // Acknowledge blobs that were deleted (or already gone) and poison tokens that can never succeed - // so the backend can hard-delete their rows; transient failures stay tombstoned to retry. - if (outcome.ShouldAck) - { - acks.Add(outcome.Ack); - } - } + return results; } - async Task DeleteOneAsync(TaskOrchestrationContext context, TombstonedPayload tombstone) + async Task DeleteOneAsync( + TaskOrchestrationContext context, LargePayloadTombstone tombstone) { - BlobDeleteResult result = await context.CallActivityAsync( + BlobPurgeOutcome outcome = await context.CallActivityAsync( nameof(DeleteExternalBlobActivity), tombstone.Token, new TaskOptions(PurgeActivityRetryPolicy)); - return new DeleteOutcome( - result != BlobDeleteResult.Retry, - new PayloadPurgeAck(tombstone.PartitionId, tombstone.InstanceKey, tombstone.PayloadId)); + // The revision is echoed back unchanged so the backend can detect a tombstone that was rewritten while + // this attempt was in flight and ignore the stale result. Retry scheduling is the backend's job, so no + // next-attempt time is computed here. + return new LargePayloadPurgeResult( + tombstone.PartitionId, + tombstone.InstanceKey, + tombstone.PayloadId, + tombstone.Revision, + outcome.Disposition, + outcome.Reason, + outcome.StorageErrorCode); } - - readonly record struct DeleteOutcome(bool ShouldAck, PayloadPurgeAck Ack); } diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs index c63db137..142f96eb 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -84,19 +84,24 @@ static IDurableTaskWorkerBuilder UseExternalizedPayloadsCore(IDurableTaskWorkerB } opt.Capabilities.Add(P.WorkerCapability.LargePayloads); + + // Item 2 of the pre-merge checklist: the resolved AutoPurge value rides the work-item + // handshake so the backend only tombstones payloads for a task hub whose workers opted in. + // Absent means "no opinion"; an explicit true/false is the customer's choice. + opt.LargePayloadAutoPurgeEnabled = opts.AutoPurge; }); // Register the entity/orchestrators/activities that run the singleton auto-purge job. These are // ALWAYS registered (not gated on AutoPurge) so that a client-enabled job always has something to - // execute here. The purge activities fetch/ack via the injected DurableTaskClient. + // execute here. The purge activities fetch/report via the injected DurableTaskClient. builder.AddTasks(r => { r.AddEntity(); r.AddOrchestrator(); r.AddOrchestrator(); - r.AddActivity(); + r.AddActivity(); r.AddActivity(); - r.AddActivity(); + r.AddActivity(); }); return builder; diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index d67f967c..9a22cd03 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -28,7 +28,28 @@ public sealed class BlobPayloadStore : PayloadStore /// account. Auto-purge uses this to detect and skip v1 tokens. /// internal const string TokenPrefixV1 = "blob:v1:"; - const string TokenPrefixV2 = "blob:v2:"; + + /// + /// The prefix of self-describing v2 payload tokens, which carry the blob's absolute URI including the + /// storage account. Auto-purge uses this to tell a malformed v2 token (a protocol defect) apart from an + /// unrecognized version prefix (a token written by a newer SDK). + /// + internal const string TokenPrefixV2 = "blob:v2:"; + + /// + /// The metadata name of the ownership marker written on every blob this store creates. Recognizing a + /// token proves only that its text matches the store's grammar; the marker is what proves the store + /// actually wrote the blob, so a customer's own blob is never deleted just because an orchestration + /// referenced it. Azure requires metadata names to follow the naming rules for C# identifiers, so the + /// marker is spelled with an underscore rather than a hyphen. + /// + internal const string OwnershipMarkerName = "managed_by"; + + /// + /// The fixed value of the ownership marker written on every blob this store creates. + /// + internal const string OwnershipMarkerValue = "dts"; + const string ContentEncodingGzip = "gzip"; const int MaxRetryAttempts = 8; const int BaseDelayMs = 250; @@ -124,6 +145,7 @@ public override async Task UploadAsync(string payLoad, CancellationToken BlobOpenWriteOptions writeOptions = new() { HttpHeaders = new BlobHttpHeaders { ContentEncoding = ContentEncodingGzip }, + Metadata = CreateOwnershipMetadata(), }; using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); using GZipStream compressedBlobStream = new(blobStream, System.IO.Compression.CompressionLevel.Optimal, leaveOpen: true); @@ -137,7 +159,15 @@ public override async Task UploadAsync(string payLoad, CancellationToken } else { - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + // The uncompressed path still needs write options purely to carry the ownership marker: + // the marker must be written by every path that creates a blob, or auto-purge would later + // decline to delete the store's own uncompressed payloads. It rides along in the PUT the + // upload already issues, so it costs no extra request. + BlobOpenWriteOptions writeOptions = new() + { + Metadata = CreateOwnershipMetadata(), + }; + using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); // using MemoryStream payloadStream = new(payloadBuffer, writable: false); // await payloadStream.CopyToAsync(blobStream, bufferSize: DefaultCopyBufferSize, cancellationToken); @@ -203,7 +233,7 @@ public override async Task DownloadAsync(string token, CancellationToken } /// - public override async Task DeleteAsync(string token, CancellationToken cancellationToken) + public override async Task DeleteAsync(string token, CancellationToken cancellationToken) { DecodeTokenResult decoded = DecodeToken(token); @@ -238,12 +268,43 @@ public override async Task DeleteAsync(string token, CancellationToken cancellat "account-key credentials are account-specific and cannot delete in another account."); } + // Recognizing the token proves only that its text matches this store's grammar - not that this store + // wrote the blob. A customer may keep an expensive dataset in Blob Storage and have orchestrations + // reference it by URL; deleting that would destroy data the store never created. So ownership is read + // from the object itself before anything is deleted. + BlobProperties properties; + try + { + Response response = await blob.GetPropertiesAsync( + conditions: null, cancellationToken: cancellationToken); + properties = response.Value; + } + catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.NotFound) + { + // Already gone. Deletion is idempotent, so a re-delivered tombstone or a concurrent worker + // replica that won the race is a success, not an error. + return PayloadDeleteOutcome.AlreadyAbsent; + } + + if (!HasOwnershipMarker(properties.Metadata)) + { + // Positive evidence that the blob is customer-owned: leave it untouched. The caller still resolves + // the payload reference, because a blob this store never wrote is not this store's to delete. + return PayloadDeleteOutcome.NotStoreOwned; + } + + // Pair the ownership read with the delete using the ETag from that same read. If anything rewrites the + // blob in between - including a customer overwriting it with content that no longer carries the marker + // - the If-Match condition fails the delete instead of removing the newer content, so the read-then- + // delete behaves as a single check-and-delete without taking a lease. // Idempotent by design: DeleteIfExistsAsync returns false (rather than throwing) when the blob is // already gone, so re-delivered tombstones and concurrent purges from multiple worker replicas are safe. - await blob.DeleteIfExistsAsync( + Response deleted = await blob.DeleteIfExistsAsync( DeleteSnapshotsOption.IncludeSnapshots, - conditions: null, + conditions: new BlobRequestConditions { IfMatch = properties.ETag }, cancellationToken: cancellationToken); + + return deleted.Value ? PayloadDeleteOutcome.Deleted : PayloadDeleteOutcome.AlreadyAbsent; } /// @@ -309,6 +370,38 @@ internal static DecodeTokenResult DecodeToken(string token) throw new ArgumentException("Invalid external payload token.", nameof(token)); } + /// + /// Creates the ownership metadata stamped on every blob this store writes, so a later purge can prove the + /// store created the blob before deleting it. + /// + static IDictionary CreateOwnershipMetadata() => + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [OwnershipMarkerName] = OwnershipMarkerValue, + }; + + /// + /// Returns whether the supplied blob metadata carries this store's ownership marker. Azure treats metadata + /// names as case-insensitive, so the lookup is too. + /// + static bool HasOwnershipMarker(IDictionary? metadata) + { + if (metadata is null) + { + return false; + } + + foreach (KeyValuePair entry in metadata) + { + if (string.Equals(entry.Key, OwnershipMarkerName, StringComparison.OrdinalIgnoreCase)) + { + return string.Equals(entry.Value, OwnershipMarkerValue, StringComparison.Ordinal); + } + } + + return false; + } + static async Task WritePayloadAsync(byte[] payloadBuffer, Stream target, CancellationToken cancellationToken) { #if NETSTANDARD2_0 diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs new file mode 100644 index 00000000..532cbcd9 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadDeleteOutcome.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask; + +/// +/// The outcome of deleting a payload through . All three values are +/// successful terminal outcomes; failures are surfaced as exceptions instead. +/// +public enum PayloadDeleteOutcome +{ + /// + /// The payload's backing object was deleted by this call. + /// + Deleted, + + /// + /// The payload's backing object was already absent. Deletion is idempotent, so this is a success. + /// + AlreadyAbsent, + + /// + /// The backing object exists but does not carry the store's ownership marker, so the store did not + /// create it and left it untouched. The payload reference is still resolved, because an object the + /// store never wrote is not the store's to delete. + /// + NotStoreOwned, +} diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs index 0ae5ccb6..ee6f22b2 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/PayloadStore.cs @@ -32,11 +32,17 @@ public abstract class PayloadStore /// The default implementation throws . Stores that externalize /// payloads to deletable storage (for example Azure Blob Storage) should override it. It is declared /// virtual rather than abstract so that adding it does not break existing external subclasses. + /// Implementations must delete only objects they created; an object that carries no proof of the + /// store's ownership must be left untouched and reported as + /// . /// /// The opaque reference token. /// Cancellation token. - /// A task that completes when the payload has been deleted (or was already absent). - public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => + /// + /// The outcome of the deletion: whether the object was deleted, was already absent, or was left in + /// place because the store does not own it. + /// + public virtual Task DeleteAsync(string token, CancellationToken cancellationToken) => throw new NotSupportedException( $"This {nameof(PayloadStore)} implementation does not support deleting payloads."); diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index af065941..fc007e9a 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -786,6 +786,18 @@ service TaskHubSidecarService { rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); + + // Returns a bounded, deterministically ordered batch of due large-payload tombstones whose + // external blobs the worker must delete. Scoped to the caller's authenticated task hub. + // Only rows that are pending and whose next attempt time has arrived are returned; a row stays + // pending until its outcome is reported, so this is safe under retries and duplicate callers. + rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); + + // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling: it + // deletes rows reported as DELETED, reschedules RETRY with a reason-appropriate next attempt, + // and moves QUARANTINED rows out of the active fetch while preserving their evidence. + rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); + rpc CompleteActivityTask(ActivityResponse) returns (CompleteTaskResponse); rpc CompleteOrchestratorTask(OrchestratorResponse) returns (CompleteTaskResponse); rpc CompleteEntityTask(EntityBatchResult) returns (CompleteTaskResponse); @@ -823,53 +835,133 @@ service TaskHubSidecarService { // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". // Note that a maximum of 500 orchestrations can be terminated at a time using this method. rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); - - // Returns a batch of blob-externalized payload tombstones the backend has soft-deleted so the worker - // can delete the corresponding blobs from customer storage (the backend has no storage credentials of - // its own). The worker deletes each blob and then calls AckPurgedPayloads so the backend can - // hard-delete the soft-deleted rows. This is an opt-in, worker-driven pull model. - rpc GetTombstonedPayloads(GetTombstonedPayloadsRequest) returns (GetTombstonedPayloadsResponse); - - // Acknowledges that the worker has deleted the blobs for the identified payloads so the backend can - // hard-delete the soft-deleted rows. - rpc AckPurgedPayloads(AckPurgedPayloadsRequest) returns (AckPurgedPayloadsResponse); } -// Server -> client. Identifies a blob-externalized payload that the backend has soft-deleted and whose -// blob the worker should delete from customer storage. -message TombstonedPayload { +// server -> client: one tombstoned large-payload row whose external blob the worker must delete. +message LargePayloadTombstone { int32 partitionId = 1; int64 instanceKey = 2; int64 payloadId = 3; - // The externalized payload token (e.g. "blob:v1::") whose backing blob should be deleted. + + // A self-describing SDK v2 token: "blob:v2:{fullBlobUrl}". + // Legacy v1 tokens are never tombstoned: v1 carries a container name but not the storage + // account, so a delete against the configured account cannot be verified. The backend + // hard-deletes v1 payload rows instead (design §8). string token = 4; + + // Optimistic-concurrency guard. The worker echoes this value back unmodified in the result so + // the backend can reject duplicate or stale reports without taking a per-row lease. + int64 revision = 5; +} + +// The outcome of a single blob deletion attempt. The split is by whether a failure can self-heal. +enum LargePayloadPurgeDisposition { + LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; + + // Terminal success. The blob was deleted, was already absent, or was deliberately left in + // place because it is not owned by the payload store (design §5.5). The backend deletes the + // tombstone in all three cases. + LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; + + // The failure may self-heal, so the row stays pending and the backend sets the next attempt. + LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; + + // A deterministic failure or protocol violation that retrying can never fix. The backend + // preserves the evidence, alerts, and stops automatic retries. + LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; +} + +// A stable, bounded reason for a disposition. Never carries a token or raw exception text +// (design §7: tokens expose storage account, container, and blob path). +enum LargePayloadPurgeReason { + LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; + + // --- Reported with DELETED --- + + // The blob was deleted by this attempt. + LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; + + // The blob was already absent. Deletion is idempotent, so this is a success. + LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; + + // The blob did not carry the payload store's ownership marker, so it was left untouched + // (design §5.5). This is an expected outcome, not a defect: the token text merely matched the + // v2 grammar. The tombstone is still resolved because the blob is not the store's to delete. + LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; + + // --- Reported with RETRY --- + + // Network failure, timeout, storage outage, throttling, or a 5xx response. + LARGE_PAYLOAD_PURGE_REASON_TRANSIENT_STORAGE_FAILURE = 10; + + // The registered payload store does not implement deletion. Every payload would fail the same + // way, so the work is kept recoverable until an operator registers a store that can delete. + LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; + + // The token is well formed but points at a storage account this worker's credential cannot + // reach. Recoverable after a configuration or credential change. + LARGE_PAYLOAD_PURGE_REASON_STORAGE_ACCOUNT_UNREACHABLE = 12; + + // The token uses a recognized-but-newer version prefix this worker does not understand. + // Recoverable after an SDK upgrade, so it earns a long defer rather than quarantine. + LARGE_PAYLOAD_PURGE_REASON_UNSUPPORTED_TOKEN_VERSION = 13; + + // Authorization failed in a way that may be transient or reconfigurable (401/403). + LARGE_PAYLOAD_PURGE_REASON_STORAGE_AUTHORIZATION_FAILED = 14; + + // --- Reported with QUARANTINED --- + + // The token carries a known version prefix but its body does not parse. Because the SDK and + // backend control both sides of the protocol, this indicates a producer, corruption, or + // compatibility bug. + LARGE_PAYLOAD_PURGE_REASON_MALFORMED_TOKEN = 20; + + // Storage rejected a request generated from a well-formed token as permanently invalid + // (HTTP 400, e.g. InvalidUri / InvalidResourceName). Retrying can never succeed. + LARGE_PAYLOAD_PURGE_REASON_INVALID_STORAGE_REQUEST = 21; + + // A legacy v1 token reached the worker. This is an invariant violation, because the backend + // excludes v1 at insertion time. It cannot be safely deleted (no storage account in the token) + // and cannot be fixed by retrying, so the evidence is preserved instead (design §6). + LARGE_PAYLOAD_PURGE_REASON_LEGACY_V1_TOKEN = 22; } -// Client -> server. Acknowledges that the worker has deleted the blob for the identified payload so the -// backend can hard-delete the soft-deleted row. -message PayloadPurgeAck { +// client -> server: the outcome of exactly one tombstoned row. +message LargePayloadPurgeResult { + // Row identity, echoed from the corresponding LargePayloadTombstone. int32 partitionId = 1; int64 instanceKey = 2; int64 payloadId = 3; + + // Echoed unmodified from the fetched tombstone; used as a compare-and-swap guard. + int64 revision = 4; + + LargePayloadPurgeDisposition disposition = 5; + LargePayloadPurgeReason reason = 6; + + // Optional bounded, sanitized storage status or error code for diagnostics + // (for example "BlobNotFound" or "409"). Must never contain a token or raw exception text. + string storageErrorCode = 7; } -// Client -> server. Requests up to `limit` tombstoned payloads for the worker to delete. -message GetTombstonedPayloadsRequest { +// client -> server: request up to `limit` due tombstones for the caller's task hub. +message GetLargePayloadTombstonesRequest { + // The maximum number of rows to return. The service clamps this to its own maximum. int32 limit = 1; } -// Server -> client. Carries the batch of tombstoned payloads for the worker to delete. -message GetTombstonedPayloadsResponse { - repeated TombstonedPayload payloads = 1; +// server -> client: the due tombstones whose blobs the worker must delete. +message GetLargePayloadTombstonesResponse { + repeated LargePayloadTombstone tombstones = 1; } -// Client -> server. Acknowledges a batch of payloads whose blobs the worker has deleted. -message AckPurgedPayloadsRequest { - repeated PayloadPurgeAck acks = 1; +// client -> server: a bounded batch of purge outcomes. +message ReportLargePayloadPurgeResultsRequest { + repeated LargePayloadPurgeResult results = 1; } -// Server -> client. Empty response acknowledging the acks were recorded. -message AckPurgedPayloadsResponse { +// server -> client: acknowledgement that the reported outcomes were recorded. +message ReportLargePayloadPurgeResultsResponse { } message GetWorkItemsRequest { @@ -879,6 +971,16 @@ message GetWorkItemsRequest { repeated WorkerCapability capabilities = 10; WorkItemFilters workItemFilters = 11; + + // Task-hub scoped opt-in for large-payload blob auto-purge. + // + // This is a setting, not a WorkerCapability: WORKER_CAPABILITY_LARGE_PAYLOADS means a worker + // *can resolve* externalized payloads, not that the customer opted into *deleting* them. + // A capability list is presence-only and cannot express an explicit false, so a three-state + // wrapper is used instead: + // absent -> the worker has no opinion (older SDK); the persisted setting is unchanged. + // true/false -> an explicit customer choice; persisted only when the value actually differs. + google.protobuf.BoolValue large_payload_auto_purge_enabled = 12; } enum WorkerCapability { diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index 5dd18d52..ea5f7c87 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -349,6 +349,10 @@ async ValueTask BuildRuntimeStateAsync( workerOptions.Concurrency.MaximumConcurrentEntityWorkItems, Capabilities = { this.worker.grpcOptions.Capabilities }, WorkItemFilters = this.worker.workItemFilters?.ToGrpcWorkItemFilters(), + + // Left unset when the worker has no opinion, so the backend can distinguish "not + // configured" from an explicit opt-out. + LargePayloadAutoPurgeEnabled = this.worker.grpcOptions.LargePayloadAutoPurgeEnabled, }, cancellationToken: cancellation); } diff --git a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs index 59c21a00..a7f5fa34 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorkerOptions.cs @@ -44,6 +44,13 @@ public sealed class GrpcDurableTaskWorkerOptions : DurableTaskWorkerOptions /// public HashSet Capabilities { get; } = new() { P.WorkerCapability.HistoryStreaming }; + /// + /// Gets or sets a value indicating whether this worker opts in to large-payload auto-purge, announced to + /// the backend on connection. null means the worker expresses no opinion and the backend keeps its + /// current behavior; an explicit true or false is the customer's choice. + /// + public bool? LargePayloadAutoPurgeEnabled { get; set; } + /// /// Gets or sets the maximum size of all actions in a complete orchestration work item chunk. /// The default value is 3.9MB. We leave some headroom to account for request size overhead. diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs new file mode 100644 index 00000000..4a88205e --- /dev/null +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Client.Grpc.Tests; + +/// +/// maps the managed purge enums onto +/// their protobuf counterparts by numeric value rather than by name, which is only correct while the two sides +/// agree on every value. A silent drift would not fail to compile; it would send the backend a different +/// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. +/// +public class LargePayloadPurgeEnumParityTests +{ + [Fact] + public void Disposition_ManagedAndProtobufValues_AreIdentical() + { + // Arrange & Act + Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeDisposition)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + + // Assert - same numeric values AND the same names at each value, so neither side can gain, lose, or + // renumber a member unnoticed. + managed.Should().Equal(proto); + } + + [Fact] + public void Reason_ManagedAndProtobufValues_AreIdentical() + { + // Arrange & Act + Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeReason)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeReason)) + .Cast() + .ToDictionary(v => (int)v, v => v.ToString()); + + // Assert + managed.Should().Equal(proto); + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 93989325..4002dd36 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -4,88 +4,183 @@ using Azure; using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; using Xunit; namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; public class DeleteExternalBlobActivityTests { + const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; + [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_DiscardsPoisonToken() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInvalidStorageRequest() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. - StubPayloadStore store = new(new RequestFailedException(400, "InvalidResourceName")); + StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/bad name"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - discarded so the backend acks and clears the row instead of re-streaming forever. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - quarantined (evidence preserved), never a success-shaped discard. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.InvalidStorageRequest); + outcome.StorageErrorCode.Should().Be("InvalidResourceName"); } [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_LeavesTombstonedForRetry() + public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransient() { - // Arrange - a Status 503 that escaped the SDK's internal retries is treated as transient. - StubPayloadStore store = new(new RequestFailedException(503, "ServerBusy")); + // Arrange - a Status 503 that escaped the SDK's internal retries is still treated as transient. + StubPayloadStore store = new(new RequestFailedException(503, "busy", "ServerBusy", null)); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - left tombstoned so a later purge cycle can retry; a blob is never dropped on doubt. - result.Should().Be(BlobDeleteResult.Retry); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.StorageErrorCode.Should().Be("ServerBusy"); + } + + [Theory] + [InlineData(401)] + [InlineData(403)] + public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int status) + { + // Arrange - authorization can be fixed by reconfiguration, so it stays recoverable. + StubPayloadStore store = new(new RequestFailedException(status, "denied", "AuthorizationFailure", null)); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAuthorizationFailed); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_DiscardsToUnblockPipeline() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAccountUnreachable() { - // Arrange - the payload lives in a storage account the configured credential cannot reach. Retrying can - // never succeed and the backend batch is cursor-less, so a permanently unreachable row would re-stream - // every cycle and block later rows; it must be discarded (acked), not retried. + // Arrange - the payload lives in a storage account the configured credential cannot reach. That is + // recoverable after a configuration or credential change, so it is deferred rather than discarded. StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync( + null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAccountUnreachable); + } + + [Fact] + public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() + { + // Arrange - a v1 token names a container but not the storage account, so a delete against the + // configured account cannot be verified and would falsely report success if the store was repointed. + Mock store = new(); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); - // Assert - discarded so the pipeline head-of-line is not blocked by an undeletable payload. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.LegacyV1Token); + store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task RunAsync_V1Token_DiscardsWithoutCallingStore() + public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() { - // Arrange - auto-purge policy: a legacy v1 token identifies no storage account, so it is dropped before - // the store is ever consulted. + // Arrange - an unrecognized prefix most likely came from a newer SDK, which recovers after an upgrade. Mock store = new(); DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - discarded by the gate, and the store's DeleteAsync was never invoked. - result.Should().Be(BlobDeleteResult.Discarded); + // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.UnsupportedTokenVersion); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } [Fact] - public async Task RunAsync_V2Token_CallsStore() + public async Task RunAsync_MalformedV2Token_Quarantines() + { + // Arrange - a recognized v2 prefix whose body does not parse; the store signals that with + // ArgumentException. Retrying can never fix a body the SDK itself produced malformed. + StubPayloadStore store = new(new ArgumentException("Invalid token")); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); + + // Assert - contrast with the unknown-prefix case above, which is retried. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); + outcome.Reason.Should().Be(LargePayloadPurgeReason.MalformedToken); + } + + [Fact] + public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() { // Arrange - a self-describing v2 token is not gated and must reach the store. Mock store = new(); - store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())).Returns(Task.CompletedTask); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.Deleted); DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - the store deleted the blob (proves the gate is v1-only and did not break the happy path). - result.Should().Be(BlobDeleteResult.Deleted); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobDeleted); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); } + [Fact] + public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() + { + // Arrange - deletion is idempotent, so a blob a previous attempt already removed is not a failure. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.AlreadyAbsent); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobAlreadyAbsent); + } + + [Fact] + public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeleting() + { + // Arrange - the blob exists but carries no ownership marker, so the store left it untouched. + Mock store = new(); + store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(PayloadDeleteOutcome.NotStoreOwned); + DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, + // and re-serving the row forever would never make it deletable. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); + outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobNotStoreOwned); + } + [Fact] public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombstone() { @@ -94,12 +189,28 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst DeleteExternalBlobActivity activity = new(store, new TestLogger()); // Act - BlobDeleteResult result = await activity.RunAsync(null!, "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"); + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); + + // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while + // the blob survives. + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StoreCannotDelete); + } + + [Fact] + public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() + { + // Arrange - a non-Azure exception (timeout / network failure) must not drop a blob on doubt. + StubPayloadStore store = new(new TimeoutException()); + DeleteExternalBlobActivity activity = new(store, new TestLogger()); + + // Act + BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - retried (tombstone preserved), never discarded: acking would destroy the backend's cleanup - // ledger while the blob survives. - result.Should().Be(BlobDeleteResult.Retry); - result.Should().NotBe(BlobDeleteResult.Discarded); + // Assert + outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.StorageErrorCode.Should().BeNull(); } sealed class StubPayloadStore : PayloadStore @@ -108,8 +219,10 @@ sealed class StubPayloadStore : PayloadStore public StubPayloadStore(Exception? deleteError) => this.deleteError = deleteError; - public override Task DeleteAsync(string token, CancellationToken cancellationToken) => - this.deleteError is null ? Task.CompletedTask : throw this.deleteError; + public override Task DeleteAsync(string token, CancellationToken cancellationToken) => + this.deleteError is null + ? Task.FromResult(PayloadDeleteOutcome.Deleted) + : throw this.deleteError; public override Task UploadAsync(string payLoad, CancellationToken cancellationToken) => throw new NotSupportedException(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs index c03ee76b..4a2111b5 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/BlobPayloadStoreTests.cs @@ -373,6 +373,48 @@ public async Task UploadAsync_StaleContainerNotFoundFailure_DoesNotOverwriteNewe createCalls.Should().Be(2); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task UploadAsync_WritesOwnershipMarkerOnBothWritePaths(bool compressionEnabled) + { + // Arrange - the marker is what lets auto-purge prove the store wrote a blob before deleting it, so it + // must be written on the compressed and the uncompressed path alike. It rides the existing + // BlobOpenWriteOptions, so it costs no extra request. + BlobOpenWriteOptions? capturedOptions = null; + Mock containerClientMock = new(); + containerClientMock.Setup(c => c.Name).Returns("test-container"); + containerClientMock + .Setup(c => c.GetBlobClient(It.IsAny())) + .Returns(() => + { + Mock blobClientMock = new(); + blobClientMock.SetupGet(b => b.Uri).Returns( + new Uri("https://testaccount.blob.core.windows.net/test-container/payload")); + blobClientMock + .Setup(b => b.OpenWriteAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((bool _, BlobOpenWriteOptions options, CancellationToken _) => + { + capturedOptions = options; + return new MemoryStream(); + }); + return blobClientMock.Object; + }); + + LargePayloadStorageOptions options = new() { CompressionEnabled = compressionEnabled }; + BlobPayloadStore store = new(options, containerClientMock.Object); + + // Act + await store.UploadAsync("payload", CancellationToken.None); + + // Assert + capturedOptions.Should().NotBeNull(); + capturedOptions!.Metadata.Should().NotBeNull(); + capturedOptions.Metadata.Should().Contain( + BlobPayloadStore.OwnershipMarkerName, BlobPayloadStore.OwnershipMarkerValue); + } + static Mock CreateContainerClientMock() { Mock containerClientMock = new(); diff --git a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs index 05013de4..7ab88c15 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/DependencyInjection/UseExternalizedPayloadsTests.cs @@ -2,8 +2,11 @@ // Licensed under the MIT License. using FluentAssertions; +using Grpc.Core; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.Grpc; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -115,10 +118,40 @@ public void UseExternalizedPayloads_ClientOnly_RegistersResolvablePayloadStore() // Act - UseDevelopmentStorage=true is a valid connection string that BlobServiceClient accepts with no // network I/O, so the store constructs offline. Build the provider and actually resolve PayloadStore. builder.Object.UseExternalizedPayloads(options => options.ConnectionString = "UseDevelopmentStorage=true"); - using ServiceProvider provider = services.BuildServiceProvider(); + using ServiceProvider clientProvider = services.BuildServiceProvider(); // Assert - the store resolves without throwing and is the blob-backed implementation. - PayloadStore store = provider.GetRequiredService(); + PayloadStore store = clientProvider.GetRequiredService(); store.Should().BeOfType(); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void UseExternalizedPayloads_Worker_SendsResolvedAutoPurgeValueOnTheHandshake(bool autoPurge) + { + // Arrange - the backend only tombstones payloads for a task hub whose workers opted in, so the resolved + // AutoPurge value has to ride the GetWorkItems handshake. A worker that never calls this extension sends + // nothing at all, which the backend reads as "no opinion". + ServiceCollection services = new(); + Mock builder = new(); + builder.Setup(b => b.Services).Returns(services); + builder.Setup(b => b.Name).Returns(string.Empty); + services.AddOptions(string.Empty) + .Configure(o => o.CallInvoker = Mock.Of()); + + // Act + builder.Object.UseExternalizedPayloads(options => + { + options.ConnectionString = "UseDevelopmentStorage=true"; + options.AutoPurge = autoPurge; + }); + + using ServiceProvider provider = services.BuildServiceProvider(); + GrpcDurableTaskWorkerOptions grpcOptions = + provider.GetRequiredService>().Get(string.Empty); + + // Assert - an explicit true/false, never left absent, because calling this extension IS the choice. + grpcOptions.LargePayloadAutoPurgeEnabled.Should().Be(autoPurge); + } } diff --git a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs index 7ccab24a..30c7cee3 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/PayloadStore/BlobPayloadStoreDeleteTests.cs @@ -9,14 +9,17 @@ namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests; /// -/// Unit tests for , covering legacy v1 back-compatibility and the -/// self-describing v2 token resolution (same account, cross-account with identity, cross-account without). +/// Unit tests for , covering legacy v1 back-compatibility, the +/// self-describing v2 token resolution (same account, cross-account with identity, cross-account without), and +/// the ownership marker that gates every delete. /// public class BlobPayloadStoreDeleteTests { const string ContainerName = "payloads"; const string ConfiguredAccountUrl = "https://myaccount.blob.core.windows.net"; + static readonly ETag KnownETag = new("\"0x8DTEST\""); + static Mock CreateContainer(Mock blob, string expectedBlobName) { Mock container = new(); @@ -26,9 +29,34 @@ static Mock CreateContainer(Mock blob, string e return container; } - static Mock CreateBlob(bool existed) + /// + /// Creates a blob that exists and carries this store's ownership marker, which is the ordinary case for a + /// payload the store itself uploaded. + /// + static Mock CreateBlob(bool existed) => CreateBlob(existed, owned: true); + + static Mock CreateBlob(bool existed, bool owned) { Mock blob = new(); + + if (existed) + { + Dictionary metadata = owned + ? new() { [BlobPayloadStore.OwnershipMarkerName] = BlobPayloadStore.OwnershipMarkerValue } + : new() { ["customer-tag"] = "not-ours" }; + + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue( + BlobsModelFactory.BlobProperties(metadata: metadata, eTag: KnownETag), Mock.Of())); + } + else + { + blob + .Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .ThrowsAsync(new RequestFailedException(404, "not found", "BlobNotFound", null)); + } + blob .Setup(b => b.DeleteIfExistsAsync( It.IsAny(), It.IsAny(), It.IsAny())) @@ -45,17 +73,22 @@ public async Task DeleteAsync_V1Token_DeletesBackingBlobIncludingSnapshots() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act - await store.DeleteAsync($"blob:v1:{ContainerName}:abc123", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:abc123", CancellationToken.None); // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); container.Verify(c => c.GetBlobClient("abc123"), Times.Once); blob.Verify( - b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), Times.Once); } [Fact] - public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() + public async Task DeleteAsync_MissingBlob_IsIdempotentAndReportsAlreadyAbsent() { // Arrange Mock blob = CreateBlob(existed: false); @@ -63,13 +96,36 @@ public async Task DeleteAsync_MissingBlob_IsIdempotentAndDoesNotThrow() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act (a missing blob must be a no-op, not an error) - await store.DeleteAsync($"blob:v1:{ContainerName}:missing", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v1:{ContainerName}:missing", CancellationToken.None); - // Assert + // Assert - the ownership probe already proved absence, so no delete request is needed. + outcome.Should().Be(PayloadDeleteOutcome.AlreadyAbsent); blob.Verify( b => b.DeleteIfExistsAsync( It.IsAny(), It.IsAny(), It.IsAny()), - Times.Once); + Times.Never); + } + + [Fact] + public async Task DeleteAsync_BlobWithoutOwnershipMarker_LeavesBlobUntouched() + { + // Arrange - a blob whose token text matches the v2 grammar but which this store never wrote (for + // example a customer dataset referenced by URL, or a payload written before the marker shipped). + Mock blob = CreateBlob(existed: true, owned: false); + Mock container = CreateContainer(blob, "abc123"); + BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); + + // Act + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + + // Assert - reported distinctly and, critically, never deleted. + outcome.Should().Be(PayloadDeleteOutcome.NotStoreOwned); + blob.Verify( + b => b.DeleteIfExistsAsync( + It.IsAny(), It.IsAny(), It.IsAny()), + Times.Never); } [Fact] @@ -115,12 +171,17 @@ public async Task DeleteAsync_V2TokenSameContainer_DeletesViaConfiguredClient() BlobPayloadStore store = new(new LargePayloadStorageOptions(), container.Object); // Act - await store.DeleteAsync($"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); + PayloadDeleteOutcome outcome = await store.DeleteAsync( + $"blob:v2:{ConfiguredAccountUrl}/{ContainerName}/abc123", CancellationToken.None); // Assert + outcome.Should().Be(PayloadDeleteOutcome.Deleted); container.Verify(c => c.GetBlobClient("abc123"), Times.Once); blob.Verify( - b => b.DeleteIfExistsAsync(DeleteSnapshotsOption.IncludeSnapshots, null, It.IsAny()), + b => b.DeleteIfExistsAsync( + DeleteSnapshotsOption.IncludeSnapshots, + It.Is(c => c.IfMatch == KnownETag), + It.IsAny()), Times.Once); } From 2026d71bf533fafbce1a618789eea64ea24c1145 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:33:28 -0700 Subject: [PATCH 19/32] Clear three new-code warnings in the auto-purge files Pre-review polish only; no behaviour change. CS8603 in DeleteExternalBlobActivity.SanitizeErrorCode fired on netstandard2.0 only, where string.IsNullOrEmpty carries no [NotNullWhen(false)] annotation so flow analysis cannot prove the else branch non-null. Replaced with a constant pattern, which the compiler analyses itself and so behaves identically on every target framework. CA1001 in BlobPurgeJobStarter: the type owned a CancellationTokenSource but was not disposable. The source is deliberately disposed in Dispose rather than in StopAsync -- StopAsync stops waiting as soon as the host shutdown token fires, so the ensure task may still hold the token, and disposing there would fault that still-running task with ObjectDisposedException. The container disposes singletons after every StopAsync has returned, which is the safe point. SA1600: documented the BlobPurgeJobStarter constructor. CA1873 on BlobPurgeJobOrchestrator is deliberately left as-is: it occurs 77 times across this repo, so unguarded log calls are the established local convention and changing one new file would make it inconsistent with its neighbours for no benefit. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Activities/DeleteExternalBlobActivity.cs | 12 ++++++--- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 25 ++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index e811f46f..b86a0727 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -81,10 +81,16 @@ public override async Task RunAsync(TaskActivityContext contex /// example BlobNotFound) is a fixed vocabulary and the numeric status is the fallback, so neither /// can carry a token or raw exception text. /// - static string SanitizeErrorCode(RequestFailedException exception) => - string.IsNullOrEmpty(exception.ErrorCode) + static string SanitizeErrorCode(RequestFailedException exception) + { + // Pattern-matched rather than string.IsNullOrEmpty: on netstandard2.0 that method carries no + // [NotNullWhen(false)] annotation, so flow analysis cannot prove the else branch is non-null and warns. + // A constant pattern is analyzed by the compiler itself and so behaves the same on every target. + string? errorCode = exception.ErrorCode; + return errorCode is null or "" ? exception.Status.ToString(CultureInfo.InvariantCulture) - : exception.ErrorCode; + : errorCode; + } async Task DeleteAsync(string token) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index b6f10d9c..3508120d 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -18,7 +18,7 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client /// processes simply no-op. /// -sealed class BlobPurgeJobStarter : IHostedService +sealed class BlobPurgeJobStarter : IHostedService, IDisposable { static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(10); @@ -32,6 +32,14 @@ sealed class BlobPurgeJobStarter : IHostedService CancellationTokenSource? cts; Task? ensureTask; + /// + /// Initializes a new instance of the class. + /// + /// The provider used to resolve the named durable task client. + /// The registered payload store, checked for delete support before starting the job. + /// The monitor used to read the fully-resolved large payload storage options. + /// The name of the client builder this starter belongs to. + /// The logger. public BlobPurgeJobStarter( IDurableTaskClientProvider clientProvider, PayloadStore store, @@ -100,6 +108,21 @@ public async Task StopAsync(CancellationToken cancellationToken) } } + /// + /// Disposes the cancellation source backing the background ensure task. + /// + /// + /// Deliberately not disposed in : that method stops waiting as soon as the host's + /// shutdown token fires, so the ensure task may still hold this source's token. Disposing it there would + /// fault that still-running task with an when it next registers a + /// callback. The container disposes singletons after every has returned, which is + /// the safe point. + /// + public void Dispose() + { + this.cts?.Dispose(); + } + async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationToken cancellationToken) { while (!cancellationToken.IsCancellationRequested) From e25e72437b0c50787b05c51bce56690d8db4c8e2 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:37:32 -0700 Subject: [PATCH 20/32] Pin the no-inbound-enum invariant the numeric purge casts rely on The managed-to-proto enum casts in ReportLargePayloadPurgeResultsAsync are safe only because no enum crosses the wire inbound on this feature: the SDK casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. That invariant held by the shape of the contract rather than by construction, and nothing in the code stated it. Adding an enum to an inbound type would create exactly the hazard the casts avoid today -- a newer backend sending a value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member -- and it would compile silently, mis-dispositioning rows in production rather than failing a build. Asserted by reflection over the inbound types in the existing parity test file, which already exists to protect this mapping. The casts are unchanged: they are correct today, and an explicit switch would add branching for a hazard that does not currently exist on this path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../LargePayloadPurgeEnumParityTests.cs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs index 4a88205e..bd563897 100644 --- a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Reflection; using Microsoft.DurableTask.Client; using P = Microsoft.DurableTask.Protobuf; @@ -44,4 +45,36 @@ public void Reason_ManagedAndProtobufValues_AreIdentical() // Assert managed.Should().Equal(proto); } + + /// + /// The numeric casts are safe only because no enum crosses the wire inbound on this feature: the SDK + /// casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. + /// That invariant holds today by the shape of the contract, not by construction, and nothing in the code + /// states it. Adding an enum to an inbound type would create exactly that path - a newer backend sending a + /// value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member - and it would + /// compile silently. This pins the invariant so it fails here instead. + /// + /// A type carrying server-to-client data for the purge feature. + [Theory] + [InlineData(typeof(LargePayloadTombstone))] + [InlineData(typeof(P.LargePayloadTombstone))] + [InlineData(typeof(P.GetLargePayloadTombstonesResponse))] + [InlineData(typeof(P.ReportLargePayloadPurgeResultsResponse))] + public void InboundTypes_ExposeNoEnumMembers(Type inboundType) + { + // Arrange & Act + List enumMembers = inboundType + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(p => (Nullable.GetUnderlyingType(p.PropertyType) ?? p.PropertyType).IsEnum) + .Select(p => $"{inboundType.Name}.{p.Name}") + .ToList(); + + // Assert + enumMembers.Should().BeEmpty( + "an enum on an inbound type invalidates the numeric enum casts in " + + "GrpcDurableTaskClient.ReportLargePayloadPurgeResultsAsync. The SDK would map a value chosen by the " + + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + + "a switch that handles unknown values"); + } } From 7bf5da8f08e4e6db6d81ebc171ba905664c0581b Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 15:41:25 -0700 Subject: [PATCH 21/32] Clear the two remaining PR-introduced style warnings CA1859: CreateOwnershipMetadata returned IDictionary while always constructing a Dictionary. The method is private static with two call sites, both assigning to BlobOpenWriteOptions.Metadata (IDictionary), so returning the concrete type is implicitly compatible and lets the JIT devirtualize the indexer. SA1117: split the ArgumentOutOfRangeException arguments one per line. CA1873 in BlobPurgeJobOrchestrator is left deliberately: unguarded logging is the established convention here (77 occurrences solution-wide), so guarding one new file would make it inconsistent with its neighbours. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AzureBlobPayloads/Options/LargePayloadStorageOptions.cs | 3 ++- .../AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs index 2b0b807a..49cb020d 100644 --- a/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs +++ b/src/Extensions/AzureBlobPayloads/Options/LargePayloadStorageOptions.cs @@ -147,7 +147,8 @@ public int PayloadPurgeBatchSize if (value < 1 || value > BlobPurgeConstants.MaxBatchSize) { throw new ArgumentOutOfRangeException( - nameof(this.PayloadPurgeBatchSize), value, + nameof(this.PayloadPurgeBatchSize), + value, $"PayloadPurgeBatchSize must be between 1 and {BlobPurgeConstants.MaxBatchSize} (inclusive)."); } diff --git a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs index 9a22cd03..bc597e4c 100644 --- a/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/PayloadStore/BlobPayloadStore.cs @@ -374,7 +374,7 @@ internal static DecodeTokenResult DecodeToken(string token) /// Creates the ownership metadata stamped on every blob this store writes, so a later purge can prove the /// store created the blob before deleting it. /// - static IDictionary CreateOwnershipMetadata() => + static Dictionary CreateOwnershipMetadata() => new Dictionary(StringComparer.OrdinalIgnoreCase) { [OwnershipMarkerName] = OwnershipMarkerValue, From de218c84d76189c4511a852cf3e308303810f902 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:08:49 -0700 Subject: [PATCH 22/32] Narrow the purge reason enum from 11 values to 7 The backend never branches on reason - it acts on disposition alone - so reason is diagnostics only, and its granularity should match the number of distinct operator responses rather than the number of distinct causes. storageErrorCode already carries the specific storage status, so four of the old values encoded the same fact twice. TRANSIENT_STORAGE_FAILURE, STORAGE_ACCOUNT_UNREACHABLE and STORAGE_AUTHORIZATION_FAILED collapse into STORAGE_FAILURE. MALFORMED_TOKEN, INVALID_STORAGE_REQUEST, LEGACY_V1_TOKEN and UNSUPPORTED_TOKEN_VERSION collapse into TOKEN_NOT_PURGEABLE. STORE_CANNOT_DELETE stays separate because storage is never contacted, so its storageErrorCode is empty. Dispositions are unchanged on every branch. TOKEN_NOT_PURGEABLE now spans both Quarantined and Retry - an unsupported version prefix stays Retry because an SDK upgrade resolves it - so each branch continues to state its disposition explicitly rather than deriving it from the reason, which would strand those rows permanently. Proto verified byte-identical to contract-canonical.proto across all three repos by the marker-based harness. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 84 +++++++---------- .../Activities/DeleteExternalBlobActivity.cs | 20 +++-- src/Grpc/orchestrator_service.proto | 89 ++++++++++--------- .../DeleteExternalBlobActivityTests.cs | 29 +++--- 4 files changed, 106 insertions(+), 116 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs index 4be5190d..c957bad0 100644 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -4,9 +4,13 @@ namespace Microsoft.DurableTask.Client; /// -/// A stable, bounded reason for a . Never carries a token or raw -/// exception text, because tokens expose the storage account, container, and blob path. Mirrors the -/// LargePayloadPurgeReason protobuf enum. +/// Why a row received its . Diagnostic only: the backend acts on the +/// disposition alone and never branches on this value, so it exists to make a stuck or non-reclaiming ledger +/// explainable without access to worker logs. Deliberately coarse - granularity matches the number of distinct +/// operator responses, not the number of distinct causes, because +/// already carries the specific storage status. Never +/// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. +/// Mirrors the LargePayloadPurgeReason protobuf enum. /// public enum LargePayloadPurgeReason { @@ -16,76 +20,52 @@ public enum LargePayloadPurgeReason Unspecified = 0, /// - /// The blob was deleted by this attempt. Reported with . + /// The blob was deleted by this attempt, reclaiming its bytes. Reported with + /// . /// BlobDeleted = 1, /// - /// The blob was already absent. Deletion is idempotent, so this is a success. Reported with + /// The blob was already absent, so deletion was a no-op and no bytes were reclaimed. Deletion is idempotent, + /// so this is a success rather than a failure; it is reported separately from + /// because a high rate of it indicates duplicate tombstones. Reported with /// . /// BlobAlreadyAbsent = 2, /// /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an - /// expected outcome, not a defect: the token text merely matched the v2 grammar. The tombstone is still - /// resolved because the blob is not the store's to delete. Reported with - /// . + /// expected outcome, not a defect: the token text merely matched the v2 grammar, and the payload column is + /// customer-writable. The tombstone is still resolved, because a blob the store does not own will never + /// become deletable. Reported separately from so that "resolved without + /// reclaiming bytes" stays countable. Reported with . /// BlobNotStoreOwned = 3, /// - /// Network failure, timeout, storage outage, throttling, or a 5xx response. Reported with - /// . + /// The deletion failed against storage: network failure, timeout, outage, throttling, an unreachable + /// account, or an authorization failure. All of these are reconfigurable or self-healing, and they are not + /// subdivided because already carries the specific + /// status. Reported with . /// - TransientStorageFailure = 10, + StorageFailure = 10, /// - /// The registered payload store does not implement deletion. Every payload would fail the same way, so the - /// work is kept recoverable until an operator registers a store that can delete. Reported with + /// The registered payload store does not implement deletion. Every payload fails the same way, so this is a + /// deployment-wide condition rather than a per-row one, and it stays recoverable until an operator registers + /// a store that can delete. is empty because storage + /// was never contacted, which is why this is not folded into . Reported with /// . /// StoreCannotDelete = 11, /// - /// The token is well formed but points at a storage account this worker's credential cannot reach. - /// Recoverable after a configuration or credential change. Reported with - /// . - /// - StorageAccountUnreachable = 12, - - /// - /// The token uses a recognized-but-newer version prefix this worker does not understand. Recoverable after - /// an SDK upgrade, so it earns a long defer rather than quarantine. Reported with - /// . - /// - UnsupportedTokenVersion = 13, - - /// - /// Authorization failed in a way that may be transient or reconfigurable (401/403). Reported with - /// . - /// - StorageAuthorizationFailed = 14, - - /// - /// The token carries a known version prefix but its body does not parse. Because the SDK and backend - /// control both sides of the protocol, this indicates a producer, corruption, or compatibility bug. - /// Reported with . - /// - MalformedToken = 20, - - /// - /// Storage rejected a request generated from a well-formed token as permanently invalid (HTTP 400, for - /// example InvalidUri / InvalidResourceName). Retrying can never succeed. Reported with - /// . - /// - InvalidStorageRequest = 21, - - /// - /// A legacy v1 token reached the worker. This is an invariant violation, because the backend excludes v1 - /// at insertion time. It cannot be safely deleted (no storage account in the token) and cannot be fixed by - /// retrying, so the evidence is preserved instead. Reported with - /// . + /// The token cannot be acted on as it stands: its body does not parse, it names a version this worker does + /// not support, or storage rejected it as permanently invalid. + /// distinguishes the storage-rejected case, where it + /// is populated, from the parse cases, where it is empty. Reported with + /// , except for an unsupported version prefix, which + /// is reported with because an SDK upgrade resolves it. /// - LegacyV1Token = 22, + TokenNotPurgeable = 20, } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index b86a0727..7304d4db 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -107,15 +107,17 @@ async Task DeleteAsync(string token) // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so // reaching this branch is an invariant violation rather than an expected path. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.LegacyV1Token); + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); } if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) { // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That - // recovers after an upgrade, so it earns a deferral rather than quarantine. + // recovers after an upgrade, so it earns a deferral rather than quarantine. This branch shares + // TokenNotPurgeable with the quarantined token cases, so its disposition is deliberately stated + // here rather than derived from the reason: folding it in would strand rows an upgrade would fix. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.UnsupportedTokenVersion); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TokenNotPurgeable); } try @@ -142,7 +144,7 @@ async Task DeleteAsync(string token) // body that does not parse. The SDK and backend control both sides of the protocol, so that // indicates a producer, corruption, or compatibility bug; retrying can never fix it. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.MalformedToken); + LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); } catch (NotSupportedException) { @@ -157,7 +159,7 @@ async Task DeleteAsync(string token) // (account-key auth is account-specific). Recoverable after a configuration or credential change, // so it is deferred rather than discarded. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageAccountUnreachable); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { @@ -165,7 +167,7 @@ async Task DeleteAsync(string token) // example InvalidUri / InvalidResourceName). Retrying can never succeed. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Quarantined, - LargePayloadPurgeReason.InvalidStorageRequest, + LargePayloadPurgeReason.TokenNotPurgeable, SanitizeErrorCode(ex)); } catch (RequestFailedException ex) when ( @@ -175,7 +177,7 @@ async Task DeleteAsync(string token) // dropping data an operator can still reclaim. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageAuthorizationFailed, + LargePayloadPurgeReason.StorageFailure, SanitizeErrorCode(ex)); } catch (RequestFailedException ex) @@ -184,14 +186,14 @@ async Task DeleteAsync(string token) // ownership check: transient by default. return new BlobPurgeOutcome( LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.TransientStorageFailure, + LargePayloadPurgeReason.StorageFailure, SanitizeErrorCode(ex)); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TransientStorageFailure); + LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); } } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index fc007e9a..02ec0aee 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -793,9 +793,10 @@ service TaskHubSidecarService { // pending until its outcome is reported, so this is safe under retries and duplicate callers. rpc GetLargePayloadTombstones(GetLargePayloadTombstonesRequest) returns (GetLargePayloadTombstonesResponse); - // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling: it - // deletes rows reported as DELETED, reschedules RETRY with a reason-appropriate next attempt, - // and moves QUARANTINED rows out of the active fetch while preserving their evidence. + // Reports the outcome of each attempted blob deletion. The backend owns retry scheduling and + // branches solely on `disposition`: it deletes rows reported as DELETED, reschedules RETRY on + // its own backoff, and moves QUARANTINED rows out of the active fetch while preserving their + // evidence. The worker never computes a retry delay. rpc ReportLargePayloadPurgeResults(ReportLargePayloadPurgeResultsRequest) returns (ReportLargePayloadPurgeResultsResponse); rpc CompleteActivityTask(ActivityResponse) returns (CompleteTaskResponse); @@ -846,7 +847,7 @@ message LargePayloadTombstone { // A self-describing SDK v2 token: "blob:v2:{fullBlobUrl}". // Legacy v1 tokens are never tombstoned: v1 carries a container name but not the storage // account, so a delete against the configured account cannot be verified. The backend - // hard-deletes v1 payload rows instead (design §8). + // hard-deletes v1 payload rows instead. string token = 4; // Optimistic-concurrency guard. The worker echoes this value back unmodified in the result so @@ -856,11 +857,16 @@ message LargePayloadTombstone { // The outcome of a single blob deletion attempt. The split is by whether a failure can self-heal. enum LargePayloadPurgeDisposition { + // Required: proto3 reserves 0 as the first value, and scalars have no field presence, so an + // unset field arrives as 0. Keeping 0 meaningless is load-bearing here: if 0 meant DELETED, a + // client that failed to set this field would make the backend delete tombstones and orphan the + // blobs permanently. The backend must reject a result carrying this value. LARGE_PAYLOAD_PURGE_DISPOSITION_UNSPECIFIED = 0; - // Terminal success. The blob was deleted, was already absent, or was deliberately left in - // place because it is not owned by the payload store (design §5.5). The backend deletes the - // tombstone in all three cases. + // Terminal success: the tombstone is resolved and the backend deletes it. Covers the blob being + // deleted, the blob already being absent, and the blob being deliberately left in place because + // the payload store does not own it. All three are terminal because none of them can be + // improved by trying again. LARGE_PAYLOAD_PURGE_DISPOSITION_DELETED = 1; // The failure may self-heal, so the row stays pending and the backend sets the next attempt. @@ -871,59 +877,58 @@ enum LargePayloadPurgeDisposition { LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; } -// A stable, bounded reason for a disposition. Never carries a token or raw exception text -// (design §7: tokens expose storage account, container, and blob path). +// Why a row received its disposition. Diagnostic only: the backend acts on `disposition` alone and +// never branches on this value, so it exists to make a stuck or non-reclaiming ledger explainable +// without access to worker logs, which run in the customer's process. +// +// Deliberately coarse. Granularity matches the number of distinct operator responses, not the +// number of distinct causes, because `storageErrorCode` already carries the specific storage status. +// Values must never carry a token or raw exception text: a token exposes the storage account, +// container, and blob path. enum LargePayloadPurgeReason { LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; // --- Reported with DELETED --- - // The blob was deleted by this attempt. + // The blob was deleted by this attempt, reclaiming its bytes. LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - // The blob was already absent. Deletion is idempotent, so this is a success. + // The blob was already absent, so deletion was a no-op and no bytes were reclaimed by this + // attempt. Deletion is idempotent, so this is a success rather than a failure. Reported + // separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - // The blob did not carry the payload store's ownership marker, so it was left untouched - // (design §5.5). This is an expected outcome, not a defect: the token text merely matched the - // v2 grammar. The tombstone is still resolved because the blob is not the store's to delete. + // The blob was left in place because it did not carry the payload store's ownership marker, + // meaning the store did not write it. The token text merely matched the v2 grammar; the payload + // column is customer-writable, so matching text is not proof of ownership. This is an expected + // outcome rather than a defect. The tombstone is still resolved, because a blob the store does + // not own will never become deletable and retrying forever would leak the row. Reported + // separately from BLOB_DELETED so that "resolved without reclaiming bytes" stays countable. LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; // --- Reported with RETRY --- - // Network failure, timeout, storage outage, throttling, or a 5xx response. - LARGE_PAYLOAD_PURGE_REASON_TRANSIENT_STORAGE_FAILURE = 10; + // The deletion failed against storage: network failure, timeout, outage, throttling, an + // unreachable account, or an authorization failure. All of these are reconfigurable or + // self-healing, and they are not subdivided here because `storageErrorCode` already carries the + // specific status for diagnostics. Subdividing would encode the same fact twice. + LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - // The registered payload store does not implement deletion. Every payload would fail the same - // way, so the work is kept recoverable until an operator registers a store that can delete. + // The registered payload store does not implement deletion. Every payload fails the same way, + // so this is a deployment-wide condition rather than a per-row one, and it is kept recoverable + // until an operator registers a store that can delete. `storageErrorCode` is empty because + // storage was never contacted, which is why this cannot be folded into STORAGE_FAILURE. LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - // The token is well formed but points at a storage account this worker's credential cannot - // reach. Recoverable after a configuration or credential change. - LARGE_PAYLOAD_PURGE_REASON_STORAGE_ACCOUNT_UNREACHABLE = 12; - - // The token uses a recognized-but-newer version prefix this worker does not understand. - // Recoverable after an SDK upgrade, so it earns a long defer rather than quarantine. - LARGE_PAYLOAD_PURGE_REASON_UNSUPPORTED_TOKEN_VERSION = 13; - - // Authorization failed in a way that may be transient or reconfigurable (401/403). - LARGE_PAYLOAD_PURGE_REASON_STORAGE_AUTHORIZATION_FAILED = 14; - // --- Reported with QUARANTINED --- - // The token carries a known version prefix but its body does not parse. Because the SDK and - // backend control both sides of the protocol, this indicates a producer, corruption, or - // compatibility bug. - LARGE_PAYLOAD_PURGE_REASON_MALFORMED_TOKEN = 20; - - // Storage rejected a request generated from a well-formed token as permanently invalid - // (HTTP 400, e.g. InvalidUri / InvalidResourceName). Retrying can never succeed. - LARGE_PAYLOAD_PURGE_REASON_INVALID_STORAGE_REQUEST = 21; - - // A legacy v1 token reached the worker. This is an invariant violation, because the backend - // excludes v1 at insertion time. It cannot be safely deleted (no storage account in the token) - // and cannot be fixed by retrying, so the evidence is preserved instead (design §6). - LARGE_PAYLOAD_PURGE_REASON_LEGACY_V1_TOKEN = 22; + // The token cannot be acted on and no retry can change that: its body does not parse, it names + // a version this worker does not support, or storage rejected it as permanently invalid. The + // producer and consumer of this token are both controlled by the SDK and backend, so reaching + // this state indicates a producer, corruption, or compatibility bug and the evidence is + // preserved for investigation rather than discarded. `storageErrorCode` distinguishes the + // storage-rejected case, where it is populated, from the parse cases, where it is empty. + LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; } // client -> server: the outcome of exactly one tombstoned row. diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 4002dd36..7949cc57 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -14,7 +14,7 @@ public class DeleteExternalBlobActivityTests const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInvalidStorageRequest() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesAsTokenNotPurgeable() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); @@ -23,9 +23,10 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesWithInval // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - quarantined (evidence preserved), never a success-shaped discard. + // Assert - quarantined (evidence preserved), never a success-shaped discard. The storage error code + // is what distinguishes this from the parse-failure cases, which share the same reason. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.InvalidStorageRequest); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); outcome.StorageErrorCode.Should().Be("InvalidResourceName"); } @@ -41,7 +42,7 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransien // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); outcome.StorageErrorCode.Should().Be("ServerBusy"); } @@ -59,11 +60,11 @@ public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int stat // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAuthorizationFailed); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAccountUnreachable() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsStorageFailure() { // Arrange - the payload lives in a storage account the configured credential cannot reach. That is // recoverable after a configuration or credential change, so it is deferred rather than discarded. @@ -76,7 +77,7 @@ public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsAcco // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageAccountUnreachable); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); } [Fact] @@ -92,7 +93,7 @@ public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.LegacyV1Token); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -106,9 +107,11 @@ public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. + // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. This is the one + // row where TokenNotPurgeable does not mean Quarantined, so the disposition is asserted deliberately: + // deriving it from the reason would strand rows that an SDK upgrade would have resolved. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.UnsupportedTokenVersion); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -123,9 +126,9 @@ public async Task RunAsync_MalformedV2Token_Quarantines() // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); - // Assert - contrast with the unknown-prefix case above, which is retried. + // Assert - contrast with the unknown-prefix case above, which shares this reason but is retried. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.MalformedToken); + outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); } [Fact] @@ -209,7 +212,7 @@ public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TransientStorageFailure); + outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); outcome.StorageErrorCode.Should().BeNull(); } From 02b957ce6f198392827d10f37bbce08ae5d88afd Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:18:07 -0700 Subject: [PATCH 23/32] Re-sync purge reason comments from canonical contract Comment-only. No enum value, field number, disposition, or code path changes; a non-comment-line filter over the full diff of both files returns empty. Block 3 of orchestrator_service.proto is re-spliced verbatim from the canonical contract. The "--- Reported with DELETED / RETRY / QUARANTINED ---" banners are removed: they asserted a 1:1 reason-to-disposition mapping that does not hold, because TOKEN_NOT_PURGEABLE is deliberately reported with two dispositions (QUARANTINED for a v1/malformed/invalid-request token, RETRY for an unrecognized version prefix that a newer worker can read). The disposition now lives in each value's own comment, and the enum header states outright that reason and disposition are orthogonal and that no mapping between them may be asserted. LargePayloadPurgeReason XML docs are rewritten to mirror the new canonical prose, so the managed surface no longer describes the superseded grouping. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 69 ++++++++++++++-------- src/Grpc/orchestrator_service.proto | 69 ++++++++++++---------- 2 files changed, 83 insertions(+), 55 deletions(-) diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs index c957bad0..4984bf35 100644 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ b/src/Client/Core/LargePayloadPurgeReason.cs @@ -10,6 +10,12 @@ namespace Microsoft.DurableTask.Client; /// operator responses, not the number of distinct causes, because /// already carries the specific storage status. Never /// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. +/// +/// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says what the +/// worker found; a disposition says whether trying again can change it. Most reasons occur with exactly one +/// disposition, but deliberately occurs with two. Do not assert a +/// reason-to-disposition mapping anywhere. +/// /// Mirrors the LargePayloadPurgeReason protobuf enum. /// public enum LargePayloadPurgeReason @@ -20,52 +26,65 @@ public enum LargePayloadPurgeReason Unspecified = 0, /// - /// The blob was deleted by this attempt, reclaiming its bytes. Reported with - /// . + /// Reported with . The blob was deleted by this attempt, + /// reclaiming its bytes. /// BlobDeleted = 1, /// - /// The blob was already absent, so deletion was a no-op and no bytes were reclaimed. Deletion is idempotent, - /// so this is a success rather than a failure; it is reported separately from - /// because a high rate of it indicates duplicate tombstones. Reported with - /// . + /// Reported with . The blob was already absent, so deletion + /// was a no-op and no bytes were reclaimed by this attempt. Deletion is idempotent, so this is a success + /// rather than a failure. Reported separately from because a high rate of it + /// indicates duplicate tombstones. /// BlobAlreadyAbsent = 2, /// - /// The blob did not carry the payload store's ownership marker, so it was left untouched. This is an - /// expected outcome, not a defect: the token text merely matched the v2 grammar, and the payload column is - /// customer-writable. The tombstone is still resolved, because a blob the store does not own will never - /// become deletable. Reported separately from so that "resolved without - /// reclaiming bytes" stays countable. Reported with . + /// Reported with . The blob was left in place because it + /// did not carry the payload store's ownership marker, meaning the store did not write it. The token text + /// merely matched the v2 grammar; the payload column is customer-writable, so matching text is not proof of + /// ownership. This is an expected outcome rather than a defect. The tombstone is still resolved, because a + /// blob the store does not own will never become deletable and retrying forever would leak the row. Reported + /// separately so that "resolved without reclaiming bytes" stays countable. /// BlobNotStoreOwned = 3, /// - /// The deletion failed against storage: network failure, timeout, outage, throttling, an unreachable - /// account, or an authorization failure. All of these are reconfigurable or self-healing, and they are not - /// subdivided because already carries the specific - /// status. Reported with . + /// Reported with . The deletion failed against storage: + /// network failure, timeout, outage, throttling, an unreachable account, or an authorization failure. All of + /// these are reconfigurable or self-healing, and they are not subdivided here because + /// already carries the specific status. Subdividing + /// would encode the same fact twice. /// StorageFailure = 10, /// - /// The registered payload store does not implement deletion. Every payload fails the same way, so this is a - /// deployment-wide condition rather than a per-row one, and it stays recoverable until an operator registers - /// a store that can delete. is empty because storage - /// was never contacted, which is why this is not folded into . Reported with - /// . + /// Reported with . The registered payload store does not + /// implement deletion. Every payload fails the same way, so this is a deployment-wide condition rather than a + /// per-row one, and it is kept recoverable until an operator registers a store that can delete. + /// is empty because storage was never contacted, which + /// is why this cannot be folded into . /// StoreCannotDelete = 11, /// - /// The token cannot be acted on as it stands: its body does not parse, it names a version this worker does - /// not support, or storage rejected it as permanently invalid. + /// The worker could not act on the token. This reason is reported with TWO dispositions, and a consumer must + /// not assume either one. + /// + /// Reported with when the token can never become + /// usable: its body does not parse, it is a legacy v1 token, or storage rejected a well-formed token as + /// permanently invalid. The SDK and backend control both sides of this protocol, so reaching that state + /// indicates a producer, corruption, or compatibility bug, and the evidence is preserved rather than + /// discarded. + /// + /// + /// Reported with in exactly one case: the token names a + /// version this worker does not understand. Nothing is wrong with that token, since a newer worker can read + /// it, so an SDK upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and + /// unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. + /// /// distinguishes the storage-rejected case, where it - /// is populated, from the parse cases, where it is empty. Reported with - /// , except for an unsupported version prefix, which - /// is reported with because an SDK upgrade resolves it. + /// is populated, from the parse and unknown-version cases, where it is empty. /// TokenNotPurgeable = 20, } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 02ec0aee..863f41b0 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -885,49 +885,58 @@ enum LargePayloadPurgeDisposition { // number of distinct causes, because `storageErrorCode` already carries the specific storage status. // Values must never carry a token or raw exception text: a token exposes the storage account, // container, and blob path. +// +// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says +// what the worker found; a disposition says whether trying again can change it. Most reasons occur +// with exactly one disposition, but TOKEN_NOT_PURGEABLE deliberately occurs with two. Do not assert +// a reason-to-disposition mapping anywhere. enum LargePayloadPurgeReason { LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; - // --- Reported with DELETED --- - - // The blob was deleted by this attempt, reclaiming its bytes. + // Reported with DELETED. The blob was deleted by this attempt, reclaiming its bytes. LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - // The blob was already absent, so deletion was a no-op and no bytes were reclaimed by this - // attempt. Deletion is idempotent, so this is a success rather than a failure. Reported - // separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. + // Reported with DELETED. The blob was already absent, so deletion was a no-op and no bytes were + // reclaimed by this attempt. Deletion is idempotent, so this is a success rather than a failure. + // Reported separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - // The blob was left in place because it did not carry the payload store's ownership marker, - // meaning the store did not write it. The token text merely matched the v2 grammar; the payload - // column is customer-writable, so matching text is not proof of ownership. This is an expected - // outcome rather than a defect. The tombstone is still resolved, because a blob the store does - // not own will never become deletable and retrying forever would leak the row. Reported - // separately from BLOB_DELETED so that "resolved without reclaiming bytes" stays countable. + // Reported with DELETED. The blob was left in place because it did not carry the payload store's + // ownership marker, meaning the store did not write it. The token text merely matched the v2 + // grammar; the payload column is customer-writable, so matching text is not proof of ownership. + // This is an expected outcome rather than a defect. The tombstone is still resolved, because a + // blob the store does not own will never become deletable and retrying forever would leak the + // row. Reported separately so that "resolved without reclaiming bytes" stays countable. LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; - // --- Reported with RETRY --- - - // The deletion failed against storage: network failure, timeout, outage, throttling, an - // unreachable account, or an authorization failure. All of these are reconfigurable or - // self-healing, and they are not subdivided here because `storageErrorCode` already carries the - // specific status for diagnostics. Subdividing would encode the same fact twice. + // Reported with RETRY. The deletion failed against storage: network failure, timeout, outage, + // throttling, an unreachable account, or an authorization failure. All of these are + // reconfigurable or self-healing, and they are not subdivided here because `storageErrorCode` + // already carries the specific status. Subdividing would encode the same fact twice. LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - // The registered payload store does not implement deletion. Every payload fails the same way, - // so this is a deployment-wide condition rather than a per-row one, and it is kept recoverable - // until an operator registers a store that can delete. `storageErrorCode` is empty because - // storage was never contacted, which is why this cannot be folded into STORAGE_FAILURE. + // Reported with RETRY. The registered payload store does not implement deletion. Every payload + // fails the same way, so this is a deployment-wide condition rather than a per-row one, and it + // is kept recoverable until an operator registers a store that can delete. `storageErrorCode` is + // empty because storage was never contacted, which is why this cannot be folded into + // STORAGE_FAILURE. LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - // --- Reported with QUARANTINED --- - - // The token cannot be acted on and no retry can change that: its body does not parse, it names - // a version this worker does not support, or storage rejected it as permanently invalid. The - // producer and consumer of this token are both controlled by the SDK and backend, so reaching - // this state indicates a producer, corruption, or compatibility bug and the evidence is - // preserved for investigation rather than discarded. `storageErrorCode` distinguishes the - // storage-rejected case, where it is populated, from the parse cases, where it is empty. + // The worker could not act on the token. This reason is reported with TWO dispositions, and a + // consumer must not assume either one. + // + // Reported with QUARANTINED when the token can never become usable: its body does not parse, it + // is a legacy v1 token, or storage rejected a well-formed token as permanently invalid. The SDK + // and backend control both sides of this protocol, so reaching that state indicates a producer, + // corruption, or compatibility bug, and the evidence is preserved rather than discarded. + // + // Reported with RETRY in exactly one case: the token names a version this worker does not + // understand. Nothing is wrong with that token, since a newer worker can read it, so an SDK + // upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and + // unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. + // + // `storageErrorCode` distinguishes the storage-rejected case, where it is populated, from the + // parse and unknown-version cases, where it is empty. LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; } From e432d2543c09c425121ed43c6ceb6fcddacf9b49 Mon Sep 17 00:00:00 2001 From: wangbill Date: Tue, 11 Aug 2026 16:46:42 -0700 Subject: [PATCH 24/32] Remove purge reason and storage error code from the contract Both fields were verified write-only across the whole system: the backend persists them and nothing reads either one - no SELECT, no WHERE, no API, no alert. Disposition alone drives backend behavior, so the enum was a lossy copy of information that already lives in worker telemetry. Deleting reason also deletes the hazard it created. TOKEN_NOT_PURGEABLE spanned both Retry and Quarantined, so any consumer deriving disposition from reason would have silently stranded rows an SDK upgrade would fix. All 12 activity branches keep their disposition unchanged. Verified branch by branch against the pre-change file rather than by inspection: the unknown-prefix branch stays Retry, and the v1-prefix, malformed-token, and HTTP 400 branches stay Quarantined. Failure detail is now logged rather than sent over the wire. Design section 7 forbids logging raw exceptions, so each classification site logs a bounded cause literal instead. Logging at the classification site rather than at the call site restores the full 11-way granularity the 7-value enum had lost: 401/403, 5xx, and an unreachable account are now distinct in telemetry where the enum had collapsed all three into STORAGE_FAILURE. Two deliberate deviations from the request, both reported: 1. LargePayloadPurgeEnumParityTests.cs is kept, not deleted. It always tested two enums, and the disposition cast in GrpcDurableTaskClient survives - it is the cast that decides whether a row is deleted or quarantined. Only the reason parity fact is removed. 2. This is not the last outbound enum cast. LargePayloadPurgeResult still carries Disposition, so the no-inbound-enum invariant test still guards a live cast and is not vacuous. CA1873 at BlobPurgeJobOrchestrator.cs:66 remains deliberately unfixed: unguarded logging is the established convention here, 77 times solution-wide. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Client/Core/LargePayloadPurgeReason.cs | 90 --------------- src/Client/Core/LargePayloadPurgeResult.cs | 23 ++-- src/Client/Grpc/GrpcDurableTaskClient.cs | 7 +- .../Activities/DeleteExternalBlobActivity.cs | 105 ++++++++---------- .../AzureBlobPayloads/AutoPurge/Logs.cs | 8 +- .../AutoPurge/Models/BlobPurgeOutcome.cs | 15 +-- .../BlobPurgeJobOrchestrator.cs | 4 +- src/Grpc/orchestrator_service.proto | 79 ++----------- .../LargePayloadPurgeEnumParityTests.cs | 27 +---- .../DeleteExternalBlobActivityTests.cs | 98 +++++++++------- 10 files changed, 144 insertions(+), 312 deletions(-) delete mode 100644 src/Client/Core/LargePayloadPurgeReason.cs diff --git a/src/Client/Core/LargePayloadPurgeReason.cs b/src/Client/Core/LargePayloadPurgeReason.cs deleted file mode 100644 index 4984bf35..00000000 --- a/src/Client/Core/LargePayloadPurgeReason.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -namespace Microsoft.DurableTask.Client; - -/// -/// Why a row received its . Diagnostic only: the backend acts on the -/// disposition alone and never branches on this value, so it exists to make a stuck or non-reclaiming ledger -/// explainable without access to worker logs. Deliberately coarse - granularity matches the number of distinct -/// operator responses, not the number of distinct causes, because -/// already carries the specific storage status. Never -/// carries a token or raw exception text, because a token exposes the storage account, container, and blob path. -/// -/// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says what the -/// worker found; a disposition says whether trying again can change it. Most reasons occur with exactly one -/// disposition, but deliberately occurs with two. Do not assert a -/// reason-to-disposition mapping anywhere. -/// -/// Mirrors the LargePayloadPurgeReason protobuf enum. -/// -public enum LargePayloadPurgeReason -{ - /// - /// No reason was specified. - /// - Unspecified = 0, - - /// - /// Reported with . The blob was deleted by this attempt, - /// reclaiming its bytes. - /// - BlobDeleted = 1, - - /// - /// Reported with . The blob was already absent, so deletion - /// was a no-op and no bytes were reclaimed by this attempt. Deletion is idempotent, so this is a success - /// rather than a failure. Reported separately from because a high rate of it - /// indicates duplicate tombstones. - /// - BlobAlreadyAbsent = 2, - - /// - /// Reported with . The blob was left in place because it - /// did not carry the payload store's ownership marker, meaning the store did not write it. The token text - /// merely matched the v2 grammar; the payload column is customer-writable, so matching text is not proof of - /// ownership. This is an expected outcome rather than a defect. The tombstone is still resolved, because a - /// blob the store does not own will never become deletable and retrying forever would leak the row. Reported - /// separately so that "resolved without reclaiming bytes" stays countable. - /// - BlobNotStoreOwned = 3, - - /// - /// Reported with . The deletion failed against storage: - /// network failure, timeout, outage, throttling, an unreachable account, or an authorization failure. All of - /// these are reconfigurable or self-healing, and they are not subdivided here because - /// already carries the specific status. Subdividing - /// would encode the same fact twice. - /// - StorageFailure = 10, - - /// - /// Reported with . The registered payload store does not - /// implement deletion. Every payload fails the same way, so this is a deployment-wide condition rather than a - /// per-row one, and it is kept recoverable until an operator registers a store that can delete. - /// is empty because storage was never contacted, which - /// is why this cannot be folded into . - /// - StoreCannotDelete = 11, - - /// - /// The worker could not act on the token. This reason is reported with TWO dispositions, and a consumer must - /// not assume either one. - /// - /// Reported with when the token can never become - /// usable: its body does not parse, it is a legacy v1 token, or storage rejected a well-formed token as - /// permanently invalid. The SDK and backend control both sides of this protocol, so reaching that state - /// indicates a producer, corruption, or compatibility bug, and the evidence is preserved rather than - /// discarded. - /// - /// - /// Reported with in exactly one case: the token names a - /// version this worker does not understand. Nothing is wrong with that token, since a newer worker can read - /// it, so an SDK upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and - /// unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. - /// - /// distinguishes the storage-rejected case, where it - /// is populated, from the parse and unknown-version cases, where it is empty. - /// - TokenNotPurgeable = 20, -} diff --git a/src/Client/Core/LargePayloadPurgeResult.cs b/src/Client/Core/LargePayloadPurgeResult.cs index f63f45fb..0a3ee98a 100644 --- a/src/Client/Core/LargePayloadPurgeResult.cs +++ b/src/Client/Core/LargePayloadPurgeResult.cs @@ -6,11 +6,19 @@ namespace Microsoft.DurableTask.Client; /// /// Serializable outcome of exactly one attempted large-payload blob deletion. Mirrors the /// LargePayloadPurgeResult protobuf message but is safe to pass through the orchestration/activity -/// boundary. The backend owns retry scheduling: it deletes rows reported as +/// boundary. The backend owns retry scheduling and branches solely on +/// : it deletes rows reported as /// , reschedules -/// with a reason-appropriate next attempt, and moves -/// rows out of the active fetch. +/// on its own backoff, and moves +/// rows out of the active fetch. The worker never +/// computes a retry delay. /// +/// +/// The disposition is deliberately the only outcome field: anything finer would be write-only on the backend. +/// Why an attempt failed stays in the worker's own telemetry, which holds the cause at full fidelity rather +/// than as a lossy classification, and a row is correlated to it by +/// (, , ). +/// /// The backend partition that owns the tombstoned row. /// The orchestration instance key the payload belonged to. /// The backend identifier of the tombstoned payload row. @@ -19,16 +27,9 @@ namespace Microsoft.DurableTask.Client; /// as a compare-and-swap guard. /// /// The disposition of the deletion attempt. -/// The stable reason code explaining the disposition. -/// -/// An optional bounded, sanitized storage status or error code for diagnostics (for example -/// BlobNotFound or 409). Never contains a token or raw exception text. -/// public sealed record LargePayloadPurgeResult( int PartitionId, long InstanceKey, long PayloadId, long Revision, - LargePayloadPurgeDisposition Disposition, - LargePayloadPurgeReason Reason, - string? StorageErrorCode = null); + LargePayloadPurgeDisposition Disposition); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index adaa2116..0fdbf161 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -677,11 +677,10 @@ public override async Task ReportLargePayloadPurgeResultsAsync( PayloadId = result.PayloadId, Revision = result.Revision, - // The managed enums declare the same numeric values as their protobuf counterparts, so the - // disposition and reason map across by value. + // The managed disposition enum declares the same numeric values as its protobuf counterpart, + // so it maps across by value. This is the only enum on the message and it only travels + // outbound, so the SDK can never receive a value it does not know. Disposition = (P.LargePayloadPurgeDisposition)result.Disposition, - Reason = (P.LargePayloadPurgeReason)result.Reason, - StorageErrorCode = result.StorageErrorCode ?? string.Empty, }); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs index 7304d4db..4d74b601 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Activities/DeleteExternalBlobActivity.cs @@ -12,8 +12,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Activity that deletes a single externalized payload blob given its token, and classifies the attempt as /// , , or -/// with a stable reason code. Deletion is idempotent, -/// so re-delivered tokens and concurrent workers are safe. +/// . Deletion is idempotent, so re-delivered tokens and +/// concurrent workers are safe. /// /// /// The split between retry and quarantine is whether the failure can self-heal, verified against the @@ -37,9 +37,12 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// failure is returned as a disposition rather than thrown. /// /// -/// Per design §7 no log here carries the token or raw exception text - a token exposes the storage account, -/// container, and blob path. Diagnostics are the stable reason enum plus a bounded, sanitized storage error -/// code; the token itself is preserved on the backend's quarantined row. +/// The reported result carries the disposition alone, so every branch below logs its cause where the cause is +/// still exact, rather than deriving it afterwards from a value that crossed the wire. That log is the only +/// record of why an attempt failed. Per design §7 it still carries neither the token nor raw exception text - +/// a token exposes the storage account, container, and blob path - so the cause is a bounded classification +/// string plus a bounded, sanitized storage error code. The token itself is preserved on the backend's +/// quarantined row. /// /// The payload store used to delete blobs. /// The logger instance. @@ -57,23 +60,7 @@ public override async Task RunAsync(TaskActivityContext contex { Check.NotNullOrEmpty(input, nameof(input)); - BlobPurgeOutcome outcome = await this.DeleteAsync(input); - - switch (outcome.Disposition) - { - case LargePayloadPurgeDisposition.Quarantined: - this.logger.BlobPurgeDeleteQuarantined(outcome.Reason.ToString(), outcome.StorageErrorCode); - break; - case LargePayloadPurgeDisposition.Retry: - this.logger.BlobPurgeDeleteRetryable(outcome.Reason.ToString(), outcome.StorageErrorCode); - break; - case LargePayloadPurgeDisposition.Deleted - when outcome.Reason == LargePayloadPurgeReason.BlobNotStoreOwned: - this.logger.BlobPurgeBlobNotStoreOwned(); - break; - } - - return outcome; + return await this.DeleteAsync(input); } /// @@ -106,94 +93,90 @@ async Task DeleteAsync(string token) // the only durable record of the blob, so the row is quarantined instead - the backend preserves // its token as evidence and stops polling it. The backend excludes v1 at insertion time, so // reaching this branch is an invariant violation rather than an expected path. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); + this.logger.BlobPurgeDeleteQuarantined("LegacyV1Token", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } if (!token.StartsWith(BlobPayloadStore.TokenPrefixV2, StringComparison.Ordinal)) { // An unrecognized prefix is most likely a token written by a newer SDK than this worker runs. That - // recovers after an upgrade, so it earns a deferral rather than quarantine. This branch shares - // TokenNotPurgeable with the quarantined token cases, so its disposition is deliberately stated - // here rather than derived from the reason: folding it in would strand rows an upgrade would fix. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.TokenNotPurgeable); + // recovers after an upgrade, so it earns a deferral rather than quarantine. Quarantine is + // permanent and requires an operator to unwind; a deferral only leaves the row idle and visible, + // so an unrecognized token is deliberately kept on the recoverable side of that asymmetry. + this.logger.BlobPurgeDeleteRetryable("UnsupportedTokenVersion", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } try { PayloadDeleteOutcome outcome = await this.store.DeleteAsync(token, CancellationToken.None); - return outcome switch + + // The blob exists but this store never wrote it, so it was left untouched. That is an expected + // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it would + // fill the quarantine set with non-defects. The tombstone is still resolved, because a blob the + // store does not own is not the store's to delete. + if (outcome == PayloadDeleteOutcome.NotStoreOwned) { - PayloadDeleteOutcome.Deleted => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobDeleted), - PayloadDeleteOutcome.AlreadyAbsent => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobAlreadyAbsent), + this.logger.BlobPurgeBlobNotStoreOwned(); + } - // The blob exists but this store never wrote it, so it was left untouched. That is an expected - // outcome, not a defect - the token text merely matched the v2 grammar - and quarantining it - // would fill the quarantine set with non-defects. The tombstone is still resolved, because a - // blob the store does not own is not the store's to delete. - _ => new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Deleted, LargePayloadPurgeReason.BlobNotStoreOwned), - }; + // Deleted, AlreadyAbsent, and NotStoreOwned are all terminal successes: none can be improved by + // trying again. + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Deleted); } catch (ArgumentException) { // The prefix gate above proves this is a v2 token, so the only remaining decode failure is a v2 // body that does not parse. The SDK and backend control both sides of the protocol, so that // indicates a producer, corruption, or compatibility bug; retrying can never fix it. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, LargePayloadPurgeReason.TokenNotPurgeable); + this.logger.BlobPurgeDeleteQuarantined("MalformedToken", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } catch (NotSupportedException) { // The registered store does not implement deletion. Every payload would fail the same way, so the // work is kept recoverable until an operator registers a store that can delete. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StoreCannotDelete); + this.logger.BlobPurgeDeleteRetryable("StoreCannotDelete", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (PayloadStorageException) { // The token is well formed but points at a storage account this worker's credential cannot reach // (account-key auth is account-specific). Recoverable after a configuration or credential change, // so it is deferred rather than discarded. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); + this.logger.BlobPurgeDeleteRetryable("StorageAccountUnreachable", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (RequestFailedException ex) when (ex.Status == (int)HttpStatusCode.BadRequest) { // Storage rejected a request generated from a well-formed token as permanently invalid (for // example InvalidUri / InvalidResourceName). Retrying can never succeed. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Quarantined, - LargePayloadPurgeReason.TokenNotPurgeable, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteQuarantined("InvalidStorageRequest", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Quarantined); } catch (RequestFailedException ex) when ( ex.Status == (int)HttpStatusCode.Unauthorized || ex.Status == (int)HttpStatusCode.Forbidden) { // Authorization can be transient or fixed by reconfiguration, so it stays recoverable rather than // dropping data an operator can still reclaim. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageFailure, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteRetryable("StorageAuthorizationFailed", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (RequestFailedException ex) { // Throttling, 5xx, and anything else the service reported, including a failed If-Match on the // ownership check: transient by default. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, - LargePayloadPurgeReason.StorageFailure, - SanitizeErrorCode(ex)); + this.logger.BlobPurgeDeleteRetryable("TransientStorageFailure", SanitizeErrorCode(ex)); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { // Timeouts, cancellation, and network failures. A blob is never dropped on an uncertain error. - return new BlobPurgeOutcome( - LargePayloadPurgeDisposition.Retry, LargePayloadPurgeReason.StorageFailure); + // Storage reported no code here, so the exception's type name is appended to the cause: it is a + // bounded value that cannot carry a token, and it is the only thing separating a timeout from a + // cancellation or a DNS failure now that no classification crosses the wire. + this.logger.BlobPurgeDeleteRetryable($"UnexpectedFailure:{ex.GetType().Name}", null); + return new BlobPurgeOutcome(LargePayloadPurgeDisposition.Retry); } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index c20cc5d4..8b40c970 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -19,8 +19,8 @@ static partial class Logs [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] public static partial void BlobPurgeJobOrchestratorStopping(this ILogger logger, string? jobId, string status); - [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; reason '{reason}', storage error code '{storageErrorCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it.")] - public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string reason, string? storageErrorCode); + [LoggerMessage(EventId = 813, Level = LogLevel.Warning, Message = "Blob payload auto-purge quarantined a payload; cause '{cause}', storage code '{storageCode}'. The failure is deterministic and cannot succeed on a retry. The backend preserves the tombstone row and its token as evidence and stops polling it. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteQuarantined(this ILogger logger, string cause, string? storageCode); [LoggerMessage(EventId = 814, Level = LogLevel.Debug, Message = "Blob payload auto-purge fetched {count} tombstoned payload(s) from the backend.")] public static partial void BlobPurgeFetchedTombstones(this ILogger logger, int count); @@ -34,8 +34,8 @@ static partial class Logs [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); - [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; reason '{reason}', storage error code '{storageErrorCode}'. The backend reschedules the tombstone for a later attempt.")] - public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string reason, string? storageErrorCode); + [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; cause '{cause}', storage code '{storageCode}'. The backend reschedules the tombstone for a later attempt. The reported result carries the disposition alone, so this log is the only record of the cause.")] + public static partial void BlobPurgeDeleteRetryable(this ILogger logger, string cause, string? storageCode); [LoggerMessage(EventId = 820, Level = LogLevel.Warning, Message = "Blob payload auto-purge cycle for job '{jobId}' failed; backing off before retrying so the job keeps running.")] public static partial void BlobPurgeCycleFailed(this ILogger logger, Exception exception, string? jobId); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs index e5c563ec..7f4a1714 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeOutcome.cs @@ -10,13 +10,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// orchestrator combines it with the tombstone's identity and revision to build the reported /// . /// +/// +/// Carries the disposition alone, because that is the only outcome field the contract reports. Why an attempt +/// reached its disposition is logged by at the point it is +/// classified, at higher fidelity than any value that could be carried here. +/// /// Whether the row is resolved, should be retried, or must be quarantined. -/// The stable reason code explaining the disposition. -/// -/// An optional bounded, sanitized storage status or error code for diagnostics. Never a token or raw -/// exception text. -/// -public sealed record BlobPurgeOutcome( - LargePayloadPurgeDisposition Disposition, - LargePayloadPurgeReason Reason, - string? StorageErrorCode = null); +public sealed record BlobPurgeOutcome(LargePayloadPurgeDisposition Disposition); diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 7d1730c6..4e964612 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -184,8 +184,6 @@ async Task DeleteOneAsync( tombstone.InstanceKey, tombstone.PayloadId, tombstone.Revision, - outcome.Disposition, - outcome.Reason, - outcome.StorageErrorCode); + outcome.Disposition); } } diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 863f41b0..51f490a2 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -872,74 +872,14 @@ enum LargePayloadPurgeDisposition { // The failure may self-heal, so the row stays pending and the backend sets the next attempt. LARGE_PAYLOAD_PURGE_DISPOSITION_RETRY = 2; - // A deterministic failure or protocol violation that retrying can never fix. The backend - // preserves the evidence, alerts, and stops automatic retries. + // A deterministic failure or protocol violation that retrying can never fix. The row leaves the + // polling set but is never deleted or expired: it keeps the token, which after the payload row is + // gone is the only durable record of the blob, so discarding it would orphan the blob silently. + // Resolving a quarantined row is a deliberate operator action. Why it failed is not recorded here + // and is not meant to be; that detail lives in the worker's telemetry at full fidelity. LARGE_PAYLOAD_PURGE_DISPOSITION_QUARANTINED = 3; } -// Why a row received its disposition. Diagnostic only: the backend acts on `disposition` alone and -// never branches on this value, so it exists to make a stuck or non-reclaiming ledger explainable -// without access to worker logs, which run in the customer's process. -// -// Deliberately coarse. Granularity matches the number of distinct operator responses, not the -// number of distinct causes, because `storageErrorCode` already carries the specific storage status. -// Values must never carry a token or raw exception text: a token exposes the storage account, -// container, and blob path. -// -// Reason and disposition are orthogonal and there is no fixed mapping between them. A reason says -// what the worker found; a disposition says whether trying again can change it. Most reasons occur -// with exactly one disposition, but TOKEN_NOT_PURGEABLE deliberately occurs with two. Do not assert -// a reason-to-disposition mapping anywhere. -enum LargePayloadPurgeReason { - LARGE_PAYLOAD_PURGE_REASON_UNSPECIFIED = 0; - - // Reported with DELETED. The blob was deleted by this attempt, reclaiming its bytes. - LARGE_PAYLOAD_PURGE_REASON_BLOB_DELETED = 1; - - // Reported with DELETED. The blob was already absent, so deletion was a no-op and no bytes were - // reclaimed by this attempt. Deletion is idempotent, so this is a success rather than a failure. - // Reported separately from BLOB_DELETED because a high rate of it indicates duplicate tombstones. - LARGE_PAYLOAD_PURGE_REASON_BLOB_ALREADY_ABSENT = 2; - - // Reported with DELETED. The blob was left in place because it did not carry the payload store's - // ownership marker, meaning the store did not write it. The token text merely matched the v2 - // grammar; the payload column is customer-writable, so matching text is not proof of ownership. - // This is an expected outcome rather than a defect. The tombstone is still resolved, because a - // blob the store does not own will never become deletable and retrying forever would leak the - // row. Reported separately so that "resolved without reclaiming bytes" stays countable. - LARGE_PAYLOAD_PURGE_REASON_BLOB_NOT_STORE_OWNED = 3; - - // Reported with RETRY. The deletion failed against storage: network failure, timeout, outage, - // throttling, an unreachable account, or an authorization failure. All of these are - // reconfigurable or self-healing, and they are not subdivided here because `storageErrorCode` - // already carries the specific status. Subdividing would encode the same fact twice. - LARGE_PAYLOAD_PURGE_REASON_STORAGE_FAILURE = 10; - - // Reported with RETRY. The registered payload store does not implement deletion. Every payload - // fails the same way, so this is a deployment-wide condition rather than a per-row one, and it - // is kept recoverable until an operator registers a store that can delete. `storageErrorCode` is - // empty because storage was never contacted, which is why this cannot be folded into - // STORAGE_FAILURE. - LARGE_PAYLOAD_PURGE_REASON_STORE_CANNOT_DELETE = 11; - - // The worker could not act on the token. This reason is reported with TWO dispositions, and a - // consumer must not assume either one. - // - // Reported with QUARANTINED when the token can never become usable: its body does not parse, it - // is a legacy v1 token, or storage rejected a well-formed token as permanently invalid. The SDK - // and backend control both sides of this protocol, so reaching that state indicates a producer, - // corruption, or compatibility bug, and the evidence is preserved rather than discarded. - // - // Reported with RETRY in exactly one case: the token names a version this worker does not - // understand. Nothing is wrong with that token, since a newer worker can read it, so an SDK - // upgrade resolves it. The asymmetry is deliberate: quarantining it would be permanent and - // unrecoverable, whereas a retry that never succeeds only leaves the row idle and visible. - // - // `storageErrorCode` distinguishes the storage-rejected case, where it is populated, from the - // parse and unknown-version cases, where it is empty. - LARGE_PAYLOAD_PURGE_REASON_TOKEN_NOT_PURGEABLE = 20; -} - // client -> server: the outcome of exactly one tombstoned row. message LargePayloadPurgeResult { // Row identity, echoed from the corresponding LargePayloadTombstone. @@ -950,12 +890,11 @@ message LargePayloadPurgeResult { // Echoed unmodified from the fetched tombstone; used as a compare-and-swap guard. int64 revision = 4; + // The only field the backend acts on. Deliberately the only outcome field on this message: + // anything finer would be write-only. Failure detail stays in the worker's own telemetry, which + // holds the full exception rather than a lossy classification, and a row is correlated to it by + // (partitionId, instanceKey, payloadId) plus the ledger's LastAttemptAt. LargePayloadPurgeDisposition disposition = 5; - LargePayloadPurgeReason reason = 6; - - // Optional bounded, sanitized storage status or error code for diagnostics - // (for example "BlobNotFound" or "409"). Must never contain a token or raw exception text. - string storageErrorCode = 7; } // client -> server: request up to `limit` due tombstones for the caller's task hub. diff --git a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs index bd563897..48658410 100644 --- a/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs +++ b/test/Client/Grpc.Tests/LargePayloadPurgeEnumParityTests.cs @@ -8,9 +8,9 @@ namespace Microsoft.DurableTask.Client.Grpc.Tests; /// -/// maps the managed purge enums onto -/// their protobuf counterparts by numeric value rather than by name, which is only correct while the two sides -/// agree on every value. A silent drift would not fail to compile; it would send the backend a different +/// maps the managed purge disposition +/// onto its protobuf counterpart by numeric value rather than by name, which is only correct while the two +/// sides agree on every value. A silent drift would not fail to compile; it would send the backend a different /// disposition than the worker decided and delete or quarantine the wrong rows. These tests pin the mapping. /// public class LargePayloadPurgeEnumParityTests @@ -31,24 +31,9 @@ public void Disposition_ManagedAndProtobufValues_AreIdentical() managed.Should().Equal(proto); } - [Fact] - public void Reason_ManagedAndProtobufValues_AreIdentical() - { - // Arrange & Act - Dictionary managed = Enum.GetValues(typeof(LargePayloadPurgeReason)) - .Cast() - .ToDictionary(v => (int)v, v => v.ToString()); - Dictionary proto = Enum.GetValues(typeof(P.LargePayloadPurgeReason)) - .Cast() - .ToDictionary(v => (int)v, v => v.ToString()); - - // Assert - managed.Should().Equal(proto); - } - /// - /// The numeric casts are safe only because no enum crosses the wire inbound on this feature: the SDK - /// casts values it defined itself, so it can never receive an unknown value and silently reinterpret it. + /// The numeric cast is safe only because no enum crosses the wire inbound on this feature: the SDK + /// casts a value it defined itself, so it can never receive an unknown value and silently reinterpret it. /// That invariant holds today by the shape of the contract, not by construction, and nothing in the code /// states it. Adding an enum to an inbound type would create exactly that path - a newer backend sending a /// value this SDK does not know, mapped by raw numeric cast onto a valid-but-wrong member - and it would @@ -71,7 +56,7 @@ public void InboundTypes_ExposeNoEnumMembers(Type inboundType) // Assert enumMembers.Should().BeEmpty( - "an enum on an inbound type invalidates the numeric enum casts in " + + "an enum on an inbound type invalidates the numeric enum cast in " + "GrpcDurableTaskClient.ReportLargePayloadPurgeResultsAsync. The SDK would map a value chosen by the " + "backend - including one a newer backend added that this SDK does not know - onto a managed member " + "by raw numeric value, silently mis-dispositioning rows. Map inbound enums explicitly instead, with " + diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs index 7949cc57..94ad4830 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/DeleteExternalBlobActivityTests.cs @@ -9,25 +9,32 @@ namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; +/// +/// The reported outcome carries the disposition alone, so each case pins two things: the disposition, which is +/// what the backend acts on, and the logged cause, which is now the only record of why an attempt reached that +/// disposition. Several branches share a disposition and are told apart only by their cause, so asserting the +/// disposition alone would let two branches collapse into one unnoticed. +/// public class DeleteExternalBlobActivityTests { const string V2Token = "blob:v2:https://acct.blob.core.windows.net/payloads/abc123"; [Fact] - public async Task RunAsync_WhenDeleteThrowsRequestFailed400_QuarantinesAsTokenNotPurgeable() + public async Task RunAsync_WhenDeleteThrowsRequestFailed400_Quarantines() { // Arrange - a Status 400 (e.g. InvalidResourceName) is a permanent service rejection. StubPayloadStore store = new(new RequestFailedException(400, "bad", "InvalidResourceName", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert - quarantined (evidence preserved), never a success-shaped discard. The storage error code - // is what distinguishes this from the parse-failure cases, which share the same reason. + // Assert - quarantined (evidence preserved), never a success-shaped discard. The sanitized storage + // error code is logged alongside the cause and is what distinguishes this from the parse failures. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); - outcome.StorageErrorCode.Should().Be("InvalidResourceName"); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("InvalidStorageRequest") && l.Message.Contains("InvalidResourceName")); } [Fact] @@ -35,15 +42,16 @@ public async Task RunAsync_WhenDeleteThrowsRequestFailedNon400_RetriesAsTransien { // Arrange - a Status 503 that escaped the SDK's internal retries is still treated as transient. StubPayloadStore store = new(new RequestFailedException(503, "busy", "ServerBusy", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); - outcome.StorageErrorCode.Should().Be("ServerBusy"); + logger.Logs.Should().ContainSingle( + l => l.Message.Contains("TransientStorageFailure") && l.Message.Contains("ServerBusy")); } [Theory] @@ -53,31 +61,34 @@ public async Task RunAsync_WhenDeleteThrowsAuthorizationFailure_Retries(int stat { // Arrange - authorization can be fixed by reconfiguration, so it stays recoverable. StubPayloadStore store = new(new RequestFailedException(status, "denied", "AuthorizationFailure", null)); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAuthorizationFailed")); } [Fact] - public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_RetriesAsStorageFailure() + public async Task RunAsync_WhenDeleteThrowsPayloadStorageException_Retries() { // Arrange - the payload lives in a storage account the configured credential cannot reach. That is // recoverable after a configuration or credential change, so it is deferred rather than discarded. StubPayloadStore store = new(new PayloadStorageException("cross-account delete requires identity auth")); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync( null!, "blob:v2:https://other.blob.core.windows.net/c/abc123"); - // Assert + // Assert - the cause is what separates this from the other retryable storage failures, which the + // contract no longer distinguishes. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StorageAccountUnreachable")); } [Fact] @@ -86,14 +97,15 @@ public async Task RunAsync_V1Token_QuarantinesWithoutCallingStore() // Arrange - a v1 token names a container but not the storage account, so a delete against the // configured account cannot be verified and would falsely report success if the store was repointed. Mock store = new(); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v1:payloads:abc123"); // Assert - quarantined by the gate, and the store's DeleteAsync was never invoked. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("LegacyV1Token")); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -102,16 +114,18 @@ public async Task RunAsync_UnknownTokenVersion_RetriesWithoutCallingStore() { // Arrange - an unrecognized prefix most likely came from a newer SDK, which recovers after an upgrade. Mock store = new(); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v9:https://acct.blob.core.windows.net/c/x"); - // Assert - retried, NOT quarantined: quarantine is terminal and this can self-heal. This is the one - // row where TokenNotPurgeable does not mean Quarantined, so the disposition is asserted deliberately: - // deriving it from the reason would strand rows that an SDK upgrade would have resolved. + // Assert - retried, NOT quarantined: quarantine is terminal and requires an operator to unwind, while a + // deferral only leaves the row idle and visible. This branch and the token branches that quarantine are + // told apart only by cause, so the disposition is asserted deliberately: folding them together would + // strand rows that an SDK upgrade would have resolved. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnsupportedTokenVersion")); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Never); } @@ -121,14 +135,15 @@ public async Task RunAsync_MalformedV2Token_Quarantines() // Arrange - a recognized v2 prefix whose body does not parse; the store signals that with // ArgumentException. Retrying can never fix a body the SDK itself produced malformed. StubPayloadStore store = new(new ArgumentException("Invalid token")); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, "blob:v2:not-a-uri"); - // Assert - contrast with the unknown-prefix case above, which shares this reason but is retried. + // Assert - contrast with the unknown-prefix case above, which is retried. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Quarantined); - outcome.Reason.Should().Be(LargePayloadPurgeReason.TokenNotPurgeable); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("MalformedToken")); } [Fact] @@ -138,14 +153,15 @@ public async Task RunAsync_V2Token_CallsStoreAndReportsDeleted() Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.Deleted); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert + // Assert - an ordinary success is silent; any log here would mean a failure branch was taken. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobDeleted); + logger.Logs.Should().BeEmpty(); store.Verify(s => s.DeleteAsync(It.IsAny(), It.IsAny()), Times.Once); } @@ -156,14 +172,15 @@ public async Task RunAsync_WhenBlobAlreadyAbsent_ReportsDeleted() Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.AlreadyAbsent); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobAlreadyAbsent); + logger.Logs.Should().BeEmpty(); } [Fact] @@ -173,15 +190,17 @@ public async Task RunAsync_WhenBlobNotStoreOwned_ResolvesTombstoneWithoutDeletin Mock store = new(); store.Setup(s => s.DeleteAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(PayloadDeleteOutcome.NotStoreOwned); - DeleteExternalBlobActivity activity = new(store.Object, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store.Object, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); // Assert - the tombstone is still resolved: a blob the store does not own is not the store's to delete, - // and re-serving the row forever would never make it deletable. + // and re-serving the row forever would never make it deletable. The reported result is now identical to + // an ordinary delete, so the log is the only thing that records the difference. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Deleted); - outcome.Reason.Should().Be(LargePayloadPurgeReason.BlobNotStoreOwned); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("ownership marker")); } [Fact] @@ -189,7 +208,8 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst { // Arrange - a store that cannot delete (the base PayloadStore.DeleteAsync throws NotSupportedException). StubPayloadStore store = new(new NotSupportedException()); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); @@ -197,7 +217,7 @@ public async Task RunAsync_WhenStoreDoesNotSupportDelete_RetriesToPreserveTombst // Assert - retried (tombstone preserved): resolving it would destroy the backend's cleanup ledger while // the blob survives. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StoreCannotDelete); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("StoreCannotDelete")); } [Fact] @@ -205,15 +225,15 @@ public async Task RunAsync_WhenDeleteTimesOut_RetriesAsTransient() { // Arrange - a non-Azure exception (timeout / network failure) must not drop a blob on doubt. StubPayloadStore store = new(new TimeoutException()); - DeleteExternalBlobActivity activity = new(store, new TestLogger()); + TestLogger logger = new(); + DeleteExternalBlobActivity activity = new(store, logger); // Act BlobPurgeOutcome outcome = await activity.RunAsync(null!, V2Token); - // Assert + // Assert - storage reported no code here, so the exception's type name is what identifies the failure. outcome.Disposition.Should().Be(LargePayloadPurgeDisposition.Retry); - outcome.Reason.Should().Be(LargePayloadPurgeReason.StorageFailure); - outcome.StorageErrorCode.Should().BeNull(); + logger.Logs.Should().ContainSingle(l => l.Message.Contains("UnexpectedFailure:TimeoutException")); } sealed class StubPayloadStore : PayloadStore From 1ac131ff00a53e66b601251a3762adbf84acc4e5 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 11:35:54 -0700 Subject: [PATCH 25/32] Dedupe the purge bridge in the backend instead of checking first EnsureJobAsync read the bridge orchestration's status and then scheduled it. Those are two independent RPCs with no atomicity between them, so two hosts starting together could both observe "absent" and both schedule. The status check is replaced by a dedupe policy on the create call, which makes the decision atomic in the backend. The policy is the exact semantic equivalent of the predicate it replaces: the old code rescheduled only when the bridge was absent, Failed, or Terminated, so Failed and Terminated are the only replaceable statuses and every other status is deduped. The list must name all five explicitly because the wire format is an inverted whitelist - the policy is computed as (all statuses - dedupe statuses), so any status omitted silently becomes replaceable. Pending in particular is not optional: instances are created Pending and only become Running after awaiting their first task, so leaving it out would open a real window in which a just-scheduled bridge is replaced. This also makes the OrchestrationAlreadyExistsException handler live. It was previously dead code: the exception is only thrown for statuses in DedupeStatuses, and no policy was supplied. It is now the normal steady-state outcome on every host start after the first, so its comment is corrected rather than its behavior. OrchestrationRuntimeStatus.Canceled is [Obsolete], so naming it emits CS0618. It is suppressed with a scoped pragma, matching how ProtoUtils and StartOrchestrationOptionsExtensions handle the same member. Dropping the status instead would have silently changed behavior on Canceled. Adds a test pinning the exact dedupe set. An omitted status is invisible to the compiler and would surface only as a healthy singleton being purged and replaced in production; EnsureJobAsync had no coverage before this. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 60 ++++++++++++------- .../AutoPurge/BlobPurgeJobStarterTests.cs | 46 ++++++++++++++ 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 3508120d..a2825430 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -129,41 +129,57 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT { try { - // The singleton is already guaranteed by the entity's fixed key (Create no-ops when the job is - // active) and the orchestrator's fixed instance id. The bridge orchestration's only job is to - // apply the entity's Create once, under a fixed instance id. Before (re)scheduling it, check the - // existing bridge: if it already Completed - or is still alive (Running/Pending/Suspended) - the - // job is set up, so do not reschedule. (Re-running a Completed bridge is wasteful: with a fixed - // id and no dedupe policy the backend would purge and replace the terminal instance on every - // host restart.) Only (re)schedule when the bridge is absent, or ended in a Failed/Terminated - // state that may never have applied Create - which lets a failed setup self-heal. - OrchestrationMetadata? existing = await client.GetInstanceAsync( - BlobPurgeConstants.StarterInstanceId, cancellationToken); - - bool needsSchedule = existing is null - or { RuntimeStatus: OrchestrationRuntimeStatus.Failed or OrchestrationRuntimeStatus.Terminated }; - if (!needsSchedule) - { - this.logger.BlobPurgeJobEnsured(); - return; - } - BlobPurgeJobOperationRequest request = new( this.entityId, nameof(BlobPurgeJob.Create), batchSize); + // The bridge orchestration's only job is to apply the entity's Create once, under a fixed + // instance id. Whether it should be (re)scheduled is decided by the backend as part of the + // create call, not by reading its status first: a read-then-schedule pair is two independent + // RPCs with no atomicity between them, so two hosts starting together can both observe "absent" + // and both schedule. + // + // The dedupe list is an inverted whitelist. The wire policy is computed as + // (all statuses - dedupe statuses) = the statuses that may be REPLACED, so any status left out + // of this list silently becomes replaceable. It must therefore name every status that must not + // be disturbed, not just the interesting ones: + // Completed - re-running a finished bridge is wasteful, and with a fixed id the backend + // would purge and replace the terminal instance on every host restart. There is + // nothing to gain: the bridge would only re-signal Create, which no-ops while + // the entity is Active. + // Canceled - terminal and not a failed setup, so it is left alone like Completed. + // Pending - instances are created Pending and only become Running after awaiting their + // first task, so omitting it would leave a real window in which a just-scheduled + // bridge is replaced. + // Running, + // Suspended - still alive; the job is already set up. + // Failed and Terminated are deliberately absent, which makes them the only replaceable + // statuses: those may never have applied Create, so rescheduling is what lets a failed setup + // self-heal. Terminated must also stay out of the list because supplying it alongside a + // reusable running status is rejected outright. +#pragma warning disable CS0618 // Canceled is obsolete, but is a real dedupe status and must be named here. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, - new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId), + new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId) + .WithDedupeStatuses( + OrchestrationRuntimeStatus.Completed, + OrchestrationRuntimeStatus.Canceled, + OrchestrationRuntimeStatus.Pending, + OrchestrationRuntimeStatus.Running, + OrchestrationRuntimeStatus.Suspended), cancellationToken); +#pragma warning restore CS0618 this.logger.BlobPurgeJobEnsured(); return; } catch (OrchestrationAlreadyExistsException) { - // Race: another client scheduled the bridge between our status check and schedule call. That is - // fine - the singleton is already kicked off; treat it as ensured and stop. + // The expected steady-state outcome, not a rare race: the backend throws this whenever the + // bridge already exists in one of the dedupe statuses above, which is every host start after + // the first one succeeded. It means the singleton is already set up, so treat it as ensured + // and stop. It also covers the race this replaced a status check to close - two hosts starting + // together - because exactly one create wins and the loser lands here. this.logger.BlobPurgeJobEnsured(); return; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index f79c79e6..a5c79d32 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -86,6 +86,52 @@ public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() logger.Logs.Should().BeEmpty(); } + [Fact] + public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTerminated() + { + // Arrange - the dedupe list is an inverted whitelist: the wire policy is (all statuses - dedupe), so a + // status omitted from the call silently becomes replaceable. Nothing in the compiler or the type system + // catches that, so the exact set is pinned here. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + Mock client = new("test"); + TaskCompletionSource scheduled = new(); + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback( + (_, _, options, _) => scheduled.TrySetResult(options)) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client.Object); + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), + "test", + new TestLogger()); + + // Act - the ensure work runs on a background task, so wait for the scheduling call rather than assuming + // it already happened. + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(scheduled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + // Assert + completed.Should().BeSameAs(scheduled.Task, "the starter must schedule the bridge orchestration"); + StartOrchestrationOptions? options = await scheduled.Task; + options.Should().NotBeNull(); + options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); + + // Failed and Terminated are the only replaceable statuses, which is what lets a failed setup self-heal. + // Every other status is deduped so a healthy or finished bridge is never purged and replaced. + options.DedupeStatuses.Should().BeEquivalentTo( + ["Completed", "Canceled", "Pending", "Running", "Suspended"]); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); From ea0b4c773f6fe7490cf4a17c3fab690d8bbc94a3 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 14:35:49 -0700 Subject: [PATCH 26/32] Dedupe the purge bridge on Pending and Running only The bridge deduped five statuses, which preserved the exact semantics of the status check it replaced. That preserved a dead end along with them: a bridge that had already run was Completed and therefore never re-ran, so there was no path back from a job that had stopped. Deduping only Pending and Running says something narrower - while the bridge is alive leave it alone, in any other state re-run it - and that is what makes the job self-healing. The perpetual orchestrator exits cleanly when it reads back a null entity state, which is reachable when the entity is removed by CleanEntityStorageAsync. A removed entity is back to its default Pending status, so the job is left dead with a Completed bridge behind it. Making Completed replaceable means the next host start re-runs Create, which finds the entity not Active and rebuilds the job. Re-running a finished bridge is cheap and safe: its only effect is calling Create, which no-ops while the entity is Active, so the steady-state cost is one instance replacement plus one entity call per host start. Pending stays in the list. Instances are created Pending and only become Running after awaiting their first task, so omitting it would leave a real window in which a just-scheduled bridge is replaced. Two comments were wrong and are corrected rather than carried forward. The first claimed the backend would purge and replace a terminal instance on every host restart; a null DedupeStatuses field means the backend default, which the contract explicitly leaves undefined, so that asserted behavior the SDK does not promise. The second described the OrchestrationAlreadyExistsException handler as the steady-state outcome. That was true when Completed was deduped, but under this policy a finished bridge is replaced rather than rejected, so the handler is once again the concurrent-start race path it was originally written for. The CS0618 suppression is removed with Canceled. ContinuedAsNew and Canceled are the only obsolete members of OrchestrationRuntimeStatus, so with Canceled gone the pragma covered nothing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 57 ++++++++++--------- .../AutoPurge/BlobPurgeJobStarterTests.cs | 10 ++-- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index a2825430..7a911feb 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -140,46 +140,49 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // The dedupe list is an inverted whitelist. The wire policy is computed as // (all statuses - dedupe statuses) = the statuses that may be REPLACED, so any status left out - // of this list silently becomes replaceable. It must therefore name every status that must not - // be disturbed, not just the interesting ones: - // Completed - re-running a finished bridge is wasteful, and with a fixed id the backend - // would purge and replace the terminal instance on every host restart. There is - // nothing to gain: the bridge would only re-signal Create, which no-ops while - // the entity is Active. - // Canceled - terminal and not a failed setup, so it is left alone like Completed. - // Pending - instances are created Pending and only become Running after awaiting their - // first task, so omitting it would leave a real window in which a just-scheduled - // bridge is replaced. - // Running, - // Suspended - still alive; the job is already set up. - // Failed and Terminated are deliberately absent, which makes them the only replaceable - // statuses: those may never have applied Create, so rescheduling is what lets a failed setup - // self-heal. Terminated must also stay out of the list because supplying it alongside a - // reusable running status is rejected outright. -#pragma warning disable CS0618 // Canceled is obsolete, but is a real dedupe status and must be named here. + // of this list silently becomes replaceable. Deduping exactly Pending and Running therefore + // means: while the bridge is alive leave it alone, and in any other state re-run it. + // + // Pending is the subtle one and is not optional. Instances are created Pending and only become + // Running after awaiting their first task, so omitting it would leave a real window in which a + // just-scheduled bridge is replaced. + // + // Re-running a bridge that already finished is safe and close to free: the bridge's only effect + // is calling Create, which no-ops while the entity is Active, so the cost is one instance + // replacement plus one entity call per host start. + // + // Re-running is also what lets the job self-heal after the entity is removed, for example by + // CleanEntityStorageAsync. The perpetual orchestrator exits cleanly when it reads back a null + // entity state, and a removed entity is back to its default Pending status, so the job is left + // dead with a Completed bridge behind it. Keeping Completed deduped would keep it dead + // permanently; making it replaceable means the next host start re-runs Create, which finds the + // entity not Active and rebuilds the job. + // + // This deliberately does NOT recover the case where the perpetual orchestrator dies while the + // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the + // orchestrator stays down. That gap is tracked separately as an open lifecycle item. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, new StartOrchestrationOptions(BlobPurgeConstants.StarterInstanceId) .WithDedupeStatuses( - OrchestrationRuntimeStatus.Completed, - OrchestrationRuntimeStatus.Canceled, OrchestrationRuntimeStatus.Pending, - OrchestrationRuntimeStatus.Running, - OrchestrationRuntimeStatus.Suspended), + OrchestrationRuntimeStatus.Running), cancellationToken); -#pragma warning restore CS0618 this.logger.BlobPurgeJobEnsured(); return; } catch (OrchestrationAlreadyExistsException) { - // The expected steady-state outcome, not a rare race: the backend throws this whenever the - // bridge already exists in one of the dedupe statuses above, which is every host start after - // the first one succeeded. It means the singleton is already set up, so treat it as ensured - // and stop. It also covers the race this replaced a status check to close - two hosts starting - // together - because exactly one create wins and the loser lands here. + // Thrown only when the bridge already exists in one of the dedupe statuses above, so under this + // policy it means another host scheduled the bridge and it is still Pending or Running. That is + // exactly the concurrent-start race this replaced a status check to close: one create wins and + // the loser lands here. Either way the singleton is already being set up, so treat it as + // ensured and stop. + // + // Note this is NOT the steady-state path. A bridge that already finished is Completed, which is + // replaceable, so a later host start re-runs it rather than landing here. this.logger.BlobPurgeJobEnsured(); return; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index a5c79d32..2aea6a84 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -87,7 +87,7 @@ public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() } [Fact] - public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTerminated() + public async Task EnsureJob_SchedulesBridge_DedupingOnlyPendingAndRunning() { // Arrange - the dedupe list is an inverted whitelist: the wire policy is (all statuses - dedupe), so a // status omitted from the call silently becomes replaceable. Nothing in the compiler or the type system @@ -126,10 +126,10 @@ public async Task EnsureJob_SchedulesBridge_DedupingEveryStatusExceptFailedAndTe options.Should().NotBeNull(); options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); - // Failed and Terminated are the only replaceable statuses, which is what lets a failed setup self-heal. - // Every other status is deduped so a healthy or finished bridge is never purged and replaced. - options.DedupeStatuses.Should().BeEquivalentTo( - ["Completed", "Canceled", "Pending", "Running", "Suspended"]); + // Every status other than Pending and Running is replaceable, so a finished bridge is re-run on the + // next host start. That is what lets the job rebuild itself after the entity is removed. The set is + // asserted exactly, never as a superset, because the hazard is a silent omission. + options.DedupeStatuses.Should().BeEquivalentTo(["Pending", "Running"]); } static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) From 3c5c7ba0152fdc357496fd27f9e781c716e7c404 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 16:13:16 -0700 Subject: [PATCH 27/32] Add Stop operation and make the purge batch size configurable at runtime Answers the lifecycle gap on the singleton purge job: disabling auto-purge or changing the batch size was permanently ignored once the job existed. Stop: a new entity operation moves the job off Active and is idempotent. The perpetual orchestrator is deliberately not terminated - it already reads the entity at the top of every cycle and already has a clean-exit path, so Stop simply makes that path reachable. Shutdown is cooperative and race-free: the in-flight cycle finishes and the orchestrator exits on its own terms. Disable path: the starter used to return silently when auto-purge was off, so no host ever told the backend and a job created while the flag was on kept deleting blobs forever. It now signals Stop on the same fire-and-forget, retrying background task as the enable path. The store gate is deliberately not applied there: stopping needs no ability to delete, and a user who has switched to a store that cannot delete is exactly who needs the job stopped. Batch size: two independent breaks, both fixed. Create now takes the batch size even when the job is already Active (without re-signalling Run), and the orchestrator reads it from the entity state it already fetches each cycle instead of from its fixed input, falling back to the input when the stored value is not positive. Automatic restart of a dead orchestrator is a deliberate non-goal, documented in place: the only available restart schedules on the entity path, which carries no dedupe policy, so signalling a healthy orchestrator would terminate and replace it mid-work. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 94 ++++++++++++--- .../AutoPurge/Entity/BlobPurgeJob.cs | 56 ++++++++- .../AzureBlobPayloads/AutoPurge/Logs.cs | 11 +- .../AutoPurge/Models/BlobPurgeJobStatus.cs | 6 +- .../BlobPurgeJobOrchestrator.cs | 24 +++- .../BlobPurgeJobOrchestratorTests.cs | 113 ++++++++++++++++++ .../AutoPurge/BlobPurgeJobStarterTests.cs | 81 ++++++++++--- .../AutoPurge/BlobPurgeJobTests.cs | 93 +++++++++++++- 8 files changed, 434 insertions(+), 44 deletions(-) create mode 100644 test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 7a911feb..477eac9b 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -11,12 +11,12 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// -/// Client-side hosted service that ensures the per-task-hub singleton blob payload auto-purge job exists. It is -/// registered unconditionally by UseExternalizedPayloads and decides what to do at startup, once options are -/// fully resolved: it no-ops silently when auto-purge is disabled, and no-ops with an error log when the -/// registered store cannot delete. It never blocks host startup - the ensure work runs on a background task -/// that retries until the backend is reachable. The job is a per-task-hub singleton, so racing client -/// processes simply no-op. +/// Client-side hosted service that reconciles the per-task-hub singleton blob payload auto-purge job with the +/// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at +/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, asks the job +/// to stop when it is disabled, and no-ops with an error log when the registered store cannot delete. It never +/// blocks host startup - the work runs on a background task that retries until the backend is reachable. The +/// job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable { @@ -30,7 +30,7 @@ sealed class BlobPurgeJobStarter : IHostedService, IDisposable readonly EntityInstanceId entityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); CancellationTokenSource? cts; - Task? ensureTask; + Task? backgroundTask; /// /// Initializes a new instance of the class. @@ -63,10 +63,23 @@ public Task StartAsync(CancellationToken cancellationToken) // auto-purge is enabled can only be known once options are fully resolved - the flag can be set by the // inline configure delegate, services.Configure, configuration binding or PostConfigure. Deciding at // registration time (by running the delegate against a probe instance) both invoked user code twice and - // missed every enable path except the inline delegate. This is the normal path for apps that externalize - // payloads without auto-purge, so it returns silently without logging. + // missed every enable path except the inline delegate. if (!opts.AutoPurge) { + // Turning auto-purge off has to reach the backend to mean anything. The job is a perpetual + // orchestrator owned by the task hub, not by this process, so a job created while the flag was on + // keeps deleting blobs forever no matter how many hosts start with it off. Ask it to stop. + // + // The store gate below deliberately does NOT apply to this path. Stopping a job requires no ability + // to delete anything, and a user who has switched to a store that cannot delete is precisely + // someone who needs the job stopped; that gate's error is worded for the enabled case and would be + // a false alarm here. + // + // One deliberate cost: entity state is persisted after every operation, so this signal creates the + // job entity in its default, not-running state for apps that never enabled auto-purge. That is a + // single small entity per task hub, and it is the same one the job would use if it were enabled. + this.cts = new CancellationTokenSource(); + this.backgroundTask = Task.Run(() => this.SignalJobStopAsync(this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -91,7 +104,7 @@ public Task StartAsync(CancellationToken cancellationToken) // Do not block host startup; ensure the job on a background task with basic retry until the backend // is reachable. this.cts = new CancellationTokenSource(); - this.ensureTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); + this.backgroundTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; } @@ -100,20 +113,21 @@ public async Task StopAsync(CancellationToken cancellationToken) { this.cts?.Cancel(); - Task? pending = this.ensureTask; + Task? pending = this.backgroundTask; if (pending is not null) { - // The ensure loop observes cancellation and returns promptly; swallow any faulted/cancelled result. + // The background loop observes cancellation and returns promptly; swallow any faulted/cancelled + // result. await Task.WhenAny(pending, Task.Delay(Timeout.Infinite, cancellationToken)).ConfigureAwait(false); } } /// - /// Disposes the cancellation source backing the background ensure task. + /// Disposes the cancellation source backing the background task. /// /// /// Deliberately not disposed in : that method stops waiting as soon as the host's - /// shutdown token fires, so the ensure task may still hold this source's token. Disposing it there would + /// shutdown token fires, so the background task may still hold this source's token. Disposing it there would /// fault that still-running task with an when it next registers a /// callback. The container disposes singletons after every has returned, which is /// the safe point. @@ -160,7 +174,12 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // This deliberately does NOT recover the case where the perpetual orchestrator dies while the // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the - // orchestrator stays down. That gap is tracked separately as an open lifecycle item. + // orchestrator stays down. That is a deliberate non-goal rather than an oversight. The only + // available restart is signalling the entity's Run, which schedules the orchestrator on the + // entity path with no dedupe policy available to it, so signalling a healthy orchestrator would + // terminate and replace it mid-work. Deciding it is safe to signal requires a check-then-act on + // the orchestrator's status, which is exactly the non-atomic pattern this call replaced. + // Recovery is therefore a supervised manual step. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, @@ -204,4 +223,49 @@ await client.ScheduleNewOrchestrationInstanceAsync( } } } + + async Task SignalJobStopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + // Resolved inside the loop rather than on the host-start path. This path is taken by every app + // that externalizes payloads without auto-purge, and constructing a DurableTaskClient at host + // start for them is exactly what the enable path's lazy resolution avoids. It also means a + // client that cannot be resolved yet is retried here instead of throwing out of host startup, + // which must never happen for a feature the user has turned off. + DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + + // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no + // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the + // same state and a repeat costs nothing beyond the signal itself. + await client.Entities.SignalEntityAsync( + this.entityId, + nameof(BlobPurgeJob.Stop), + null, + null, + cancellationToken); + + this.logger.BlobPurgeJobStopRequested(); + return; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + this.logger.BlobPurgeStarterRetry(ex); + try + { + await Task.Delay(RetryDelay, cancellationToken); + } + catch (OperationCanceledException) + { + return; + } + } + } + } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 2d55a979..0c13984a 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,9 +13,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this is - /// intentionally a no-op when the job is already so that extra - /// client processes racing to create it do not disturb the running job. + /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this does not + /// restart a job that is already so that extra client processes + /// racing to create it do not disturb the running job. It does still take the batch size in that case, which + /// is what lets a configuration change reach a job that is already running. /// /// The entity context. /// @@ -25,6 +26,14 @@ public void Create(TaskEntityContext context, int purgeBatchSize) { if (this.State.Status == BlobPurgeJobStatus.Active) { + // Deliberately not re-signalling Run: the orchestrator is already up, and starting a second one + // over a live one is destructive. The batch size is still taken, because this is the only path by + // which a changed configuration reaches an active job - the orchestrator re-reads it from here + // every cycle. Without this the value written by the very first Create would be the only one the + // job ever used, and a batch size the backend rejects would wedge it permanently. + this.State.PurgeBatchSize = purgeBatchSize; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + logger.BlobPurgeJobAlreadyRunning(context.Id.Key); return; } @@ -64,6 +73,47 @@ public void Run(TaskEntityContext context) this.State.LastModifiedAt = DateTimeOffset.UtcNow; } + /// + /// Stops the auto-purge job. + /// + /// + /// + /// The perpetual orchestrator is deliberately not terminated here, and this operation does not touch its + /// instance ID at all. The orchestrator reads this entity at the top of every cycle and exits on its own + /// once the job is no longer , so shutdown is cooperative: there is + /// no window in which one party terminates an orchestrator that the other believes is healthy, and the + /// in-flight cycle finishes rather than being cut off part-way through a batch of deletes. + /// + /// + /// , and + /// are preserved. They are the job's history and its + /// configuration, both of which are wanted if it is started again, and keeping CreatedAt is also what + /// distinguishes a stopped job from one that was never started. + /// + /// + /// The entity context. + public void Stop(TaskEntityContext context) + { + if (this.State.Status != BlobPurgeJobStatus.Active) + { + // Idempotent, and the repeat is the common case rather than the exception: a host with auto-purge + // disabled signals Stop on every start, including for an app that never enabled auto-purge at all. + // Returning here leaves the state exactly as it was found. + // + // This does not avoid materializing the entity - the framework persists entity state after every + // operation, so a stop signal to an entity that does not exist yet creates it with default state. + // What the guard preserves is LastModifiedAt: rewriting it on every host restart would destroy its + // only useful meaning, which is when the job actually stopped. + logger.BlobPurgeJobAlreadyStopped(context.Id.Key); + return; + } + + this.State.Status = BlobPurgeJobStatus.Pending; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + + logger.BlobPurgeJobStopped(context.Id.Key); + } + /// /// Records progress after a purge cycle completes. /// diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index 8b40c970..a1f1eaf3 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -28,10 +28,13 @@ static partial class Logs [LoggerMessage(EventId = 815, Level = LogLevel.Debug, Message = "Blob payload auto-purge reported {count} purge result(s) to the backend.")] public static partial void BlobPurgeReportedResults(this ILogger logger, int count); + [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' stopped. The perpetual orchestrator is not terminated; it reads the job state at the start of its next cycle and exits on its own.")] + public static partial void BlobPurgeJobStopped(this ILogger logger, string? jobId); + [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); - [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not ensure the singleton job; retrying.")] + [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not reach the singleton job; retrying.")] public static partial void BlobPurgeStarterRetry(this ILogger logger, Exception exception); [LoggerMessage(EventId = 819, Level = LogLevel.Warning, Message = "Blob payload auto-purge could not delete a payload; cause '{cause}', storage code '{storageCode}'. The backend reschedules the tombstone for a later attempt. The reported result carries the disposition alone, so this log is the only record of the cause.")] @@ -43,6 +46,12 @@ static partial class Logs [LoggerMessage(EventId = 821, Level = LogLevel.Warning, Message = "An externalized payload blob does not carry this store's ownership marker, so it was left untouched; the tombstone is still resolved. This is expected for payloads written before the marker shipped, and for blobs the store never created whose token text matches the payload token grammar.")] public static partial void BlobPurgeBlobNotStoreOwned(this ILogger logger); + [LoggerMessage(EventId = 822, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already stopped; ignoring the stop request.")] + public static partial void BlobPurgeJobAlreadyStopped(this ILogger logger, string? jobId); + [LoggerMessage(EventId = 823, Level = LogLevel.Error, Message = "Blob payload auto-purge is enabled but the registered PayloadStore ('{storeType}') is not an Azure Blob payload store and cannot delete payloads. The auto-purge job was not started; externalized payloads will not be reclaimed. Register the Azure Blob payload store, or disable AutoPurge.")] public static partial void BlobPurgeStoreCannotDelete(this ILogger logger, string? storeType); + + [LoggerMessage(EventId = 824, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled, so a stop was requested for the singleton job. A job left running by an earlier configuration exits after its current cycle.")] + public static partial void BlobPurgeJobStopRequested(this ILogger logger); } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs index 97c37e66..fe6fe99c 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobStatus.cs @@ -9,8 +9,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; public enum BlobPurgeJobStatus { /// - /// The job has not been started yet. This is the default status of a freshly initialized entity, so it is - /// kept as the zero value to avoid a brand-new entity accidentally appearing active. + /// The job is not running. This is both the state of a job that has never been started and the resting + /// state of one that has been stopped, which distinguishes: it is + /// null only for a job that was never created. It is kept as the zero value so a brand-new entity does not + /// accidentally appear active. /// Pending, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs index 4e964612..a4e455e4 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Orchestrations/BlobPurgeJobOrchestrator.cs @@ -57,9 +57,14 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator( - input.JobEntityId, nameof(BlobPurgeJob.Get), null); + input.JobEntityId, nameof(BlobPurgeJob.Get), input: null); if (state is null || state.Status != BlobPurgeJobStatus.Active) { @@ -67,9 +72,22 @@ public class BlobPurgeJobOrchestrator : TaskOrchestrator 0 ? state.PurgeBatchSize : batchSize; + List tombstones = await context.CallActivityAsync>( nameof(GetLargePayloadTombstonesActivity), - batchSize, + cycleBatchSize, new TaskOptions(PurgeActivityRetryPolicy)); if (tombstones is null || tombstones.Count == 0) diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs new file mode 100644 index 00000000..609cdcb9 --- /dev/null +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobOrchestratorTests.cs @@ -0,0 +1,113 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using FluentAssertions; +using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Entities; +using Xunit; + +namespace Microsoft.DurableTask.Extensions.AzureBlobPayloads.Tests.AutoPurge; + +public class BlobPurgeJobOrchestratorTests +{ + static readonly EntityInstanceId JobEntityId = new(nameof(BlobPurgeJob), BlobPurgeConstants.JobId); + + readonly List requested = []; + readonly TestLogger logger = new(); + + [Fact] + public async Task RunAsync_UsesBatchSizeFromEntity_NotFromInput() + { + // Arrange - the orchestrator is perpetual, so its input is fixed at creation and carried verbatim + // through every continue-as-new. Reading the batch size from the input would pin the value written by + // the very first Create for the life of the job, which is what made a configuration change impossible + // to apply. The entity is the authority. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(777); + } + + [Fact] + public async Task RunAsync_WhenEntityBatchSizeIsUnset_FallsBackToInput() + { + // Arrange - an entity written by an older build carries no batch size at all. Passing that zero to the + // fetch activity would ask the backend for nothing on every cycle: a silent, permanent stall. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Active, PurgeBatchSize = 0 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert + this.AssertNoCycleFailed(); + this.requested.Should().Equal(100); + } + + [Fact] + public async Task RunAsync_WhenJobIsStopped_ExitsWithoutFetching() + { + // Arrange - Stop moves the entity off Active without touching this orchestrator, so the exit is the + // orchestrator's own decision on its own schedule. Nothing must be fetched on a stopped job. + Mock context = this.ContextFor( + new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending, PurgeBatchSize = 777 }); + + // Act + await new BlobPurgeJobOrchestrator().RunAsync( + context.Object, new BlobPurgeJobRunRequest(JobEntityId, PurgeBatchSize: 100)); + + // Assert - an empty fetch list is also what a broken mock produces, so the stopping log is asserted + // too: it is the only evidence that the orchestrator read the state and chose to exit. + this.AssertNoCycleFailed(); + this.requested.Should().BeEmpty(); + this.logger.Logs.Should().Contain(entry => entry.Message.Contains("stopping")); + } + + /// + /// Guards against the whole test passing through the orchestrator's catch-all cycle handler, which would + /// leave every observation empty and make the assertions vacuous. + /// + void AssertNoCycleFailed() => + this.logger.Logs.Should().NotContain(entry => entry.Message.Contains("cycle for job")); + + /// + /// Builds a context whose first entity read returns and whose second returns a + /// stopped job, so the perpetual loop runs at most one cycle and then exits. Batch sizes passed to the + /// fetch activity are recorded. + /// + Mock ContextFor(BlobPurgeJobState first) + { + Mock context = new(); + Mock entities = new(); + + context.Setup(c => c.Entities).Returns(entities.Object); + context.Setup(c => c.CreateReplaySafeLogger()).Returns(this.logger); + context.Setup(c => c.CreateTimer(It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + + entities + .SetupSequence(e => e.CallEntityAsync( + It.IsAny(), + nameof(BlobPurgeJob.Get), + It.IsAny(), + It.IsAny())) + .ReturnsAsync(first) + .ReturnsAsync(new BlobPurgeJobState { Status = BlobPurgeJobStatus.Pending }); + + context + .Setup(c => c.CallActivityAsync>( + It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((_, input, _) => this.requested.Add((int)input!)) + .ReturnsAsync([]); + + return context; + } +} diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 2aea6a84..2e6a3fff 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -4,6 +4,8 @@ using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Options; using Xunit; @@ -63,27 +65,36 @@ public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() } [Fact] - public async Task StartAsync_WhenAutoPurgeDisabled_DoesNotResolveClientOrLog() + public async Task StartAsync_WhenAutoPurgeDisabled_SignalsJobToStop() { - // Arrange - auto-purge is off. Even with a delete-capable blob store, the starter must no-op silently: - // it is registered unconditionally, so the not-opted-in path is the common case and must not log or - // resolve a client. + // Arrange - auto-purge is off. Returning silently was the whole disable bug: the job is a perpetual + // orchestrator owned by the task hub, so one created while the flag was on keeps deleting blobs no + // matter how many hosts start with it off. The disable path has to reach the backend. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); - Mock provider = new(); - TestLogger logger = new(); - BlobPurgeJobStarter starter = new( - provider.Object, - store, - OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), - "test", - logger); // Act - await starter.StartAsync(CancellationToken.None); + (EntityInstanceId Id, string Operation)? signal = await SignalFromDisabledStarterAsync(store); - // Assert - returned before resolving a client and without logging anything at all. - provider.Verify(p => p.GetClient(It.IsAny()), Times.Never); - logger.Logs.Should().BeEmpty(); + // Assert + signal.Should().NotBeNull("the disable path must tell the job to stop"); + signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndStoreCannotDelete_StillSignalsJobToStop() + { + // Arrange - the store-capability gate deliberately does not apply to the disable path. Stopping a job + // requires no ability to delete anything, and a user who has switched to a store that cannot delete is + // precisely the user whose still-running job must be stopped. + + // Act + (EntityInstanceId Id, string Operation)? signal = + await SignalFromDisabledStarterAsync(new NonDeletingPayloadStore()); + + // Assert + signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); + signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -139,6 +150,44 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag return monitor.Object; } + /// + /// Runs a starter with auto-purge disabled and returns the entity signal it emitted, or null if none was + /// emitted within the timeout. The signal runs on a background task, so it is awaited rather than assumed. + /// + static async Task<(EntityInstanceId Id, string Operation)?> SignalFromDisabledStarterAsync(PayloadStore store) + { + Mock entities = new("test"); + TaskCompletionSource<(EntityInstanceId Id, string Operation)> signalled = new(); + entities + .Setup(e => e.SignalEntityAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback( + (id, operation, _, _, _) => signalled.TrySetResult((id, operation))) + .Returns(Task.CompletedTask); + + Mock client = new("test"); + client.Setup(c => c.Entities).Returns(entities.Object); + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client.Object); + + BlobPurgeJobStarter starter = new( + provider.Object, + store, + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = false }), + "test", + new TestLogger()); + + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(signalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + return completed == signalled.Task ? await signalled.Task : null; + } + sealed class NonDeletingPayloadStore : PayloadStore { // DeleteAsync is intentionally NOT overridden: the base PayloadStore.DeleteAsync throws diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 2117299c..8a6ca306 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -3,6 +3,7 @@ using FluentAssertions; using Microsoft.DurableTask.AzureBlobPayloads; +using Microsoft.DurableTask.Entities; using Microsoft.DurableTask.Entities.Tests; using Xunit; @@ -31,12 +32,24 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() state.PurgeBatchSize.Should().Be(250); state.CreatedAt.Should().NotBeNull(); state.LastModifiedAt.Should().NotBeNull(); + + // Starting the job means signalling Run. Asserted here so that the Times.Never assertion in the + // already-active test below is meaningful rather than passing because the mock records nothing. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); } [Fact] - public async Task Create_WhenAlreadyActive_IsNoOp() + public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() { - // Arrange + // Arrange - the job is already running and the configured batch size has changed. Create is the only + // path by which a new batch size can reach an active job, so it must be taken even though the job is + // not restarted. BlobPurgeJobState existing = new() { Status = BlobPurgeJobStatus.Active, @@ -50,11 +63,83 @@ public async Task Create_WhenAlreadyActive_IsNoOp() // Act await this.job.RunAsync(operation); - // Assert - status stays Active and the original batch size is retained, proving the create no-op'd. + // Assert BlobPurgeJobState state = Assert.IsType( operation.State.GetState(typeof(BlobPurgeJobState))); state.Status.Should().Be(BlobPurgeJobStatus.Active); - state.PurgeBatchSize.Should().Be(100); + state.PurgeBatchSize.Should().Be(999); + + // Run is not re-signalled: the orchestrator is already up, and scheduling a second one over a live one + // would terminate and replace it mid-work. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Fact] + public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() + { + // Arrange - a running job. Stopping must not discard the configuration or the progress counters: they + // are wanted if the job is started again, and CreatedAt is what distinguishes a stopped job from one + // that was never started. + DateTimeOffset createdAt = DateTimeOffset.UtcNow.AddDays(-3); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + CreatedAt = createdAt, + PurgedCount = 17, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.CreatedAt.Should().Be(createdAt); + state.PurgedCount.Should().Be(17); + state.PurgeBatchSize.Should().Be(250); + state.LastModifiedAt.Should().NotBeNull(); + } + + [Fact] + public async Task Stop_WhenNotActive_LeavesStateUntouched() + { + // Arrange - a job that is already stopped. Every host with auto-purge disabled signals Stop on each + // start, so the repeat is the common case; rewriting LastModifiedAt each time would destroy its only + // useful meaning, which is when the job actually stopped. + DateTimeOffset stoppedAt = DateTimeOffset.UtcNow.AddHours(-6); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + CreatedAt = DateTimeOffset.UtcNow.AddDays(-3), + LastModifiedAt = stoppedAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Stop), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.Status.Should().Be(BlobPurgeJobStatus.Pending); + state.LastModifiedAt.Should().Be(stoppedAt); + state.PurgeBatchSize.Should().Be(250); } [Fact] From 7d409ab9d71ae557a3350e330e2e0d246f55aeb9 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 20:28:07 -0700 Subject: [PATCH 28/32] Read the purge job state before signalling Stop The disable path signalled Stop unconditionally. Entity state is persisted after every operation, so that signal created the job entity - in its default, never-started state - for every app that externalizes payloads without ever enabling auto-purge, and persisted a fresh write on every host start for an app whose job was already stopped. The starter now reads the job through the client first and signals only when a job exists and is Active. The read is a query against the instance store rather than an entity operation, so it never dispatches to the entity and cannot materialize one; that is the property that makes it different from the Get operation the orchestrator calls. Three things the pre-check must not break, and does not: - The Stop guard in the entity stays. The pre-check is read-then-signal across two round trips with nothing holding the state still, so it is racy by construction; the guard is what makes losing that race harmless. Its comment now states the race as the justification rather than the steady state, which the pre-check has just made false. - A read that fails falls through to signalling in the same iteration rather than being retried. Every 'entities are not supported' gate in this SDK lives on the DurableTaskClient.Entities property, not on individual methods, so a client that cannot answer the query cannot receive the signal either - retrying a permanent failure would spin forever and the running job would never be told to stop. - EventId 824 now fires only when a stop is actually requested. New EventIds 825 (no running job found) and 826 (state unreadable, stopping anyway) cover the paths where it would otherwise have asserted something that did not happen. The status is reached through EntityMetadata.IncludesState rather than by reading .State, which throws when the metadata carries none - reachable for an entity whose state was cleared but which still reports as existing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 76 ++++++++-- .../AutoPurge/Entity/BlobPurgeJob.cs | 19 ++- .../AzureBlobPayloads/AutoPurge/Logs.cs | 6 + .../AutoPurge/BlobPurgeJobStarterTests.cs | 141 +++++++++++++++--- 4 files changed, 207 insertions(+), 35 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index 477eac9b..db8cd248 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -3,6 +3,7 @@ using DurableTask.Core.Exceptions; using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -13,10 +14,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// /// Client-side hosted service that reconciles the per-task-hub singleton blob payload auto-purge job with the /// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at -/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, asks the job -/// to stop when it is disabled, and no-ops with an error log when the registered store cannot delete. It never -/// blocks host startup - the work runs on a background task that retries until the backend is reachable. The -/// job is a per-task-hub singleton, so racing client processes converge on the same result. +/// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, stops a +/// running job when it is disabled, and no-ops with an error log when the registered store cannot delete. It +/// never blocks host startup - the work runs on a background task that retries until the backend is reachable. +/// The job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable { @@ -68,16 +69,15 @@ public Task StartAsync(CancellationToken cancellationToken) { // Turning auto-purge off has to reach the backend to mean anything. The job is a perpetual // orchestrator owned by the task hub, not by this process, so a job created while the flag was on - // keeps deleting blobs forever no matter how many hosts start with it off. Ask it to stop. + // keeps deleting blobs forever no matter how many hosts start with it off. Stop a running one. // // The store gate below deliberately does NOT apply to this path. Stopping a job requires no ability // to delete anything, and a user who has switched to a store that cannot delete is precisely // someone who needs the job stopped; that gate's error is worded for the enabled case and would be // a false alarm here. // - // One deliberate cost: entity state is persisted after every operation, so this signal creates the - // job entity in its default, not-running state for apps that never enabled auto-purge. That is a - // single small entity per task hub, and it is the same one the job would use if it were enabled. + // The job's state is read before anything is signalled, so an app that never enabled auto-purge + // signals nothing and no entity is created for it. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.SignalJobStopAsync(this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -237,10 +237,68 @@ async Task SignalJobStopAsync(CancellationToken cancellationToken) // which must never happen for a feature the user has turned off. DurableTaskClient client = this.clientProvider.GetClient(this.builderName); + // Held in a local so the query and the signal below are issued against the same object. Both go + // through this one property, which is also where every "entities are not supported" gate in this + // SDK lives, so a client that cannot answer the query cannot receive the signal either. + DurableEntityClient entities = client.Entities; + + // Defaults to true so that every path which fails to establish the job's state falls through to + // signalling. Signalling is the correctness-critical outcome - a job left running keeps deleting + // payloads - while skipping it is only an optimization, so the optimization must never be the + // reason the stop does not happen. + bool stopNeeded = true; + + try + { + // A query against the instance store, not an entity operation: it never dispatches to the + // entity, so it cannot materialize one. That is the whole point of reading through the + // client here instead of relying on the entity's own idempotence. Signalling Stop + // unconditionally would create the job entity - with default, never-started state - for + // every app that externalizes payloads without auto-purge, and would persist a fresh write + // on every host start for an app whose job is already stopped. + // The three-argument overload is called deliberately. The shorter GetEntityAsync(id, + // cancellation) is a virtual forwarder that supplies includeState itself, and binding to a + // forwarder rather than to the abstract method is what made an earlier set of tests in this + // feature pass against a mock that was never wired up. Naming includeState also states the + // requirement directly: this call exists to read the status, so metadata alone is useless. + EntityMetadata? metadata = await entities + .GetEntityAsync(this.entityId, includeState: true, cancellationToken); + + // IncludesState is checked rather than reading metadata.State directly: State throws when + // the metadata carries none, and an entity whose state has been cleared still reports as + // existing until entity storage is cleaned. Such an entity has no running job, so it is + // treated the same as an absent one. + BlobPurgeJobState? state = metadata is { IncludesState: true } ? metadata.State : null; + + stopNeeded = state?.Status == BlobPurgeJobStatus.Active; + + if (!stopNeeded) + { + this.logger.BlobPurgeJobNotRunning(); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) + { + // Deliberately not rethrown into the retry loop. Retrying would be right for a transient + // failure but fatal for a permanent one: a client that never answers this query would keep + // the loop spinning and the running job would never be told to stop. Falling through costs + // at most the entity write this pre-check exists to avoid. + this.logger.BlobPurgeJobStateUnknown(ex); + } + + if (!stopNeeded) + { + return; + } + // Signalled rather than driven through the bridge orchestration: unlike Create, Stop needs no // fixed-instance dedupe. It is idempotent in the entity, so concurrent hosts converge on the // same state and a repeat costs nothing beyond the signal itself. - await client.Entities.SignalEntityAsync( + await entities.SignalEntityAsync( this.entityId, nameof(BlobPurgeJob.Stop), null, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 0c13984a..fb9303e2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -96,14 +96,19 @@ public void Stop(TaskEntityContext context) { if (this.State.Status != BlobPurgeJobStatus.Active) { - // Idempotent, and the repeat is the common case rather than the exception: a host with auto-purge - // disabled signals Stop on every start, including for an app that never enabled auto-purge at all. - // Returning here leaves the state exactly as it was found. + // Load-bearing, and NOT redundant with the client-side pre-check in BlobPurgeJobStarter that + // normally prevents this operation from being signalled at all. That pre-check reads the job state + // and then signals: two separate round trips with nothing holding the state still between them, so + // the job can stop - or never have existed - in the gap. This guard is what makes losing that race + // harmless, which is the only reason the pre-check is allowed to be a plain read. Deleting it + // because "the caller already checked" would reintroduce exactly the window it was written to + // absorb. Concurrent hosts signalling at once land here for the same reason. // - // This does not avoid materializing the entity - the framework persists entity state after every - // operation, so a stop signal to an entity that does not exist yet creates it with default state. - // What the guard preserves is LastModifiedAt: rewriting it on every host restart would destroy its - // only useful meaning, which is when the job actually stopped. + // Returning here also leaves the state exactly as it was found. That preserves LastModifiedAt, + // whose only useful meaning is when the job actually stopped; rewriting it on a redundant stop + // would destroy that. It does not avoid materializing the entity - the framework persists entity + // state after every operation, so a stop signal to an entity that does not exist yet creates it + // with default state. Not creating it is the pre-check's job, not this guard's. logger.BlobPurgeJobAlreadyStopped(context.Id.Key); return; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index a1f1eaf3..ce1ba658 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -54,4 +54,10 @@ static partial class Logs [LoggerMessage(EventId = 824, Level = LogLevel.Information, Message = "Blob payload auto-purge is disabled, so a stop was requested for the singleton job. A job left running by an earlier configuration exits after its current cycle.")] public static partial void BlobPurgeJobStopRequested(this ILogger logger); + + [LoggerMessage(EventId = 825, Level = LogLevel.Debug, Message = "Blob payload auto-purge is disabled and no running singleton job was found, so no stop was requested.")] + public static partial void BlobPurgeJobNotRunning(this ILogger logger); + + [LoggerMessage(EventId = 826, Level = LogLevel.Warning, Message = "Blob payload auto-purge is disabled but the singleton job's state could not be read; requesting a stop anyway. Stopping a job that may be deleting payloads matters more than avoiding a redundant stop request, which the job ignores.")] + public static partial void BlobPurgeJobStateUnknown(this ILogger logger, Exception exception); } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index 2e6a3fff..a6a3db91 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -65,20 +65,94 @@ public async Task StartAsync_WhenStoreIsBlobStore_DoesNotShortCircuit() } [Fact] - public async Task StartAsync_WhenAutoPurgeDisabled_SignalsJobToStop() + public async Task StartAsync_WhenAutoPurgeDisabledAndJobIsActive_SignalsJobToStop() { - // Arrange - auto-purge is off. Returning silently was the whole disable bug: the job is a perpetual - // orchestrator owned by the task hub, so one created while the flag was on keeps deleting blobs no - // matter how many hosts start with it off. The disable path has to reach the backend. + // Arrange - auto-purge is off while a job is running. Returning silently was the whole disable bug: the + // job is a perpetual orchestrator owned by the task hub, so one created while the flag was on keeps + // deleting blobs no matter how many hosts start with it off. The disable path has to reach the backend. BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); // Act - (EntityInstanceId Id, string Operation)? signal = await SignalFromDisabledStarterAsync(store); + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => MetadataFor(BlobPurgeJobStatus.Active)); // Assert - signal.Should().NotBeNull("the disable path must tell the job to stop"); - signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); - signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + run.Signal.Should().NotBeNull("the disable path must tell a running job to stop"); + run.Signal!.Value.Id.Should().Be(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + run.Signal.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndNoJobExists_DoesNotSignal() + { + // Arrange - the common case by far: an app that externalizes payloads and never enabled auto-purge. + // Signalling regardless would create the job entity, because the framework persists entity state after + // every operation, so an app that never used the feature would still carry one entity per task hub. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => null); + + // Assert - Reads is the positive control. A null signal is also what a mock that was never reached + // produces, so the assertion that nothing was signalled is only meaningful alongside proof that the + // code did run and did ask. The same wiring signals when the job is active, which the test above pins. + run.Reads.Should().Be(1, "the starter must consult the job state rather than skipping the path entirely"); + run.Signal.Should().BeNull("there is no job to stop, so nothing may be created"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndJobIsAlreadyStopped_DoesNotSignal() + { + // Arrange - a job that has already been stopped needs nothing. The entity would ignore the signal, but + // the framework still persists state on every operation, so signalling would write on every host start. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = + await RunDisabledStarterAsync(store, () => MetadataFor(BlobPurgeJobStatus.Pending)); + + // Assert + run.Reads.Should().Be(1); + run.Signal.Should().BeNull("a job that is not active must not be signalled on every restart"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndEntityHasNoState_DoesNotSignal() + { + // Arrange - an entity whose state has been cleared still reports as existing until entity storage is + // cleaned, and its metadata carries no state at all. Reading EntityMetadata.State in that condition + // throws, so the status has to be reached through IncludesState. Such an entity has no running job. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + store, + () => new EntityMetadata( + new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId))); + + // Assert + run.Reads.Should().Be(1); + run.Signal.Should().BeNull("an entity with no state has no running job"); + } + + [Fact] + public async Task StartAsync_WhenAutoPurgeDisabledAndStateCannotBeRead_SignalsJobToStopAnyway() + { + // Arrange - the pre-check is an optimization; stopping the job is the correctness-critical outcome. A + // client that cannot answer the query must not be able to prevent the stop, or the optimization's own + // failure mode would defeat the fix it is attached to. Retrying instead of falling through would be + // just as bad for a permanent failure: the loop would spin and the running job would never be told. + BlobPayloadStore store = new(new LargePayloadStorageOptions("UseDevelopmentStorage=true")); + + // Act + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + store, + () => throw new NotSupportedException("entity queries are not supported")); + + // Assert + run.Signal.Should().NotBeNull("an unreadable state must not suppress the stop"); + run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -89,12 +163,12 @@ public async Task StartAsync_WhenAutoPurgeDisabledAndStoreCannotDelete_StillSign // precisely the user whose still-running job must be stopped. // Act - (EntityInstanceId Id, string Operation)? signal = - await SignalFromDisabledStarterAsync(new NonDeletingPayloadStore()); + (( EntityInstanceId Id, string Operation)? Signal, int Reads) run = await RunDisabledStarterAsync( + new NonDeletingPayloadStore(), () => MetadataFor(BlobPurgeJobStatus.Active)); // Assert - signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); - signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); + run.Signal.Should().NotBeNull("a store that cannot delete must not block stopping the job"); + run.Signal!.Value.Operation.Should().Be(nameof(BlobPurgeJob.Stop)); } [Fact] @@ -151,13 +225,39 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag } /// - /// Runs a starter with auto-purge disabled and returns the entity signal it emitted, or null if none was - /// emitted within the timeout. The signal runs on a background task, so it is awaited rather than assumed. + /// Builds entity metadata carrying a job in the given status. + /// + static EntityMetadata MetadataFor(BlobPurgeJobStatus status) => + new( + new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId), + new BlobPurgeJobState { Status = status }); + + /// + /// Runs a starter with auto-purge disabled, using as the job state the client + /// reports (it may return null for an absent entity, or throw to simulate a client that cannot answer). + /// Returns the entity signal the starter emitted, if any, and how many times the state was read. /// - static async Task<(EntityInstanceId Id, string Operation)?> SignalFromDisabledStarterAsync(PayloadStore store) + static async Task<(( EntityInstanceId Id, string Operation)? Signal, int Reads)> RunDisabledStarterAsync( + PayloadStore store, Func?> read) { Mock entities = new("test"); - TaskCompletionSource<(EntityInstanceId Id, string Operation)> signalled = new(); + TaskCompletionSource readCalled = new(); + (EntityInstanceId Id, string Operation)? signal = null; + int reads = 0; + + // The three-argument overload is set up because that is the one the starter calls. The shorter + // GetEntityAsync(id, cancellation) is virtual, so a mock overrides it instead of forwarding, and a + // setup placed on the abstract method alone would silently never match. + entities + .Setup(e => e.GetEntityAsync( + It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + reads++; + readCalled.TrySetResult(true); + return Task.FromResult(read()); + }); + entities .Setup(e => e.SignalEntityAsync( It.IsAny(), @@ -166,7 +266,7 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag It.IsAny(), It.IsAny())) .Callback( - (id, operation, _, _, _) => signalled.TrySetResult((id, operation))) + (id, operation, _, _, _) => signal = (id, operation)) .Returns(Task.CompletedTask); Mock client = new("test"); @@ -181,11 +281,14 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag "test", new TestLogger()); + // The work runs on a background task, so wait for the read before shutting down. StopAsync then waits + // for that task to finish, which makes the observation deterministic: by the time it returns the + // starter has either signalled or decided not to, rather than being timed out mid-decision. await starter.StartAsync(CancellationToken.None); - Task completed = await Task.WhenAny(signalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await Task.WhenAny(readCalled.Task, Task.Delay(TimeSpan.FromSeconds(30))); await starter.StopAsync(CancellationToken.None); - return completed == signalled.Task ? await signalled.Task : null; + return (signal, reads); } sealed class NonDeletingPayloadStore : PayloadStore From 7e0d0ed5bc9385352f77941751c8d51b87f154f8 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:09:26 -0700 Subject: [PATCH 29/32] Self-heal the blob purge job by re-signalling Run on every host start Create's already-active branch previously did not re-signal Run, on the stated grounds that starting a second orchestrator over a live one is destructive. That premise is false. An entity-initiated start carries no reuse policy - neither TaskEntityShim.ScheduleNewOrchestration nor the StartNewOrchestrationAction proto message has a field for one - so the backend resolves it atomically: OkToPurge refuses to replace an instance that is not IsCompleted, and the start message is discarded; a completed, terminated, failed or canceled instance is purged and replaced. Healthy jobs are therefore undisturbed and dead ones are recovered, with no check-then-act on our side. Create now re-signals Run after updating the batch size, which makes the job recover from a dead orchestrator on the next host start. Also corrects two comments and one log message that asserted the old, false premise, including a longer restatement of it in BlobPurgeJobStarter that described manual recovery as the only option. Run had no test coverage; both its guarded and unguarded paths are now pinned, since the guard is what makes a Stop win against an in-flight Create. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 17 +-- .../AutoPurge/Entity/BlobPurgeJob.cs | 42 +++++-- .../AzureBlobPayloads/AutoPurge/Logs.cs | 2 +- .../AutoPurge/BlobPurgeJobTests.cs | 107 ++++++++++++++++-- 4 files changed, 140 insertions(+), 28 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index db8cd248..d43e47c5 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -172,14 +172,15 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // permanently; making it replaceable means the next host start re-runs Create, which finds the // entity not Active and rebuilds the job. // - // This deliberately does NOT recover the case where the perpetual orchestrator dies while the - // entity is still Active: the bridge re-runs, Create no-ops on the Active state, and the - // orchestrator stays down. That is a deliberate non-goal rather than an oversight. The only - // available restart is signalling the entity's Run, which schedules the orchestrator on the - // entity path with no dedupe policy available to it, so signalling a healthy orchestrator would - // terminate and replace it mid-work. Deciding it is safe to signal requires a check-then-act on - // the orchestrator's status, which is exactly the non-atomic pattern this call replaced. - // Recovery is therefore a supervised manual step. + // This also recovers the case where the perpetual orchestrator dies while the entity is still + // Active. Create leaves the entity's status alone in that case, but it re-signals Run, and the + // resulting start is resolved by the backend: discarded outright while the orchestrator is + // alive, and allowed to replace it once it has completed, terminated, failed or been canceled. + // So a healthy job is left untouched and a dead one is rebuilt, without this side ever having to + // ask which of the two it is looking at. + // + // Recovery is bounded by host starts rather than being continuous: nothing re-signals Run + // between them, so an orchestrator that dies mid-lifetime stays down until the next Create. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index fb9303e2..8c4dd893 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -13,10 +13,10 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; class BlobPurgeJob(ILogger logger) : TaskEntity { /// - /// Creates (or reactivates) the auto-purge job. Because the job is a per-task-hub singleton, this does not - /// restart a job that is already so that extra client processes - /// racing to create it do not disturb the running job. It does still take the batch size in that case, which - /// is what lets a configuration change reach a job that is already running. + /// Creates the auto-purge job, and starts its orchestrator if one is not already running. Because the job is + /// a per-task-hub singleton, client processes racing to create it converge on the same result rather than + /// disturbing a running job. It also takes the batch size when the job is already + /// , which is what lets a configuration change reach a running job. /// /// The entity context. /// @@ -26,15 +26,28 @@ public void Create(TaskEntityContext context, int purgeBatchSize) { if (this.State.Status == BlobPurgeJobStatus.Active) { - // Deliberately not re-signalling Run: the orchestrator is already up, and starting a second one - // over a live one is destructive. The batch size is still taken, because this is the only path by - // which a changed configuration reaches an active job - the orchestrator re-reads it from here - // every cycle. Without this the value written by the very first Create would be the only one the - // job ever used, and a batch size the backend rejects would wedge it permanently. + // The batch size is taken because this is the only path by which a changed configuration reaches an + // active job - the orchestrator re-reads it from here every cycle. Without this the value written by + // the very first Create would be the only one the job ever used, and a batch size the backend + // rejects would wedge it permanently. this.State.PurgeBatchSize = purgeBatchSize; this.State.LastModifiedAt = DateTimeOffset.UtcNow; logger.BlobPurgeJobAlreadyRunning(context.Id.Key); + + // Run is re-signalled even though the job is already active, and this is what makes the job + // self-heal: Create runs on every host start, so a job whose orchestrator has died is rebuilt at the + // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so + // the backend decides its fate: it discards the start while the target instance exists in any + // non-completed status, and purges and replaces it once the instance has completed, terminated, + // failed or been canceled (its OkToPurge / IsCompleted rule). A healthy orchestrator is therefore + // left strictly alone and only a dead one is replaced. + // + // Being blind is the point, not a shortcut. The backend reaches that decision atomically within one + // partition operation, so delegating it removes the race entirely. Checking the orchestrator's + // status here and signalling only when it looked dead would reintroduce the window in which it dies + // - or recovers - between the read and the signal, which is strictly worse than not asking. + context.SignalEntity(context.Id, nameof(this.Run)); return; } @@ -51,9 +64,16 @@ public void Create(TaskEntityContext context, int purgeBatchSize) } /// - /// Starts the purge orchestrator if the job is active. Uses a fixed orchestrator instance ID so only one - /// orchestrator ever runs for the singleton job. + /// Starts the purge orchestrator if the job is active. /// + /// + /// The orchestrator runs under a fixed instance ID, which is what keeps the singleton a singleton. No reuse + /// policy is passed, and none can be: the entity's start action has no field to carry one, so anything set + /// here would be dropped before it reached the wire. That default is the behaviour the job relies on rather + /// than an omission - the backend discards a start aimed at an instance that already exists in a + /// non-completed status, and replaces the instance only once it has completed, terminated, failed or been + /// canceled. Signalling this operation is therefore always safe, whatever the orchestrator is doing. + /// /// The entity context. public void Run(TaskEntityContext context) { diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index ce1ba658..b4042edd 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -13,7 +13,7 @@ static partial class Logs [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already running; ignoring the create request.")] + [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was updated from the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 8a6ca306..8b61237f 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -33,8 +33,9 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() state.CreatedAt.Should().NotBeNull(); state.LastModifiedAt.Should().NotBeNull(); - // Starting the job means signalling Run. Asserted here so that the Times.Never assertion in the - // already-active test below is meaningful rather than passing because the mock records nothing. + // Starting the job means signalling Run, and this path has always done so. Asserted explicitly because + // the already-active path now signals Run too, which makes this the case that would silently stop being + // covered if the two branches were ever collapsed. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), @@ -45,11 +46,10 @@ public async Task Create_WhenStopped_ActivatesJobAndStoresBatchSize() } [Fact] - public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() + public async Task Create_WhenAlreadyActive_UpdatesBatchSizeAndReSignalsRun() { // Arrange - the job is already running and the configured batch size has changed. Create is the only - // path by which a new batch size can reach an active job, so it must be taken even though the job is - // not restarted. + // path by which a new batch size can reach an active job, so it must be taken. BlobPurgeJobState existing = new() { Status = BlobPurgeJobStatus.Active, @@ -69,17 +69,108 @@ public async Task Create_WhenAlreadyActive_UpdatesBatchSizeWithoutRestarting() state.Status.Should().Be(BlobPurgeJobStatus.Active); state.PurgeBatchSize.Should().Be(999); - // Run is not re-signalled: the orchestrator is already up, and scheduling a second one over a live one - // would terminate and replace it mid-work. + // Run is re-signalled even though the job is already active. That is what lets a job whose orchestrator + // has died be rebuilt at the next host start, and it is safe because the backend discards the resulting + // start while the orchestrator is alive rather than replacing it. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), - It.IsAny(), + nameof(BlobPurgeJob.Run), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Fact] + public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() + { + // Arrange - pins that re-signalling Run is the only signal the already-active path emits. Verifying the + // Run signal alone would still pass if a second, different signal were added beside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 100, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 999); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control for this Times.Never: both target the same + // four-argument overload on the same mock, so this cannot be passing because nothing was recorded. + Mock.Get(operation.Context).Verify( + c => c.SignalEntity( + It.IsAny(), + It.Is(name => name != nameof(BlobPurgeJob.Run)), It.IsAny(), It.IsAny()), Times.Never); } + [Fact] + public async Task Run_WhenActive_SchedulesOrchestratorAtTheFixedInstanceId() + { + // Arrange - the fixed instance ID is the mechanism the whole restart story rests on. It is what lets the + // backend recognize a start as targeting the existing orchestrator, and therefore discard it while that + // orchestrator is alive instead of running a second one alongside it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + Mock.Get(operation.Context) + .Setup(c => c.Id) + .Returns(new EntityInstanceId(nameof(BlobPurgeJob), BlobPurgeConstants.JobId)); + + // Act + await this.job.RunAsync(operation); + + // Assert + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.Is(o => + o.InstanceId == BlobPurgeConstants.GetOrchestratorInstanceId(BlobPurgeConstants.JobId))), + Times.Once); + } + + [Fact] + public async Task Run_WhenNotActive_SchedulesNothing() + { + // Arrange - a Run signal arriving after the job was stopped. Create signals Run rather than starting the + // orchestrator itself, so a Stop landing between the two leaves this signal in flight against a job that + // must no longer purge. This guard is what makes the stop win instead of the stale signal restarting it. + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Pending, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - the Times.Once above is the positive control: same mocked type, same three-argument overload, + // so this cannot be passing merely because the mock records nothing. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + [Fact] public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() { From 720e028b03b57df48f7a410b4bcfa8091f7761fa Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:32:41 -0700 Subject: [PATCH 30/32] Make LastModifiedAt mean a real change, and drop internal backend symbol names LastModifiedAt was written by five operations, read by none, and two of those writes fired on every host start - Create's already-active branch and Run, the latter on every start since Run began being re-signalled. The field had degenerated into "the most recent host start, or the most recent non-empty purge, whichever is later", which also made Stop's idempotence guard vacuous: it deliberately preserves the field, but the next host start overwrote it anyway. Run no longer writes it - it schedules an orchestrator and changes nothing about the job - and Create's already-active branch writes it only when the batch size actually differs. An entity from a build predating the field carries zero, which differs from any configured size, so the first Create after an upgrade still repairs it. The field now means what its documentation says: the last time the job was started, stopped, reconfigured, or recorded a non-zero purge. Entity state is persisted after every operation regardless, so nothing about persistence changes. Also removes the names of two private backend methods from a comment. This repository is public and cannot reference them, so they were both a leak and certain to rot; the observable behaviour they described is kept. A sweep of the rest of the diff found no other instance outside the proto, whose comments are copied verbatim from the canonical contract. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Entity/BlobPurgeJob.cs | 46 ++++++--- .../AutoPurge/Models/BlobPurgeJobState.cs | 9 +- .../AutoPurge/BlobPurgeJobTests.cs | 98 ++++++++++++++++++- 3 files changed, 134 insertions(+), 19 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index 8c4dd893..ce2199f7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -30,8 +30,17 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // active job - the orchestrator re-reads it from here every cycle. Without this the value written by // the very first Create would be the only one the job ever used, and a batch size the backend // rejects would wedge it permanently. - this.State.PurgeBatchSize = purgeBatchSize; - this.State.LastModifiedAt = DateTimeOffset.UtcNow; + // + // Written only when the value actually differs, which is what keeps LastModifiedAt tracking real + // configuration changes. Create runs on every host start, so an unconditional write would reduce + // the field to "time of the last host start". An entity written by a build that predates this + // field carries zero, which differs from any configured size, so the first Create after an upgrade + // still repairs it. + if (this.State.PurgeBatchSize != purgeBatchSize) + { + this.State.PurgeBatchSize = purgeBatchSize; + this.State.LastModifiedAt = DateTimeOffset.UtcNow; + } logger.BlobPurgeJobAlreadyRunning(context.Id.Key); @@ -40,13 +49,13 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so // the backend decides its fate: it discards the start while the target instance exists in any // non-completed status, and purges and replaces it once the instance has completed, terminated, - // failed or been canceled (its OkToPurge / IsCompleted rule). A healthy orchestrator is therefore - // left strictly alone and only a dead one is replaced. + // failed or been canceled. A healthy orchestrator is therefore left strictly alone and only a dead + // one is replaced. // - // Being blind is the point, not a shortcut. The backend reaches that decision atomically within one - // partition operation, so delegating it removes the race entirely. Checking the orchestrator's - // status here and signalling only when it looked dead would reintroduce the window in which it dies - // - or recovers - between the read and the signal, which is strictly worse than not asking. + // Being blind is the point, not a shortcut. The backend reaches that decision atomically, so + // delegating it removes the race entirely. Checking the orchestrator's status here and signalling + // only when it looked dead would reintroduce the window in which it dies - or recovers - between + // the read and the signal, which is strictly worse than not asking. context.SignalEntity(context.Id, nameof(this.Run)); return; } @@ -67,12 +76,19 @@ public void Create(TaskEntityContext context, int purgeBatchSize) /// Starts the purge orchestrator if the job is active. /// /// + /// /// The orchestrator runs under a fixed instance ID, which is what keeps the singleton a singleton. No reuse /// policy is passed, and none can be: the entity's start action has no field to carry one, so anything set /// here would be dropped before it reached the wire. That default is the behaviour the job relies on rather /// than an omission - the backend discards a start aimed at an instance that already exists in a /// non-completed status, and replaces the instance only once it has completed, terminated, failed or been /// canceled. Signalling this operation is therefore always safe, whatever the orchestrator is doing. + /// + /// + /// This operation deliberately writes no state. It schedules an orchestrator and nothing more, and it runs + /// on every host start, so touching here would overwrite a + /// real change with the time of a start that changed nothing. + /// /// /// The entity context. public void Run(TaskEntityContext context) @@ -89,8 +105,6 @@ public void Run(TaskEntityContext context) new TaskName(nameof(BlobPurgeJobOrchestrator)), new BlobPurgeJobRunRequest(context.Id, this.State.PurgeBatchSize), startOrchestrationOptions); - - this.State.LastModifiedAt = DateTimeOffset.UtcNow; } /// @@ -124,11 +138,13 @@ public void Stop(TaskEntityContext context) // because "the caller already checked" would reintroduce exactly the window it was written to // absorb. Concurrent hosts signalling at once land here for the same reason. // - // Returning here also leaves the state exactly as it was found. That preserves LastModifiedAt, - // whose only useful meaning is when the job actually stopped; rewriting it on a redundant stop - // would destroy that. It does not avoid materializing the entity - the framework persists entity - // state after every operation, so a stop signal to an entity that does not exist yet creates it - // with default state. Not creating it is the pre-check's job, not this guard's. + // Returning here also leaves the state exactly as it was found, which is what keeps + // LastModifiedAt meaning "when this job stopped" rather than "when a stop was last signalled at + // it". That only holds because no other operation rewrites the field on a no-op either: Run never + // writes it, and Create rewrites it only when the batch size actually differs. Breaking either of + // those breaks this too. It does not avoid materializing the entity - the framework persists + // entity state after every operation, so a stop signal to an entity that does not exist yet + // creates it with default state. Not creating it is the pre-check's job, not this guard's. logger.BlobPurgeJobAlreadyStopped(context.Id.Key); return; } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs index 8bd4cdd5..0eee8dd2 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -19,8 +19,15 @@ public sealed class BlobPurgeJobState public DateTimeOffset? CreatedAt { get; set; } /// - /// Gets or sets the time when the job state was last modified. + /// Gets or sets the time of the last meaningful change to the job: when it was started, when it was + /// stopped, when it was given a different from the one it already had, or + /// when it last recorded a non-zero number of purged blobs. /// + /// + /// This is not a liveness or heartbeat signal, and it must not be read as one. Starting a host does not + /// move it, and an active job whose cycles keep finding nothing to purge leaves it untouched indefinitely, + /// so a value far in the past is equally consistent with a healthy idle job and a dead one. + /// public DateTimeOffset? LastModifiedAt { get; set; } /// diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 8b61237f..37449725 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -110,6 +110,97 @@ public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() Times.Never); } + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeUnchanged_DoesNotMoveLastModifiedAt() + { + // Arrange - the steady state. Create runs on every host start, and almost every one of those carries + // the same configured batch size the job already has. If that rewrote LastModifiedAt, the field would + // degrade to "time of the last host start" and say nothing about the job. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 250); + + // Act + await this.job.RunAsync(operation); + + // Assert - the sibling test below is the positive control: identical wiring, differing only in the + // batch size passed in, and it proves this same path does move the field when something changes. + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + state.PurgeBatchSize.Should().Be(250); + } + + [Fact] + public async Task Create_WhenAlreadyActive_AndBatchSizeChanged_MovesLastModifiedAt() + { + // Arrange - a real configuration change reaching an active job, which is the one thing this path + // exists to deliver and the one case that must be recorded. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Create), + new TestEntityState(existing), + 500); + + // Act + await this.job.RunAsync(operation); + + // Assert + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.PurgeBatchSize.Should().Be(500); + state.LastModifiedAt.Should().BeAfter(configuredAt); + } + + [Fact] + public async Task Run_DoesNotMoveLastModifiedAt() + { + // Arrange - Run schedules an orchestrator and changes nothing about the job. It is signalled by every + // Create, so writing here would move the field on every host start and undo the conditional write + // above. + DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); + BlobPurgeJobState existing = new() + { + Status = BlobPurgeJobStatus.Active, + LastModifiedAt = configuredAt, + PurgeBatchSize = 250, + }; + TestEntityOperation operation = new( + nameof(BlobPurgeJob.Run), + new TestEntityState(existing), + null); + + // Act + await this.job.RunAsync(operation); + + // Assert - scheduling is asserted first as the positive control. Without it a misdispatched operation + // would write nothing and pass this test for entirely the wrong reason. + Mock.Get(operation.Context).Verify( + c => c.ScheduleNewOrchestration( + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Once); + + BlobPurgeJobState state = Assert.IsType( + operation.State.GetState(typeof(BlobPurgeJobState))); + state.LastModifiedAt.Should().Be(configuredAt); + } + [Fact] public async Task Run_WhenActive_SchedulesOrchestratorAtTheFixedInstanceId() { @@ -206,9 +297,10 @@ public async Task Stop_WhenActive_MovesToPendingAndKeepsHistory() [Fact] public async Task Stop_WhenNotActive_LeavesStateUntouched() { - // Arrange - a job that is already stopped. Every host with auto-purge disabled signals Stop on each - // start, so the repeat is the common case; rewriting LastModifiedAt each time would destroy its only - // useful meaning, which is when the job actually stopped. + // Arrange - a job that is already stopped. The starter's client-side pre-check normally suppresses a + // redundant stop, so this is the signal that races past it: the job stopped between that read and this + // signal landing. Rewriting LastModifiedAt here would report the losing side of that race as if it + // were the moment the job stopped. DateTimeOffset stoppedAt = DateTimeOffset.UtcNow.AddHours(-6); BlobPurgeJobState existing = new() { From 2b389fa41683aa510cfc954d53d02d5cace06c8c Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 21:51:24 -0700 Subject: [PATCH 31/32] Drop an uncorrelatable backend column name from a proto comment The comment on LargePayloadPurgeResult.disposition told the reader to correlate a row with worker telemetry by (partitionId, instanceKey, payloadId) plus LastAttemptAt. The first three are fields on both messages. The fourth is a column on a backend table that is never sent to the worker - LargePayloadTombstone carries partitionId, instanceKey, payloadId, token and revision, and nothing else - so the advice named a value its audience can neither receive nor query, while also naming backend internals in a public repository. The remaining three identifiers are a complete correlation key for the telemetry this comment is about. The wire contract is unaffected: only a comment changed, and the serialized file descriptor embedded in the generated code is unchanged. The generated C# does move, because protoc copies proto comments into XML documentation, and that one documentation line now matches. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- src/Grpc/orchestrator_service.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 51f490a2..080b697d 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -893,7 +893,7 @@ message LargePayloadPurgeResult { // The only field the backend acts on. Deliberately the only outcome field on this message: // anything finer would be write-only. Failure detail stays in the worker's own telemetry, which // holds the full exception rather than a lossy classification, and a row is correlated to it by - // (partitionId, instanceKey, payloadId) plus the ledger's LastAttemptAt. + // (partitionId, instanceKey, payloadId). LargePayloadPurgeDisposition disposition = 5; } From 3dd2b14cf26b29cbf7ebb6170358470b4d8502e0 Mon Sep 17 00:00:00 2001 From: wangbill Date: Wed, 12 Aug 2026 22:23:40 -0700 Subject: [PATCH 32/32] Reconcile the blob auto-purge job periodically instead of once per host start The ensure loop returned as soon as one schedule succeeded, so it was a retry-until-success loop rather than a periodic one. That call is the only thing that re-signals the job's Run, so an orchestrator that died in the middle of a host's lifetime stayed down until the process was restarted. Every path now picks a delay and the loop waits once at its tail: the two success paths take a five minute reconcile interval, and the failure path keeps the short retry delay, because a backend that is unreachable at host start has to be retried quickly - until one pass succeeds the job may not exist at all. Nothing else changes. A pass that finds everything healthy is already a no-op at both hops it drives: the bridge's Create no-ops while the entity is Active, and the Run it signals is discarded by the backend while the orchestrator is alive. Two logs on the repeating path drop from Information to Debug, since they now fire once per host per interval forever. EventId 811 also said the batch size "was updated", which stopped being true when that write became conditional; it now says it was reconciled, which is true whether or not it changed. The comments that described recovery as bounded by host starts are rewritten, in the starter, in the entity and in the tests. The interval carries the cost it trades: this is per host, not per task hub, because staggered hosts each find the previous bridge already Completed and therefore replaceable, so a fleet of N pays roughly N bridge runs and N entity calls per interval. What keeps the purge work itself single is the fixed orchestrator instance id, not the bridge's dedupe policy. Tests cover that the loop repeats, and that shutdown cancels a pass parked in the interval rather than sitting it out - the latter being the one real regression risk, and one the other tests could only show as a hang. Both were confirmed to fail when the behaviour they name is removed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: b69ecb19-b596-4e46-bb44-12ce571ec31f --- .../AutoPurge/Client/BlobPurgeJobStarter.cs | 79 ++++++++++++---- .../AutoPurge/Entity/BlobPurgeJob.cs | 17 ++-- .../AzureBlobPayloads/AutoPurge/Logs.cs | 4 +- .../AutoPurge/Models/BlobPurgeJobState.cs | 7 +- .../AutoPurge/BlobPurgeJobStarterTests.cs | 93 ++++++++++++++++++- .../AutoPurge/BlobPurgeJobTests.cs | 14 +-- 6 files changed, 174 insertions(+), 40 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs index d43e47c5..af197a99 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Client/BlobPurgeJobStarter.cs @@ -16,7 +16,8 @@ namespace Microsoft.DurableTask.AzureBlobPayloads; /// resolved options. It is registered unconditionally by UseExternalizedPayloads and decides what to do at /// startup, once options are fully resolved: it ensures the job exists when auto-purge is enabled, stops a /// running job when it is disabled, and no-ops with an error log when the registered store cannot delete. It -/// never blocks host startup - the work runs on a background task that retries until the backend is reachable. +/// never blocks host startup - the work runs on a background task that retries until the backend is reachable +/// and then, on the enabled path, keeps reconciling on a fixed interval for the lifetime of the process. /// The job is a per-task-hub singleton, so racing client processes converge on the same result. /// sealed class BlobPurgeJobStarter : IHostedService, IDisposable @@ -55,6 +56,33 @@ public BlobPurgeJobStarter( this.logger = Check.NotNull(logger); } + /// + /// Gets or sets the interval between reconciliation passes on the enabled path. Settable only so tests can + /// drive the loop without waiting on wall-clock time; nothing in the product ever assigns it. + /// + /// + /// + /// This value is the worst-case time a dead job stays dead. Nothing else re-signals the job's orchestrator, + /// so recovery cannot be faster than one pass. What it buys that back with is one schedule call per host + /// per interval, forever - so it trades a bounded recovery time against a standing, fleet-sized cost, and + /// shortening it makes that cost grow in proportion. + /// + /// + /// That cost is per host, not per task hub. Hosts start at different times, so their intervals are + /// staggered, and a pass almost always finds the previous bridge already Completed - which this call leaves + /// replaceable on purpose - so it schedules and runs a bridge of its own. The + /// path only absorbs passes that overlap a bridge which is + /// still Pending or Running, and that window is narrow because a bridge makes one entity call and exits. So + /// a fleet of N hosts costs roughly N bridge runs and N entity calls per interval, not one. + /// + /// + /// What keeps the actual purge work single is not that dedupe policy but the fixed orchestrator instance id: + /// every bridge run signals Run, and the backend discards a start aimed at an orchestrator that is still + /// alive. One perpetual orchestrator therefore serves the whole fleet however many bridges ran. + /// + /// + internal TimeSpan ReconcileInterval { get; set; } = TimeSpan.FromMinutes(5); + /// public Task StartAsync(CancellationToken cancellationToken) { @@ -102,7 +130,7 @@ public Task StartAsync(CancellationToken cancellationToken) int batchSize = opts.PayloadPurgeBatchSize; // Do not block host startup; ensure the job on a background task with basic retry until the backend - // is reachable. + // is reachable, and keep reconciling it from there. this.cts = new CancellationTokenSource(); this.backgroundTask = Task.Run(() => this.EnsureJobAsync(client, batchSize, this.cts.Token), CancellationToken.None); return Task.CompletedTask; @@ -141,6 +169,8 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT { while (!cancellationToken.IsCancellationRequested) { + TimeSpan delay; + try { BlobPurgeJobOperationRequest request = new( @@ -163,13 +193,13 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // // Re-running a bridge that already finished is safe and close to free: the bridge's only effect // is calling Create, which no-ops while the entity is Active, so the cost is one instance - // replacement plus one entity call per host start. + // replacement plus one entity call per reconciliation pass. // // Re-running is also what lets the job self-heal after the entity is removed, for example by // CleanEntityStorageAsync. The perpetual orchestrator exits cleanly when it reads back a null // entity state, and a removed entity is back to its default Pending status, so the job is left // dead with a Completed bridge behind it. Keeping Completed deduped would keep it dead - // permanently; making it replaceable means the next host start re-runs Create, which finds the + // permanently; making it replaceable means the next pass re-runs Create, which finds the // entity not Active and rebuilds the job. // // This also recovers the case where the perpetual orchestrator dies while the entity is still @@ -179,8 +209,13 @@ async Task EnsureJobAsync(DurableTaskClient client, int batchSize, CancellationT // So a healthy job is left untouched and a dead one is rebuilt, without this side ever having to // ask which of the two it is looking at. // - // Recovery is bounded by host starts rather than being continuous: nothing re-signals Run - // between them, so an orchestrator that dies mid-lifetime stays down until the next Create. + // Recovery repeats on a fixed interval rather than being bounded by host starts. This loop runs + // for the lifetime of the process and re-issues the same call every ReconcileInterval, so an + // orchestrator that dies mid-lifetime is rebuilt within roughly one interval instead of staying + // down until the next deployment. Repeating it costs nothing extra to reason about, because a + // pass that finds everything healthy is already a no-op at both hops: the bridge's Create + // no-ops while the entity is Active, and the Run it signals is discarded by the backend while + // the orchestrator is alive. await client.ScheduleNewOrchestrationInstanceAsync( new TaskName(nameof(ExecuteBlobPurgeJobOperationOrchestrator)), request, @@ -191,7 +226,7 @@ await client.ScheduleNewOrchestrationInstanceAsync( cancellationToken); this.logger.BlobPurgeJobEnsured(); - return; + delay = this.ReconcileInterval; } catch (OrchestrationAlreadyExistsException) { @@ -199,12 +234,12 @@ await client.ScheduleNewOrchestrationInstanceAsync( // policy it means another host scheduled the bridge and it is still Pending or Running. That is // exactly the concurrent-start race this replaced a status check to close: one create wins and // the loser lands here. Either way the singleton is already being set up, so treat it as - // ensured and stop. + // ensured and wait for the next pass. // // Note this is NOT the steady-state path. A bridge that already finished is Completed, which is - // replaceable, so a later host start re-runs it rather than landing here. + // replaceable, so a later pass re-runs it rather than landing here. this.logger.BlobPurgeJobEnsured(); - return; + delay = this.ReconcileInterval; } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -212,15 +247,23 @@ await client.ScheduleNewOrchestrationInstanceAsync( } catch (Exception ex) when (ex is not OutOfMemoryException and not StackOverflowException) { + // Deliberately the short delay and not the reconcile interval. A backend that is unreachable + // when the host starts has to be retried quickly, because until one pass succeeds the job may + // not exist at all; the long interval is only the price of keeping a job that already exists + // healthy. this.logger.BlobPurgeStarterRetry(ex); - try - { - await Task.Delay(RetryDelay, cancellationToken); - } - catch (OperationCanceledException) - { - return; - } + delay = RetryDelay; + } + + try + { + await Task.Delay(delay, cancellationToken); + } + catch (OperationCanceledException) + { + // Shutdown. This is the only thing that ends the loop on the success path, which is why + // StopAsync can cancel a pass that is parked here instead of waiting out the interval. + return; } } } diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs index ce2199f7..f1b704e7 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Entity/BlobPurgeJob.cs @@ -32,10 +32,10 @@ public void Create(TaskEntityContext context, int purgeBatchSize) // rejects would wedge it permanently. // // Written only when the value actually differs, which is what keeps LastModifiedAt tracking real - // configuration changes. Create runs on every host start, so an unconditional write would reduce - // the field to "time of the last host start". An entity written by a build that predates this - // field carries zero, which differs from any configured size, so the first Create after an upgrade - // still repairs it. + // configuration changes. Create runs on every reconciliation pass, not only at host start, so an + // unconditional write would reduce the field to "time of the last pass". An entity written by a + // build that predates this field carries zero, which differs from any configured size, so the + // first Create after an upgrade still repairs it. if (this.State.PurgeBatchSize != purgeBatchSize) { this.State.PurgeBatchSize = purgeBatchSize; @@ -45,8 +45,9 @@ public void Create(TaskEntityContext context, int purgeBatchSize) logger.BlobPurgeJobAlreadyRunning(context.Id.Key); // Run is re-signalled even though the job is already active, and this is what makes the job - // self-heal: Create runs on every host start, so a job whose orchestrator has died is rebuilt at the - // next one. The signal is deliberately blind. An entity-initiated start carries no reuse policy, so + // self-heal: Create runs on every reconciliation pass, so a job whose orchestrator has died is + // rebuilt within roughly one interval rather than waiting for the next host start. The signal is + // deliberately blind. An entity-initiated start carries no reuse policy, so // the backend decides its fate: it discards the start while the target instance exists in any // non-completed status, and purges and replaces it once the instance has completed, terminated, // failed or been canceled. A healthy orchestrator is therefore left strictly alone and only a dead @@ -86,8 +87,8 @@ public void Create(TaskEntityContext context, int purgeBatchSize) /// /// /// This operation deliberately writes no state. It schedules an orchestrator and nothing more, and it runs - /// on every host start, so touching here would overwrite a - /// real change with the time of a start that changed nothing. + /// on every reconciliation pass, so touching here would + /// overwrite a real change with the time of a pass that changed nothing. /// /// /// The entity context. diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs index b4042edd..3f3ae217 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Logs.cs @@ -13,7 +13,7 @@ static partial class Logs [LoggerMessage(EventId = 810, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' created.")] public static partial void BlobPurgeJobCreated(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 811, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was updated from the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] + [LoggerMessage(EventId = 811, Level = LogLevel.Debug, Message = "Blob payload auto-purge job '{jobId}' is already active. Its batch size was reconciled against the current configuration, and its orchestrator was re-signalled, which starts one only if none is running.")] public static partial void BlobPurgeJobAlreadyRunning(this ILogger logger, string? jobId); [LoggerMessage(EventId = 812, Level = LogLevel.Information, Message = "Blob payload auto-purge orchestrator for job '{jobId}' stopping; job status is {status}.")] @@ -31,7 +31,7 @@ static partial class Logs [LoggerMessage(EventId = 816, Level = LogLevel.Information, Message = "Blob payload auto-purge job '{jobId}' stopped. The perpetual orchestrator is not terminated; it reads the job state at the start of its next cycle and exits on its own.")] public static partial void BlobPurgeJobStopped(this ILogger logger, string? jobId); - [LoggerMessage(EventId = 817, Level = LogLevel.Information, Message = "Blob payload auto-purge singleton job ensured.")] + [LoggerMessage(EventId = 817, Level = LogLevel.Debug, Message = "Blob payload auto-purge singleton job ensured.")] public static partial void BlobPurgeJobEnsured(this ILogger logger); [LoggerMessage(EventId = 818, Level = LogLevel.Warning, Message = "Blob payload auto-purge starter could not reach the singleton job; retrying.")] diff --git a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs index 0eee8dd2..0a17caae 100644 --- a/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs +++ b/src/Extensions/AzureBlobPayloads/AutoPurge/Models/BlobPurgeJobState.cs @@ -24,9 +24,10 @@ public sealed class BlobPurgeJobState /// when it last recorded a non-zero number of purged blobs. /// /// - /// This is not a liveness or heartbeat signal, and it must not be read as one. Starting a host does not - /// move it, and an active job whose cycles keep finding nothing to purge leaves it untouched indefinitely, - /// so a value far in the past is equally consistent with a healthy idle job and a dead one. + /// This is not a liveness or heartbeat signal, and it must not be read as one. Neither starting a host nor + /// a reconciliation pass moves it, and an active job whose cycles keep finding nothing to purge leaves it + /// untouched indefinitely, so a value far in the past is equally consistent with a healthy idle job and a + /// dead one. /// public DateTimeOffset? LastModifiedAt { get; set; } diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs index a6a3db91..83d51fb2 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobStarterTests.cs @@ -212,11 +212,82 @@ public async Task EnsureJob_SchedulesBridge_DedupingOnlyPendingAndRunning() options!.InstanceId.Should().Be(BlobPurgeConstants.StarterInstanceId); // Every status other than Pending and Running is replaceable, so a finished bridge is re-run on the - // next host start. That is what lets the job rebuild itself after the entity is removed. The set is - // asserted exactly, never as a superset, because the hazard is a silent omission. + // next reconciliation pass. That is what lets the job rebuild itself after the entity is removed. The + // set is asserted exactly, never as a superset, because the hazard is a silent omission. options.DedupeStatuses.Should().BeEquivalentTo(["Pending", "Running"]); } + [Fact] + public async Task EnsureJob_AfterSuccessfulSchedule_KeepsReconciling() + { + // Arrange - the loop used to return as soon as one schedule succeeded, which left recovery bounded by + // host starts: this call is the only thing that re-signals the job's Run, so an orchestrator that died + // mid-lifetime stayed down until the process was restarted. It has to keep re-issuing the call instead. + Mock client = new("test"); + TaskCompletionSource secondSchedule = new(); + int schedules = 0; + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => + { + if (Interlocked.Increment(ref schedules) >= 2) + { + secondSchedule.TrySetResult(true); + } + }) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + // A short interval rather than a fake clock. The test still gates on the second call happening, never + // on time having passed, so it neither sleeps nor assumes anything about how long a pass takes. + BlobPurgeJobStarter starter = EnabledStarterFor(client.Object); + starter.ReconcileInterval = TimeSpan.FromMilliseconds(10); + + // Act + await starter.StartAsync(CancellationToken.None); + Task completed = await Task.WhenAny(secondSchedule.Task, Task.Delay(TimeSpan.FromSeconds(30))); + await starter.StopAsync(CancellationToken.None); + + // Assert - a loop that returned after its first success leaves this waiting until the timeout fires. + completed.Should().BeSameAs( + secondSchedule.Task, "reconciliation must repeat rather than ending after the first success"); + } + + [Fact] + public async Task StopAsync_WhileWaitingForNextReconcile_ReturnsPromptly() + { + // Arrange - the default interval, so the loop parks in a five-minute wait once its first pass succeeds. + // StopAsync awaits the background task, so a wait that did not observe cancellation would hold host + // shutdown for the rest of the interval. That is the one real regression risk in making the loop + // periodic, and it is invisible in the other tests because they would simply hang. + Mock client = new("test"); + TaskCompletionSource scheduled = new(); + client + .Setup(c => c.ScheduleNewOrchestrationInstanceAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Callback(() => scheduled.TrySetResult(true)) + .ReturnsAsync(BlobPurgeConstants.StarterInstanceId); + + BlobPurgeJobStarter starter = EnabledStarterFor(client.Object); + + // Act - waiting for the first schedule is the positive control. Without it a prompt StopAsync would + // also be what a starter that never reached the loop at all produces. + await starter.StartAsync(CancellationToken.None); + Task firstPass = await Task.WhenAny(scheduled.Task, Task.Delay(TimeSpan.FromSeconds(30))); + Task stop = starter.StopAsync(CancellationToken.None); + Task stopped = await Task.WhenAny(stop, Task.Delay(TimeSpan.FromSeconds(30))); + + // Assert + firstPass.Should().BeSameAs(scheduled.Task, "the loop must have reached the wait being cancelled here"); + stopped.Should().BeSameAs(stop, "shutdown must cancel the wait rather than sit out the interval"); + } + static IOptionsMonitor OptionsFor(LargePayloadStorageOptions options) { Mock> monitor = new(); @@ -224,6 +295,24 @@ static IOptionsMonitor OptionsFor(LargePayloadStorag return monitor.Object; } + /// + /// Builds a starter with auto-purge enabled over the given client. The real blob store is used because the + /// store-capability gate refuses to start the job for any other kind, so a stub would never reach the + /// ensure loop. UseDevelopmentStorage=true constructs it offline, with no network I/O. + /// + static BlobPurgeJobStarter EnabledStarterFor(DurableTaskClient client) + { + Mock provider = new(); + provider.Setup(p => p.GetClient(It.IsAny())).Returns(client); + + return new BlobPurgeJobStarter( + provider.Object, + new BlobPayloadStore(new LargePayloadStorageOptions("UseDevelopmentStorage=true")), + OptionsFor(new LargePayloadStorageOptions("UseDevelopmentStorage=true") { AutoPurge = true }), + "test", + new TestLogger()); + } + /// /// Builds entity metadata carrying a job in the given status. /// diff --git a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs index 37449725..88d4fa6a 100644 --- a/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs +++ b/test/Extensions/AzureBlobPayloads.Tests/AutoPurge/BlobPurgeJobTests.cs @@ -70,8 +70,8 @@ public async Task Create_WhenAlreadyActive_UpdatesBatchSizeAndReSignalsRun() state.PurgeBatchSize.Should().Be(999); // Run is re-signalled even though the job is already active. That is what lets a job whose orchestrator - // has died be rebuilt at the next host start, and it is safe because the backend discards the resulting - // start while the orchestrator is alive rather than replacing it. + // has died be rebuilt at the next reconciliation pass, and it is safe because the backend discards the + // resulting start while the orchestrator is alive rather than replacing it. Mock.Get(operation.Context).Verify( c => c.SignalEntity( It.IsAny(), @@ -113,9 +113,9 @@ public async Task Create_WhenAlreadyActive_SignalsNothingOtherThanRun() [Fact] public async Task Create_WhenAlreadyActive_AndBatchSizeUnchanged_DoesNotMoveLastModifiedAt() { - // Arrange - the steady state. Create runs on every host start, and almost every one of those carries - // the same configured batch size the job already has. If that rewrote LastModifiedAt, the field would - // degrade to "time of the last host start" and say nothing about the job. + // Arrange - the steady state. Create runs on every reconciliation pass, and almost every one of those + // carries the same configured batch size the job already has. If that rewrote LastModifiedAt, the field + // would degrade to "time of the last pass" and say nothing about the job. DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); BlobPurgeJobState existing = new() { @@ -170,8 +170,8 @@ public async Task Create_WhenAlreadyActive_AndBatchSizeChanged_MovesLastModified public async Task Run_DoesNotMoveLastModifiedAt() { // Arrange - Run schedules an orchestrator and changes nothing about the job. It is signalled by every - // Create, so writing here would move the field on every host start and undo the conditional write - // above. + // Create, so writing here would move the field on every reconciliation pass and undo the conditional + // write above. DateTimeOffset configuredAt = DateTimeOffset.UtcNow.AddDays(-2); BlobPurgeJobState existing = new() {