From 901e786f07c201d36cf8757e0dd33650b8cc3966 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 22:21:50 +0000 Subject: [PATCH 01/37] chore: sync Java client with Apify OpenAPI spec v2-2026-07-10T105921Z Forward-compatible spec update (error responses + relaxed nullability); version bump 0.1.3 -> 0.1.4. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 11 +++++++++++ README.md | 4 ++-- pom.xml | 2 +- src/main/java/com/apify/client/Version.java | 4 ++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df5035b..94fcf43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.1.4] - 2026-07-10 + +### Changed + +- Verified the client against OpenAPI specification version `v2-2026-07-10T105921Z` and bumped + `Version.API_SPEC_VERSION` accordingly. The spec delta is forward-compatible: new `401`/`402` + error responses on several endpoints (handled generically by `ApifyApiException`) and relaxed + response-field nullability/optionality (`RunOptions.maxTotalChargeUsd`, `maxItems` minimum, + `KeyValueStoreStats.deleteCount`/`listCount`, `UserPrivateInfo.proxy`, `Webhook.requestUrl`, + `StoreListActor.currentPricingInfo`, `notice`), all already tolerated by the response models. + ## [0.1.3] - 2026-07-10 ### Added diff --git a/README.md b/README.md index ef9d203..6396d0b 100644 --- a/README.md +++ b/README.md @@ -153,9 +153,9 @@ try { ## Versioning -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.1.3`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.1.4`). - `Version.API_SPEC_VERSION` — the Apify OpenAPI specification version this client was verified - against (`v2-2026-07-08T143931Z`). + against (`v2-2026-07-10T105921Z`). Changes to the public interface other than additive ones are considered breaking changes and follow [Semantic Versioning](https://semver.org/). diff --git a/pom.xml b/pom.xml index 69c1d5a..5c049dd 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.1.3 + 0.1.4 jar Apify Java Client diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index 290ebee..ff049eb 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,13 +13,13 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.1.3"; + public static final String CLIENT_VERSION = "0.1.4"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. * Corresponds to the {@code info.version} field of the Apify OpenAPI document. */ - public static final String API_SPEC_VERSION = "v2-2026-07-08T143931Z"; + public static final String API_SPEC_VERSION = "v2-2026-07-10T105921Z"; private Version() {} } From 0c3926ae90c8b86cff4912d9b826a8123232b5b3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 22:31:08 +0000 Subject: [PATCH 02/37] docs: bump install-snippet version to 0.1.4 and reword changelog Addresses review items: README Maven/Gradle install coordinates still pinned 0.1.3; CHANGELOG cited OpenAPI schema names as if Java classes. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 5 ++--- README.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94fcf43..08912d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,8 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Verified the client against OpenAPI specification version `v2-2026-07-10T105921Z` and bumped `Version.API_SPEC_VERSION` accordingly. The spec delta is forward-compatible: new `401`/`402` error responses on several endpoints (handled generically by `ApifyApiException`) and relaxed - response-field nullability/optionality (`RunOptions.maxTotalChargeUsd`, `maxItems` minimum, - `KeyValueStoreStats.deleteCount`/`listCount`, `UserPrivateInfo.proxy`, `Webhook.requestUrl`, - `StoreListActor.currentPricingInfo`, `notice`), all already tolerated by the response models. + nullability/optionality on some response fields, all already tolerated by the response models + (which ignore unknown fields and collect them in an `extra` map). ## [0.1.3] - 2026-07-10 diff --git a/README.md b/README.md index 6396d0b..53c57c1 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ Maven: com.apify apify-client - 0.1.3 + 0.1.4 ``` Gradle: ```groovy -implementation 'com.apify:apify-client:0.1.3' +implementation 'com.apify:apify-client:0.1.4' ``` ## Quick start From d0217a6e453f4147a68a10cf2e59772d53351a5c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 23:05:05 +0000 Subject: [PATCH 03/37] feat: add lazy iteration helpers across all paginated collections Adds iterate()/iterateItems()/iterateKeys() matching the reference JS iterable list(); shared PaginatedIterator engine; hermetic + integration tests; docs. Store.iterate signature change (limit = total cap, adds chunkSize) is breaking. Bumps client to 0.2.0. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 24 +- README.md | 6 +- docs/actors.md | 3 + docs/builds.md | 1 + docs/examples.md | 2 +- docs/misc.md | 5 +- docs/runs.md | 1 + docs/schedules.md | 1 + docs/storages.md | 12 +- docs/tasks.md | 1 + docs/webhooks.md | 2 + pom.xml | 2 +- .../AbstractWebhookCollectionClient.java | 13 + .../apify/client/ActorCollectionClient.java | 13 + .../client/ActorEnvVarCollectionClient.java | 11 + .../com/apify/client/ActorListOptions.java | 22 +- .../client/ActorVersionCollectionClient.java | 18 + .../apify/client/BuildCollectionClient.java | 13 + .../java/com/apify/client/DatasetClient.java | 43 ++- .../apify/client/DatasetCollectionClient.java | 13 + .../apify/client/DatasetListItemsOptions.java | 20 +- .../com/apify/client/KeyValueStoreClient.java | 75 ++++ .../client/KeyValueStoreCollectionClient.java | 18 + .../com/apify/client/ListKeysOptions.java | 21 +- .../java/com/apify/client/ListOptions.java | 18 +- .../com/apify/client/PaginatedIterator.java | 108 ++++++ .../client/RequestQueueCollectionClient.java | 18 + .../com/apify/client/ResourceContext.java | 26 ++ .../com/apify/client/RunCollectionClient.java | 24 ++ .../client/ScheduleCollectionClient.java | 13 + .../com/apify/client/StorageListOptions.java | 22 +- .../apify/client/StoreCollectionClient.java | 71 +--- .../com/apify/client/StoreListOptions.java | 31 +- .../apify/client/TaskCollectionClient.java | 13 + src/main/java/com/apify/client/Version.java | 2 +- .../WebhookDispatchCollectionClient.java | 18 + .../apify/client/PaginatedIteratorTest.java | 111 ++++++ .../com/apify/client/ReviewFixesTest.java | 5 +- .../apify/client/examples/IterateStore.java | 3 +- .../integration/IterationIntegrationTest.java | 321 ++++++++++++++++++ .../integration/StoreIntegrationTest.java | 2 +- 41 files changed, 1027 insertions(+), 119 deletions(-) create mode 100644 src/main/java/com/apify/client/PaginatedIterator.java create mode 100644 src/test/java/com/apify/client/PaginatedIteratorTest.java create mode 100644 src/test/java/com/apify/client/integration/IterationIntegrationTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 08912d6..79cfb4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,15 +5,33 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.1.4] - 2026-07-10 +## [0.2.0] - 2026-07-10 + +### Added + +- Lazy iteration helpers over every paginated collection, matching the reference JS client's + iterable `list()`: `iterate(options, chunkSize)` on the Actor, Actor-version, build, run, dataset, + key-value-store, request-queue, task, schedule, webhook, and webhook-dispatch collection clients; + `ActorEnvVarCollectionClient.iterate()` (the non-paginated env-var collection); and + `DatasetClient.iterateItems(...)` for dataset items and `KeyValueStoreClient.iterateKeys(...)` for + store keys. The options' `limit` caps the total number of items yielded and `chunkSize` sets the + per-request page size. ### Changed - Verified the client against OpenAPI specification version `v2-2026-07-10T105921Z` and bumped `Version.API_SPEC_VERSION` accordingly. The spec delta is forward-compatible: new `401`/`402` error responses on several endpoints (handled generically by `ApifyApiException`) and relaxed - nullability/optionality on some response fields, all already tolerated by the response models - (which ignore unknown fields and collect them in an `extra` map). + nullability/optionality on some response fields — already tolerated because the models use + nullable boxed field types (a JSON `null` deserializes to `null`) and an optional field simply + stays unset. + +### Breaking + +- `StoreCollectionClient.iterate` now takes `iterate(StoreListOptions, Long chunkSize)`, where the + options' `limit` is the total-items cap and `chunkSize` is the page size. Previously `limit` was + the per-page size. This aligns Store iteration with the reference client and the new collection + iterators. ## [0.1.3] - 2026-07-10 diff --git a/README.md b/README.md index 53c57c1..5c6fd27 100644 --- a/README.md +++ b/README.md @@ -21,14 +21,14 @@ Maven: com.apify apify-client - 0.1.4 + 0.2.0 ``` Gradle: ```groovy -implementation 'com.apify:apify-client:0.1.4' +implementation 'com.apify:apify-client:0.2.0' ``` ## Quick start @@ -153,7 +153,7 @@ try { ## Versioning -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.1.4`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.2.0`). - `Version.API_SPEC_VERSION` — the Apify OpenAPI specification version this client was verified against (`v2-2026-07-10T105921Z`). diff --git a/docs/actors.md b/docs/actors.md index c0586a2..95d6839 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -8,6 +8,7 @@ where `id` is an Actor ID or `username~name` (a `/` in the id is accepted and no | Method | Description | |---|---| | `list(ActorListOptions)` | List the account's Actors. Returns `PaginationList`. | +| `iterate(ActorListOptions, Long chunkSize)` | Lazy `Iterator` over all matches; the options' `limit` caps the total yielded, `chunkSize` sets the page size (both `null` = all / server default). | | `create(Object)` | Create a new Actor from a JSON-serializable definition. Returns `Actor`. | `ActorListOptions` adds `my(Boolean)` (only Actors owned by the current user) and @@ -90,6 +91,7 @@ and deletes a single version and exposes its environment variables. | Method | Description | |---|---| | `list(ListOptions)` | List the Actor's versions. Returns `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all versions; `limit` caps the total, `chunkSize` sets the page size. | | `create(Object version)` | Create a version. Returns `ActorVersion`. | ### `ActorVersionClient` — `client.actor(id).version(v)` @@ -121,6 +123,7 @@ encrypted), and matching getters `getName()`, `getValue()`, `getIsSecret()`. | Method | Description | |---|---| | `list()` | List the version's environment variables. Returns `PaginationList`. | +| `iterate()` | `Iterator` over the variables. The env-var collection is not paginated (all variables are returned at once), so this iterates a single fetched page; provided for API consistency. | | `create(ActorEnvVar)` | Create an environment variable. Returns `ActorEnvVar`. | ### `ActorEnvVarClient` — `client.actor(id).version(v).envVar(name)` diff --git a/docs/builds.md b/docs/builds.md index f6fe317..7930888 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -8,6 +8,7 @@ builds) and a single build with `client.build(id)`. | Method | Description | |---|---| | `list(ListOptions)` | List builds. Returns `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; `limit` caps the total, `chunkSize` sets the page size. | ## `BuildClient` diff --git a/docs/examples.md b/docs/examples.md index 0266997..c7dcae4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -100,7 +100,7 @@ if (last.isPresent()) { ## Lazy iteration of Store Actors ```java -Iterator it = client.store().iterate(new StoreListOptions().limit(10L)); +Iterator it = client.store().iterate(new StoreListOptions().limit(10L), 10L); int shown = 0; while (shown < 5 && it.hasNext()) { System.out.println(it.next().getName()); diff --git a/docs/misc.md b/docs/misc.md index 2c19e7d..3d6ee4c 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -7,14 +7,15 @@ Browse public Actors in the Apify Store. | Method | Description | |---|---| | `list(StoreListOptions)` | A page of Store Actors. Returns `PaginationList`. | -| `iterate(StoreListOptions)` | A lazy `Iterator` fetching pages on demand. | +| `iterate(StoreListOptions, Long chunkSize)` | A lazy `Iterator` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null` = all); `chunkSize` is the per-request page size (`null` = server default). | `StoreListOptions` fields: `offset`, `limit`, `search`, `sortBy`, `category`, `username`, `pricingModel` (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`), `includeUnrunnableActors`, `allowsAgenticUsers`, `responseFormat` (`full`, `agent`). ```java -Iterator it = client.store().iterate(new StoreListOptions().search("crawler").limit(20L)); +Iterator it = + client.store().iterate(new StoreListOptions().search("crawler").limit(20L), 10L); int shown = 0; while (shown < 5 && it.hasNext()) { ActorStoreListItem item = it.next(); diff --git a/docs/runs.md b/docs/runs.md index fd7ab21..e1a17b5 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -8,6 +8,7 @@ Access the run collection with `client.runs()` (or `client.actor(id).runs()` / | Method | Description | |---|---| | `list(ListOptions, RunListOptions)` | List runs. Returns `PaginationList`. | +| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; `limit` caps the total, `chunkSize` sets the page size. | `RunListOptions` adds `status(List)` (e.g. `SUCCEEDED`, `RUNNING`; sent comma-separated) and, for Actor/task-scoped collections, `startedAfter(String)` / `startedBefore(String)` (ISO-8601). diff --git a/docs/schedules.md b/docs/schedules.md index fed9409..6512fa0 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -8,6 +8,7 @@ Schedules automatically start Actor or task runs at specified times. Access the | Method | Description | |---|---| | `list(ListOptions)` | List schedules. Returns `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; `limit` caps the total, `chunkSize` sets the page size. | | `create(Object)` | Create a schedule from a JSON-serializable definition. Returns `Schedule`. | The `actions` array describes what the schedule runs (e.g. a `RUN_ACTOR` action): diff --git a/docs/storages.md b/docs/storages.md index ab35bfc..dbc0af9 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -24,6 +24,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | Method | Description | |---|---| | `list(StorageListOptions)` | List datasets. Returns `PaginationList`. | +| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; `limit` caps the total, `chunkSize` sets the page size. | | `getOrCreate(String name)` | Get or create a named dataset (empty name → unnamed). Returns `Dataset`. | | `getOrCreate(String name, Object schema)` | As above, sending a creation-time dataset `schema` when a new dataset is created. Returns `Dataset`. | @@ -36,6 +37,8 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | +| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; `limit` caps the total, `chunkSize` sets the page size. | +| `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | | `pushItems(Object)` | Push a single item or a list of items. | | `getStatistics()` | Dataset statistics. Returns `Optional`. | @@ -74,8 +77,9 @@ unsigned. ### `KeyValueStoreCollectionClient` — `client.keyValueStores()` -`list(StorageListOptions)`, `getOrCreate(String)`, and `getOrCreate(String, Object schema)` (the -latter sends a creation-time store `schema`), as for datasets. +`list(StorageListOptions)`, `iterate(StorageListOptions, Long chunkSize)`, `getOrCreate(String)`, and +`getOrCreate(String, Object schema)` (the latter sends a creation-time store `schema`), as for +datasets. ### `KeyValueStoreClient` — `client.keyValueStore(id)` @@ -83,6 +87,7 @@ latter sends a creation-time store `schema`), as for datasets. |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listKeys(ListKeysOptions)` | List keys. Returns `KeyValueStoreKeysPage`. | +| `iterateKeys(ListKeysOptions)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`); the options' `limit` caps the total number of keys yielded. | | `recordExists(String key)` | Whether a record exists. | | `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Returns `Optional`. | | `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. | @@ -117,7 +122,8 @@ expiry-aware storage-content signature — hence only `createKeysPublicUrl` take ### `RequestQueueCollectionClient` — `client.requestQueues()` -`list(StorageListOptions)` and `getOrCreate(String)`, as for datasets. +`list(StorageListOptions)`, `iterate(StorageListOptions, Long chunkSize)`, and `getOrCreate(String)`, +as for datasets. ### `RequestQueueClient` — `client.requestQueue(id)` diff --git a/docs/tasks.md b/docs/tasks.md index 2db0d62..61d6f60 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -8,6 +8,7 @@ Tasks are pre-configured Actor runs with stored input. Access the task collectio | Method | Description | |---|---| | `list(ListOptions)` | List tasks. Returns `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; `limit` caps the total, `chunkSize` sets the page size. | | `create(Object)` | Create a task from a JSON-serializable definition. Returns `Task`. | ```java diff --git a/docs/webhooks.md b/docs/webhooks.md index 59a8533..d2a69be 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -12,6 +12,7 @@ The account-wide collection supports both listing and creation. | Method | Description | |---|---| | `list(ListOptions)` | List webhooks. Returns `PaginationList`. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; `limit` caps the total, `chunkSize` sets the page size. Also available on the read-only nested collections. | | `create(Object)` | Create a webhook. Returns `Webhook`. | ### Nested webhook collections (read-only) @@ -56,6 +57,7 @@ System.out.println(dispatch.getId()); | Method | Description | |---|---| | `webhookDispatches().list(ListOptions)` | List dispatches. Returns `PaginationList`. | +| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; `limit` caps the total, `chunkSize` sets the page size. | | `webhookDispatch(id).get()` | Fetch a dispatch. Returns `Optional`. | `WebhookDispatch` fields: `getId()`, `getWebhookId()`. diff --git a/pom.xml b/pom.xml index 5c049dd..7f831ed 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.1.4 + 0.2.0 jar Apify Java Client diff --git a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java index 14750dd..e16a626 100644 --- a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java +++ b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** * Shared read-only behavior for webhook collections. Both the account-wide collection ({@link * WebhookCollectionClient}) and the read-only collections nested under an Actor or task ({@link @@ -19,4 +21,15 @@ public PaginationList list(ListOptions options) { options.apply(params); return ctx.listResource("", params, Webhook.class); } + + /** + * Returns a lazy iterator over the webhooks. The options' {@code limit} caps the total number + * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = + * server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Webhook.class); + } } diff --git a/src/main/java/com/apify/client/ActorCollectionClient.java b/src/main/java/com/apify/client/ActorCollectionClient.java index ea631cd..430f92c 100644 --- a/src/main/java/com/apify/client/ActorCollectionClient.java +++ b/src/main/java/com/apify/client/ActorCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the Actor collection ({@code GET/POST /v2/actors}). */ public final class ActorCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,17 @@ public PaginationList list(ActorListOptions options) { return ctx.listResource("", params, Actor.class); } + /** + * Returns a lazy iterator over the account's Actors, fetching pages on demand. The options' + * {@code limit} caps the total number of Actors yielded ({@code null} = all); {@code chunkSize} + * is the per-request page size ({@code null} = server default). + */ + public Iterator iterate(ActorListOptions options, Long chunkSize) { + ActorListOptions opts = options != null ? options : new ActorListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Actor.class); + } + /** Creates a new Actor. {@code actor} is any JSON-serializable Actor definition. */ public Actor create(Object actor) { return ctx.createResource(new QueryParams(), actor, Actor.class); diff --git a/src/main/java/com/apify/client/ActorEnvVarCollectionClient.java b/src/main/java/com/apify/client/ActorEnvVarCollectionClient.java index 1058867..4c31dea 100644 --- a/src/main/java/com/apify/client/ActorEnvVarCollectionClient.java +++ b/src/main/java/com/apify/client/ActorEnvVarCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** * A client for an Actor version's environment variable collection ({@code GET/POST * /v2/actors/{actorId}/versions/{versionNumber}/env-vars}). @@ -16,6 +18,15 @@ public PaginationList list() { return ctx.listResource("", new QueryParams(), ActorEnvVar.class); } + /** + * Returns an iterator over the version's environment variables. The env-var collection is not + * paginated (the API returns every variable in one response), so this iterates a single fetched + * page; the method exists for API consistency with the other collection clients. + */ + public Iterator iterate() { + return list().getItems().iterator(); + } + /** Creates a new environment variable. */ public ActorEnvVar create(ActorEnvVar envVar) { return ctx.createResource(new QueryParams(), envVar, ActorEnvVar.class); diff --git a/src/main/java/com/apify/client/ActorListOptions.java b/src/main/java/com/apify/client/ActorListOptions.java index efddf2b..88496fc 100644 --- a/src/main/java/com/apify/client/ActorListOptions.java +++ b/src/main/java/com/apify/client/ActorListOptions.java @@ -38,11 +38,23 @@ public ActorListOptions sortBy(String sortBy) { return this; } + Long offsetValue() { + return offset; + } + + Long limitValue() { + return limit; + } + void apply(QueryParams q) { - q.addLong("offset", offset) - .addLong("limit", limit) - .addBool("desc", desc) - .addBool("my", my) - .addString("sortBy", sortBy); + q.addLong("offset", offset).addLong("limit", limit); + applyFilters(q); + } + + /** + * Applies every filter except {@code offset}/{@code limit}, which the iterator drives per page. + */ + void applyFilters(QueryParams q) { + q.addBool("desc", desc).addBool("my", my).addString("sortBy", sortBy); } } diff --git a/src/main/java/com/apify/client/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/ActorVersionCollectionClient.java index 22c7fdf..1072364 100644 --- a/src/main/java/com/apify/client/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/ActorVersionCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for an Actor's version collection ({@code GET/POST /v2/actors/{actorId}/versions}). */ public final class ActorVersionCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,22 @@ public PaginationList list(ListOptions options) { return ctx.listResource("", params, ActorVersion.class); } + /** + * Returns a lazy iterator over the Actor's versions. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + opts::applyFilters, + ActorVersion.class); + } + /** Creates a new Actor version. {@code version} is any JSON-serializable version definition. */ public ActorVersion create(Object version) { return ctx.createResource(new QueryParams(), version, ActorVersion.class); diff --git a/src/main/java/com/apify/client/BuildCollectionClient.java b/src/main/java/com/apify/client/BuildCollectionClient.java index 3552395..86a36b5 100644 --- a/src/main/java/com/apify/client/BuildCollectionClient.java +++ b/src/main/java/com/apify/client/BuildCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** * A client for a build collection: the account-wide collection ({@code GET /v2/actor-builds}) or an * Actor's builds ({@code GET /v2/actors/{id}/builds}). @@ -17,4 +19,15 @@ public PaginationList list(ListOptions options) { options.apply(params); return ctx.listResource("", params, Build.class); } + + /** + * Returns a lazy iterator over the builds. The options' {@code limit} caps the total number + * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = + * server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Build.class); + } } diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 24d57b1..6b0eae5 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.JsonNode; +import java.util.Iterator; import java.util.List; import java.util.Optional; @@ -70,6 +71,44 @@ public PaginationList listItems(DatasetListItemsOptions options) { public PaginationList listItems(DatasetListItemsOptions options, Class itemClass) { QueryParams params = new QueryParams(); options.apply(params); + return fetchItemsPage(params, options.descValue(), itemClass); + } + + /** + * Returns a lazy iterator over the dataset's items, decoding each into a generic {@link + * JsonNode}. For typed decoding use {@link #iterateItems(DatasetListItemsOptions, Long, Class)}. + */ + public Iterator iterateItems(DatasetListItemsOptions options, Long chunkSize) { + return iterateItems(options, chunkSize, JsonNode.class); + } + + /** + * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, + * fetching pages on demand. The options' {@code limit} caps the total number of items yielded + * ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = server + * default). + */ + public Iterator iterateItems( + DatasetListItemsOptions options, Long chunkSize, Class itemClass) { + DatasetListItemsOptions opts = options != null ? options : new DatasetListItemsOptions(); + return new PaginatedIterator<>( + opts.limitValue(), + chunkSize, + opts.offsetValue(), + (offset, pageLimit) -> { + QueryParams p = new QueryParams().addLong("offset", offset).addLong("limit", pageLimit); + opts.applyFilters(p); + return fetchItemsPage(p, opts.descValue(), itemClass); + }); + } + + /** + * Fetches a single page of dataset items for the already-built query {@code params}. The dataset + * items endpoint returns a bare JSON array (not a data envelope) and reports pagination via + * {@code X-Apify-Pagination-*} headers, surfaced in the returned {@link PaginationList}. + */ + private PaginationList fetchItemsPage( + QueryParams params, Boolean desc, Class itemClass) { String url = ctx.mergedParams(params).applyToUrl(ctx.subUrl("items")); ApiResponse resp = http.call("GET", url, null, "", http.baseRequestTimeout()); @@ -83,8 +122,8 @@ public PaginationList listItems(DatasetListItemsOptions options, Class result.setTotal(headerLong(resp, "X-Apify-Pagination-Total", count)); result.setOffset(headerLong(resp, "X-Apify-Pagination-Offset", 0)); result.setLimit(headerLong(resp, "X-Apify-Pagination-Limit", count)); - if (options.descValue() != null) { - result.setDesc(options.descValue()); + if (desc != null) { + result.setDesc(desc); } return result; } diff --git a/src/main/java/com/apify/client/DatasetCollectionClient.java b/src/main/java/com/apify/client/DatasetCollectionClient.java index 6c7972f..2dea4ee 100644 --- a/src/main/java/com/apify/client/DatasetCollectionClient.java +++ b/src/main/java/com/apify/client/DatasetCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the dataset collection ({@code GET/POST /v2/datasets}). */ public final class DatasetCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,17 @@ public PaginationList list(StorageListOptions options) { return ctx.listResource("", params, Dataset.class); } + /** + * Returns a lazy iterator over the datasets. The options' {@code limit} caps the total number + * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = + * server default). + */ + public Iterator iterate(StorageListOptions options, Long chunkSize) { + StorageListOptions opts = options != null ? options : new StorageListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Dataset.class); + } + /** * Gets the dataset with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed dataset. diff --git a/src/main/java/com/apify/client/DatasetListItemsOptions.java b/src/main/java/com/apify/client/DatasetListItemsOptions.java index a449924..ad01795 100644 --- a/src/main/java/com/apify/client/DatasetListItemsOptions.java +++ b/src/main/java/com/apify/client/DatasetListItemsOptions.java @@ -120,10 +120,24 @@ Boolean descValue() { return desc; } + Long offsetValue() { + return offset; + } + + Long limitValue() { + return limit; + } + void apply(QueryParams q) { - q.addLong("offset", offset) - .addLong("limit", limit) - .addBool("desc", desc) + q.addLong("offset", offset).addLong("limit", limit); + applyFilters(q); + } + + /** + * Applies every option except {@code offset}/{@code limit}, which the iterator drives per page. + */ + void applyFilters(QueryParams q) { + q.addBool("desc", desc) .addCsv("fields", fields) .addCsv("outputFields", outputFields) .addCsv("omit", omit) diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 16c8f80..5b8c630 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -1,5 +1,8 @@ package com.apify.client; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; import java.util.Optional; /** A client for a specific key-value store (and run-nested variants). */ @@ -53,6 +56,78 @@ public KeyValueStoreKeysPage listKeys(ListKeysOptions options) { return ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); } + /** + * Returns a lazy iterator over this store's keys, fetching pages on demand via the cursor-based + * ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} caps the total number + * of keys yielded ({@code null} = all); any {@code exclusiveStartKey} sets the starting point. + */ + public Iterator iterateKeys(ListKeysOptions options) { + return new KeysIterator(options != null ? options : new ListKeysOptions()); + } + + /** + * Lazily iterates over a store's keys via the cursor-based ({@code exclusiveStartKey}) listing. + */ + private final class KeysIterator implements Iterator { + private final ListKeysOptions options; + private List buffer = List.of(); + private int pos; + private String cursor; + private Long remaining; + private boolean exhausted; + + KeysIterator(ListKeysOptions options) { + this.options = options; + this.cursor = options.exclusiveStartKeyValue(); + Long limit = options.limitValue(); + this.remaining = limit != null && limit > 0 ? limit : null; + } + + @Override + public boolean hasNext() { + while (pos >= buffer.size()) { + if (exhausted) { + return false; + } + fetchPage(); + } + return true; + } + + @Override + public KeyValueStoreKey next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return buffer.get(pos++); + } + + private void fetchPage() { + QueryParams params = new QueryParams(); + // The API caps the page size; requesting `remaining` keeps the last page from overshooting + // the + // caller's total cap. A null limit lets the server choose its default page size. + params.addLong("limit", remaining); + params.addString("exclusiveStartKey", cursor); + options.applyFilters(params); + KeyValueStoreKeysPage page = + ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); + buffer = page.getItems(); + pos = 0; + cursor = page.getNextExclusiveStartKey(); + if (remaining != null) { + remaining -= buffer.size(); + } + // Stop when the page is empty, there is no next cursor, or the total cap is reached. + if (buffer.isEmpty() + || cursor == null + || cursor.isEmpty() + || (remaining != null && remaining <= 0)) { + exhausted = true; + } + } + } + /** Reports whether a record with the given key exists. */ public boolean recordExists(String key) { return ctx.headExists("records/" + ResourceContext.encodePathSegment(key), new QueryParams()); diff --git a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java index 63e2b29..6140d71 100644 --- a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the key-value store collection ({@code GET/POST /v2/key-value-stores}). */ public final class KeyValueStoreCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,22 @@ public PaginationList list(StorageListOptions options) { return ctx.listResource("", params, KeyValueStore.class); } + /** + * Returns a lazy iterator over the key-value stores. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(StorageListOptions options, Long chunkSize) { + StorageListOptions opts = options != null ? options : new StorageListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + opts::applyFilters, + KeyValueStore.class); + } + /** * Gets the store with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed store. diff --git a/src/main/java/com/apify/client/ListKeysOptions.java b/src/main/java/com/apify/client/ListKeysOptions.java index a9d3d70..7044243 100644 --- a/src/main/java/com/apify/client/ListKeysOptions.java +++ b/src/main/java/com/apify/client/ListKeysOptions.java @@ -42,10 +42,25 @@ String signatureValue() { return signature; } + Long limitValue() { + return limit; + } + + String exclusiveStartKeyValue() { + return exclusiveStartKey; + } + void apply(QueryParams q) { - q.addLong("limit", limit) - .addString("exclusiveStartKey", exclusiveStartKey) - .addString("prefix", prefix) + q.addLong("limit", limit).addString("exclusiveStartKey", exclusiveStartKey); + applyFilters(q); + } + + /** + * Applies every filter except {@code limit}/{@code exclusiveStartKey}, which the key iterator + * drives per page. + */ + void applyFilters(QueryParams q) { + q.addString("prefix", prefix) .addString("collection", collection) .addString("signature", signature); } diff --git a/src/main/java/com/apify/client/ListOptions.java b/src/main/java/com/apify/client/ListOptions.java index f18a0dc..4c3e5de 100644 --- a/src/main/java/com/apify/client/ListOptions.java +++ b/src/main/java/com/apify/client/ListOptions.java @@ -28,7 +28,23 @@ public ListOptions desc(Boolean desc) { return this; } + Long offsetValue() { + return offset; + } + + Long limitValue() { + return limit; + } + void apply(QueryParams q) { - q.addLong("offset", offset).addLong("limit", limit).addBool("desc", desc); + q.addLong("offset", offset).addLong("limit", limit); + applyFilters(q); + } + + /** + * Applies every filter except {@code offset}/{@code limit}, which the iterator drives per page. + */ + void applyFilters(QueryParams q) { + q.addBool("desc", desc); } } diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java new file mode 100644 index 0000000..a74f34e --- /dev/null +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -0,0 +1,108 @@ +package com.apify.client; + +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * A lazy {@link Iterator} over an offset/limit-paginated list endpoint, fetching one page at a + * time. Internal reusable engine shared by every collection client's {@code iterate} method, so the + * paging arithmetic lives in one place (DRY). + * + *

The end-user behaviour matches the reference JS client's {@code _listPaginatedFromCallback}: + * + *

    + *
  • {@code totalLimit} caps the total number of items yielded across all pages ({@code + * null} = yield everything). + *
  • {@code chunkSize} is the per-request page size ({@code null} = let the API choose). Each + * page requests {@code min(remaining-under-cap, chunkSize)} items so the cap is never + * overshot. + *
+ * + *

Iteration stops when the cap is reached, the API returns an empty page, or a page comes back + * shorter than requested (the last page). The short-page check — rather than relying solely on the + * reported {@code total} — makes iteration robust to a momentarily under-reported {@code total} + * (the count can lag right after a write), which would otherwise truncate a full page. The reported + * total is only consulted to terminate the "unbounded, server-chosen page size" case, where no page + * size is known to compare against. + */ +final class PaginatedIterator implements Iterator { + + /** Fetches a single page starting at {@code offset}, requesting at most {@code limit} items. */ + @FunctionalInterface + interface PageFetcher { + PaginationList fetch(long offset, Long limit); + } + + private final PageFetcher fetcher; + private final Long totalLimit; + private final Long chunkSize; + + private List buffer = List.of(); + private int pos; + private long offset; + private long yielded; + private boolean exhausted; + + PaginatedIterator(Long totalLimit, Long chunkSize, Long startOffset, PageFetcher fetcher) { + this.totalLimit = totalLimit != null && totalLimit > 0 ? totalLimit : null; + this.chunkSize = chunkSize; + this.offset = startOffset != null && startOffset > 0 ? startOffset : 0; + this.fetcher = fetcher; + } + + @Override + public boolean hasNext() { + while (pos >= buffer.size()) { + if (exhausted) { + return false; + } + fetchNextPage(); + } + return true; + } + + @Override + public T next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return buffer.get(pos++); + } + + private void fetchNextPage() { + Long capRemaining = totalLimit != null ? totalLimit - yielded : null; + Long pageLimit = minForLimit(capRemaining, chunkSize); + PaginationList page = fetcher.fetch(offset, pageLimit); + buffer = page.getItems(); + pos = 0; + int count = buffer.size(); + offset += count; + yielded += count; + + boolean capReached = totalLimit != null && yielded >= totalLimit; + boolean shortPage = pageLimit != null && count < pageLimit; + // With no explicit page size the API returns its default page, so there is nothing to compare a + // short page against — fall back to the reported total to detect the end of the collection. + boolean totalReached = pageLimit == null && offset >= page.getTotal(); + if (count == 0 || capReached || shortPage || totalReached) { + exhausted = true; + } + } + + /** + * The API treats {@code 0} as "unset" for a limit, so this returns the smaller of the two + * positive bounds, or {@code null} (server default) when neither is a positive value. + */ + private static Long minForLimit(Long a, Long b) { + Long x = a != null && a > 0 ? a : null; + Long y = b != null && b > 0 ? b : null; + if (x == null) { + return y; + } + if (y == null) { + return x; + } + return Math.min(x, y); + } +} diff --git a/src/main/java/com/apify/client/RequestQueueCollectionClient.java b/src/main/java/com/apify/client/RequestQueueCollectionClient.java index a713b32..235f612 100644 --- a/src/main/java/com/apify/client/RequestQueueCollectionClient.java +++ b/src/main/java/com/apify/client/RequestQueueCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the request queue collection ({@code GET/POST /v2/request-queues}). */ public final class RequestQueueCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,22 @@ public PaginationList list(StorageListOptions options) { return ctx.listResource("", params, RequestQueue.class); } + /** + * Returns a lazy iterator over the request queues. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(StorageListOptions options, Long chunkSize) { + StorageListOptions opts = options != null ? options : new StorageListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + opts::applyFilters, + RequestQueue.class); + } + /** * Gets the queue with the given name, creating it if it does not exist. An empty/{@code null} * name creates a new unnamed queue. diff --git a/src/main/java/com/apify/client/ResourceContext.java b/src/main/java/com/apify/client/ResourceContext.java index e75cf4c..9d47622 100644 --- a/src/main/java/com/apify/client/ResourceContext.java +++ b/src/main/java/com/apify/client/ResourceContext.java @@ -4,7 +4,9 @@ import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.time.Duration; +import java.util.Iterator; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Predicate; /** @@ -167,6 +169,30 @@ PaginationList listResource(String subPath, QueryParams params, Class return getResourceRequired(subPath, params, listType); } + /** + * Builds a lazy iterator over an offset/limit-paginated endpoint. {@code applyFilters} adds the + * per-endpoint filter params (everything except {@code offset}/{@code limit}, which the iterator + * drives per page). {@code totalLimit} caps the total items yielded; {@code chunkSize} is the + * page size (both {@code null} meaning "unbounded" / "server default"). + */ + Iterator iterateResource( + String subPath, + Long totalLimit, + Long chunkSize, + Long startOffset, + Consumer applyFilters, + Class itemClass) { + return new PaginatedIterator<>( + totalLimit, + chunkSize, + startOffset, + (offset, pageLimit) -> { + QueryParams p = new QueryParams().addLong("offset", offset).addLong("limit", pageLimit); + applyFilters.accept(p); + return listResource(subPath, p, itemClass); + }); + } + T createResource(QueryParams params, Object body, Class dataClass) { String u = mergedParams(params).applyToUrl(subUrl("")); ApiResponse resp = diff --git a/src/main/java/com/apify/client/RunCollectionClient.java b/src/main/java/com/apify/client/RunCollectionClient.java index 1bcde19..10d0980 100644 --- a/src/main/java/com/apify/client/RunCollectionClient.java +++ b/src/main/java/com/apify/client/RunCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** * A client for a run collection: the account-wide collection ({@code GET /v2/actor-runs}), an * Actor's runs ({@code GET /v2/actors/{id}/runs}), or a task's runs ({@code GET @@ -26,4 +28,26 @@ public PaginationList list(ListOptions options, RunListOptions filter) } return ctx.listResource("", params, ActorRun.class); } + + /** + * Returns a lazy iterator over the runs, applying the standard pagination and run-specific + * filters. The options' {@code limit} caps the total number yielded ({@code null} = all); {@code + * chunkSize} is the per-request page size ({@code null} = server default). Both {@code options} + * and {@code filter} may be {@code null}. + */ + public Iterator iterate(ListOptions options, RunListOptions filter, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + p -> { + opts.applyFilters(p); + if (filter != null) { + filter.apply(p); + } + }, + ActorRun.class); + } } diff --git a/src/main/java/com/apify/client/ScheduleCollectionClient.java b/src/main/java/com/apify/client/ScheduleCollectionClient.java index b82d569..5914c0d 100644 --- a/src/main/java/com/apify/client/ScheduleCollectionClient.java +++ b/src/main/java/com/apify/client/ScheduleCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the schedule collection ({@code GET/POST /v2/schedules}). */ public final class ScheduleCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,17 @@ public PaginationList list(ListOptions options) { return ctx.listResource("", params, Schedule.class); } + /** + * Returns a lazy iterator over the account's schedules. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Schedule.class); + } + /** Creates a new schedule. {@code schedule} is any JSON-serializable schedule definition. */ public Schedule create(Object schedule) { return ctx.createResource(new QueryParams(), schedule, Schedule.class); diff --git a/src/main/java/com/apify/client/StorageListOptions.java b/src/main/java/com/apify/client/StorageListOptions.java index 8993a81..ab14c21 100644 --- a/src/main/java/com/apify/client/StorageListOptions.java +++ b/src/main/java/com/apify/client/StorageListOptions.java @@ -42,11 +42,23 @@ public StorageListOptions ownership(String ownership) { return this; } + Long offsetValue() { + return offset; + } + + Long limitValue() { + return limit; + } + void apply(QueryParams q) { - q.addLong("offset", offset) - .addLong("limit", limit) - .addBool("desc", desc) - .addBool("unnamed", unnamed) - .addString("ownership", ownership); + q.addLong("offset", offset).addLong("limit", limit); + applyFilters(q); + } + + /** + * Applies every filter except {@code offset}/{@code limit}, which the iterator drives per page. + */ + void applyFilters(QueryParams q) { + q.addBool("desc", desc).addBool("unnamed", unnamed).addString("ownership", ownership); } } diff --git a/src/main/java/com/apify/client/StoreCollectionClient.java b/src/main/java/com/apify/client/StoreCollectionClient.java index 2d43ca7..5add221 100644 --- a/src/main/java/com/apify/client/StoreCollectionClient.java +++ b/src/main/java/com/apify/client/StoreCollectionClient.java @@ -1,8 +1,6 @@ package com.apify.client; import java.util.Iterator; -import java.util.List; -import java.util.NoSuchElementException; /** A client for browsing the Apify Store ({@code GET /v2/store}). */ public final class StoreCollectionClient { @@ -20,63 +18,18 @@ public PaginationList list(StoreListOptions options) { } /** - * Returns a lazy iterator over all Store Actors matching the options, fetching pages on demand. - * The options' {@code limit} (if set) is used as the per-page size. + * Returns a lazy iterator over Store Actors matching the options, fetching pages on demand. The + * options' {@code limit} caps the total number of Actors yielded ({@code null} = all); {@code + * chunkSize} is the per-request page size ({@code null} = server default). */ - public Iterator iterate(StoreListOptions options) { - return new StoreIterator(options); - } - - /** Lazily iterates over Apify Store Actors, fetching one page at a time. */ - private final class StoreIterator implements Iterator { - private final StoreListOptions options; - private List buffer = List.of(); - private int pos; - private long offset; - private boolean exhausted; - - StoreIterator(StoreListOptions options) { - // Copy so paging state stays internal: the caller's instance is never mutated (safe to reuse - // or iterate twice), and its initial offset is honored as the starting page. - this.options = options.copy(); - Long initialOffset = this.options.offsetValue(); - this.offset = initialOffset != null ? initialOffset : 0; - } - - @Override - public boolean hasNext() { - while (pos >= buffer.size()) { - if (exhausted) { - return false; - } - fetchPage(); - } - return true; - } - - @Override - public ActorStoreListItem next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - return buffer.get(pos++); - } - - private void fetchPage() { - options.offsetInternal(offset); - PaginationList page = list(options); - buffer = page.getItems(); - pos = 0; - offset += page.getItems().size(); - // Terminate on an empty page, or on a short page when a page size (limit) is known — a page - // returning fewer items than requested is the last one. We deliberately do NOT stop on - // `offset >= total`: a momentarily stale (under-reported) `total`, e.g. from read-replica - // lag right after a write, could otherwise cut iteration short while items remain. - Long limit = options.limitValue(); - boolean shortPage = limit != null && limit > 0 && page.getItems().size() < limit; - if (page.getItems().isEmpty() || shortPage) { - exhausted = true; - } - } + public Iterator iterate(StoreListOptions options, Long chunkSize) { + StoreListOptions opts = options != null ? options : new StoreListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + opts::applyFilters, + ActorStoreListItem.class); } } diff --git a/src/main/java/com/apify/client/StoreListOptions.java b/src/main/java/com/apify/client/StoreListOptions.java index 00b76a7..d2947ed 100644 --- a/src/main/java/com/apify/client/StoreListOptions.java +++ b/src/main/java/com/apify/client/StoreListOptions.java @@ -84,31 +84,16 @@ Long offsetValue() { return offset; } - StoreListOptions offsetInternal(long offset) { - this.offset = offset; - return this; - } - - /** Returns an independent copy, so iteration paging cannot mutate a caller-owned instance. */ - StoreListOptions copy() { - StoreListOptions c = new StoreListOptions(); - c.offset = offset; - c.limit = limit; - c.search = search; - c.sortBy = sortBy; - c.category = category; - c.username = username; - c.pricingModel = pricingModel; - c.includeUnrunnableActors = includeUnrunnableActors; - c.allowsAgenticUsers = allowsAgenticUsers; - c.responseFormat = responseFormat; - return c; + void apply(QueryParams q) { + q.addLong("offset", offset).addLong("limit", limit); + applyFilters(q); } - void apply(QueryParams q) { - q.addLong("offset", offset) - .addLong("limit", limit) - .addString("search", search) + /** + * Applies every filter except {@code offset}/{@code limit}, which the iterator drives per page. + */ + void applyFilters(QueryParams q) { + q.addString("search", search) .addString("sortBy", sortBy) .addString("category", category) .addString("username", username) diff --git a/src/main/java/com/apify/client/TaskCollectionClient.java b/src/main/java/com/apify/client/TaskCollectionClient.java index 9ca5a0c..99ac196 100644 --- a/src/main/java/com/apify/client/TaskCollectionClient.java +++ b/src/main/java/com/apify/client/TaskCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** A client for the Actor task collection ({@code GET/POST /v2/actor-tasks}). */ public final class TaskCollectionClient { private final ResourceContext ctx; @@ -15,6 +17,17 @@ public PaginationList list(ListOptions options) { return ctx.listResource("", params, Task.class); } + /** + * Returns a lazy iterator over the account's tasks. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", opts.limitValue(), chunkSize, opts.offsetValue(), opts::applyFilters, Task.class); + } + /** Creates a new task. {@code task} is any JSON-serializable task definition. */ public Task create(Object task) { return ctx.createResource(new QueryParams(), task, Task.class); diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index ff049eb..aa938d4 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,7 +13,7 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.1.4"; + public static final String CLIENT_VERSION = "0.2.0"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. diff --git a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java index 65063fb..76056a7 100644 --- a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java +++ b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java @@ -1,5 +1,7 @@ package com.apify.client; +import java.util.Iterator; + /** * A client for a webhook dispatch collection: the account-wide collection ({@code GET * /v2/webhook-dispatches}) or dispatches nested under a webhook. @@ -17,4 +19,20 @@ public PaginationList list(ListOptions options) { options.apply(params); return ctx.listResource("", params, WebhookDispatch.class); } + + /** + * Returns a lazy iterator over the webhook dispatches. The options' {@code limit} caps the total + * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). + */ + public Iterator iterate(ListOptions options, Long chunkSize) { + ListOptions opts = options != null ? options : new ListOptions(); + return ctx.iterateResource( + "", + opts.limitValue(), + chunkSize, + opts.offsetValue(), + opts::applyFilters, + WebhookDispatch.class); + } } diff --git a/src/test/java/com/apify/client/PaginatedIteratorTest.java b/src/test/java/com/apify/client/PaginatedIteratorTest.java new file mode 100644 index 0000000..53da9ed --- /dev/null +++ b/src/test/java/com/apify/client/PaginatedIteratorTest.java @@ -0,0 +1,111 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import org.junit.jupiter.api.Test; + +/** + * Hermetic (token-free) tests for the offset/limit iteration engine. They drive {@link + * PaginatedIterator} with a stub page fetcher over a synthetic collection, pinning the total-cap / + * chunk-size arithmetic and termination that mirror the reference JS {@code + * _listPaginatedFromCallback} — without any network or {@code APIFY_TOKEN}. + */ +class PaginatedIteratorTest { + + /** A fake paginated endpoint over integers {@code [0, available)} reporting {@code total}. */ + private static final class StubFetcher implements PaginatedIterator.PageFetcher { + final long total; + final long available; + final List requests = new ArrayList<>(); + + StubFetcher(long total) { + this(total, total); + } + + StubFetcher(long total, long available) { + this.total = total; + this.available = available; + } + + @Override + public PaginationList fetch(long offset, Long limit) { + requests.add(new long[] {offset, limit == null ? -1 : limit}); + long take = limit == null ? Long.MAX_VALUE : limit; + List items = new ArrayList<>(); + for (long i = offset; i < available && items.size() < take; i++) { + items.add((int) i); + } + PaginationList page = new PaginationList<>(); + page.setItems(items); + page.setTotal(total); + page.setOffset(offset); + page.setCount(items.size()); + return page; + } + } + + private static List drain(java.util.Iterator it) { + List out = new ArrayList<>(); + while (it.hasNext()) { + out.add(it.next()); + } + return out; + } + + @Test + void totalCapTrimsLastPage() { + StubFetcher f = new StubFetcher(10); + List got = drain(new PaginatedIterator<>(3L, 2L, null, f)); + assertEquals(List.of(0, 1, 2), got, "limit=3 caps the total yielded across pages"); + // First page requests min(cap,chunk)=2; second page requests min(remaining=1,chunk=2)=1. + assertEquals(2, f.requests.size()); + assertEquals(0L, f.requests.get(0)[0]); + assertEquals(2L, f.requests.get(0)[1]); + assertEquals(2L, f.requests.get(1)[0]); + assertEquals(1L, f.requests.get(1)[1]); + } + + @Test + void chunkSizePagesAll() { + StubFetcher f = new StubFetcher(5); + List got = drain(new PaginatedIterator<>(null, 2L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got); + assertEquals(3, f.requests.size(), "5 items / page size 2 => 3 pages"); + } + + @Test + void noCapNoChunkUsesServerDefault() { + StubFetcher f = new StubFetcher(4); + List got = drain(new PaginatedIterator<>(null, null, null, f)); + assertEquals(List.of(0, 1, 2, 3), got); + assertEquals(1, f.requests.size()); + assertEquals(-1L, f.requests.get(0)[1], "no cap and no chunk => null limit (server default)"); + } + + @Test + void capLargerThanTotalYieldsAll() { + StubFetcher f = new StubFetcher(4); + assertEquals(List.of(0, 1, 2, 3), drain(new PaginatedIterator<>(100L, null, null, f))); + } + + @Test + void startOffsetIsHonored() { + StubFetcher f = new StubFetcher(10); + List got = drain(new PaginatedIterator<>(null, null, 7L, f)); + assertEquals(List.of(7, 8, 9), got, "iteration starts at the requested offset"); + assertEquals(7L, f.requests.get(0)[0]); + } + + @Test + void emptyCollectionYieldsNothing() { + StubFetcher f = new StubFetcher(0); + java.util.Iterator it = new PaginatedIterator<>(null, 5L, null, f); + assertFalse(it.hasNext()); + assertThrows(NoSuchElementException.class, it::next); + } +} diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ReviewFixesTest.java index 3c17be2..0575db4 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ReviewFixesTest.java @@ -267,7 +267,7 @@ void storeIterateDoesNotMutateCallerOptionsAndHonorsOffset() { MockBackend.ofConstant( 200, "{\"data\":{\"items\":[],\"total\":0,\"offset\":0,\"limit\":0,\"count\":0}}"); StoreListOptions options = new StoreListOptions().offset(100L).limit(50L); - client(backend).store().iterate(options).hasNext(); + client(backend).store().iterate(options, null).hasNext(); // The caller's initial offset must be honored for paging and left untouched afterwards. assertTrue(backend.lastUrl.contains("offset=100"), backend.lastUrl); assertEquals(100L, options.offsetValue(), "iteration must not mutate the caller's options"); @@ -284,8 +284,9 @@ void storeIterateWalksMultiplePages() { MockBackend.ok( 200, "{\"data\":{\"items\":[{}],\"total\":3,\"offset\":2,\"limit\":2,\"count\":1}}"))); + // No total cap; page size 2 drives paging across the reported total of 3. java.util.Iterator it = - client(backend).store().iterate(new StoreListOptions().limit(2L)); + client(backend).store().iterate(new StoreListOptions(), 2L); int count = 0; while (it.hasNext()) { it.next(); diff --git a/src/test/java/com/apify/client/examples/IterateStore.java b/src/test/java/com/apify/client/examples/IterateStore.java index c1946c7..a733f02 100644 --- a/src/test/java/com/apify/client/examples/IterateStore.java +++ b/src/test/java/com/apify/client/examples/IterateStore.java @@ -15,7 +15,8 @@ private IterateStore() {} public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); - Iterator it = client.store().iterate(new StoreListOptions().limit(10L)); + Iterator it = + client.store().iterate(new StoreListOptions().limit(10L), 10L); int count = 0; while (count < 5 && it.hasNext()) { ActorStoreListItem item = it.next(); diff --git a/src/test/java/com/apify/client/integration/IterationIntegrationTest.java b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java new file mode 100644 index 0000000..1fb140a --- /dev/null +++ b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java @@ -0,0 +1,321 @@ +package com.apify.client.integration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.apify.client.Actor; +import com.apify.client.ActorEnvVar; +import com.apify.client.ActorListOptions; +import com.apify.client.ApifyClient; +import com.apify.client.Build; +import com.apify.client.Dataset; +import com.apify.client.DatasetClient; +import com.apify.client.DatasetListItemsOptions; +import com.apify.client.KeyValueStore; +import com.apify.client.KeyValueStoreClient; +import com.apify.client.KeyValueStoreKey; +import com.apify.client.ListKeysOptions; +import com.apify.client.ListOptions; +import com.apify.client.RequestQueue; +import com.apify.client.RunListOptions; +import com.apify.client.Schedule; +import com.apify.client.StorageListOptions; +import com.apify.client.Task; +import com.apify.client.Webhook; +import com.apify.client.WebhookDispatch; +import com.fasterxml.jackson.databind.JsonNode; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import org.junit.jupiter.api.Test; + +/** + * Iteration coverage for every paginated collection client that exposes an {@code iterate} helper, + * mirroring the reference client's async iterators. Where creation is cheap, tests create a few + * uniquely-named resources and assert iteration finds them (so paging is exercised across pages); + * for resources whose setup is expensive (builds, runs, dispatches) they iterate a bounded slice + * and assert the iterator behaves. All resources are uniquely named for parallel isolation. + */ +class IterationIntegrationTest extends IntegrationBase { + + /** Iterates until every target id is seen (lazily; stops early) or the iterator is exhausted. */ + private static boolean findsAll( + Iterator it, Function idOf, Set targets) { + Set remaining = new HashSet<>(targets); + int safety = 0; + while (it.hasNext() && !remaining.isEmpty() && safety++ < 10000) { + remaining.remove(idOf.apply(it.next())); + } + return remaining.isEmpty(); + } + + @Test + void iterateDatasetItems() { + ApifyClient client = requireClient(); + Dataset ds = client.datasets().getOrCreate(uniqueName("it-ds-items")); + try { + DatasetClient dataset = client.dataset(ds.getId()); + List> items = new ArrayList<>(); + for (int i = 1; i <= 5; i++) { + items.add(Map.of("n", i)); + } + dataset.pushItems(items); + + List seen = new ArrayList<>(); + Iterator it = dataset.iterateItems(new DatasetListItemsOptions(), 2L); + while (it.hasNext()) { + seen.add(it.next().get("n").asInt()); + } + assertEquals(List.of(1, 2, 3, 4, 5), seen, "iterateItems should yield all items in order"); + + // The total cap trims the tail: limit=3 over 5 items with page size 2 yields exactly 3. + List capped = new ArrayList<>(); + Iterator cappedIt = + dataset.iterateItems(new DatasetListItemsOptions().limit(3L), 2L); + while (cappedIt.hasNext()) { + capped.add(cappedIt.next().get("n").asInt()); + } + assertEquals(List.of(1, 2, 3), capped); + } finally { + client.dataset(ds.getId()).delete(); + } + } + + @Test + void iterateKeyValueStoreKeys() { + ApifyClient client = requireClient(); + KeyValueStore store = client.keyValueStores().getOrCreate(uniqueName("it-kvs-keys")); + try { + KeyValueStoreClient kvs = client.keyValueStore(store.getId()); + Set keys = new HashSet<>(); + for (int i = 0; i < 5; i++) { + String key = "key-" + i; + kvs.setRecordJson(key, Map.of("i", i)); + keys.add(key); + } + + Set seen = new HashSet<>(); + Iterator it = kvs.iterateKeys(new ListKeysOptions()); + while (it.hasNext()) { + seen.add(it.next().getKey()); + } + assertTrue(seen.containsAll(keys), "iterateKeys should yield every key; saw " + seen); + + // The limit caps the total number of keys yielded. + int count = 0; + Iterator capped = kvs.iterateKeys(new ListKeysOptions().limit(3L)); + while (capped.hasNext()) { + capped.next(); + count++; + } + assertEquals(3, count, "limit should cap the keys yielded"); + } finally { + client.keyValueStore(store.getId()).delete(); + } + } + + @Test + void iterateDatasets() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 3; i++) { + ids.add(client.datasets().getOrCreate(uniqueName("it-ds-" + i)).getId()); + } + // Page size 1 forces the offset iterator across multiple pages; desc puts the fresh ones + // first. + Iterator it = client.datasets().iterate(new StorageListOptions().desc(true), 1L); + assertTrue(findsAll(it, Dataset::getId, ids), "iterate should find every created dataset"); + } finally { + for (String id : ids) { + client.dataset(id).delete(); + } + } + } + + @Test + void iterateKeyValueStores() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add(client.keyValueStores().getOrCreate(uniqueName("it-kvs-" + i)).getId()); + } + Iterator it = + client.keyValueStores().iterate(new StorageListOptions().desc(true), 1L); + assertTrue(findsAll(it, KeyValueStore::getId, ids)); + } finally { + for (String id : ids) { + client.keyValueStore(id).delete(); + } + } + } + + @Test + void iterateRequestQueues() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add(client.requestQueues().getOrCreate(uniqueName("it-rq-" + i)).getId()); + } + Iterator it = + client.requestQueues().iterate(new StorageListOptions().desc(true), 1L); + assertTrue(findsAll(it, RequestQueue::getId, ids)); + } finally { + for (String id : ids) { + client.requestQueue(id).delete(); + } + } + } + + @Test + void iterateTasks() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add( + client.tasks().create(TaskIntegrationTest.taskDef(uniqueName("it-task-" + i))).getId()); + } + Iterator it = client.tasks().iterate(new ListOptions().desc(true), 1L); + assertTrue(findsAll(it, Task::getId, ids)); + } finally { + for (String id : ids) { + client.task(id).delete(); + } + } + } + + @Test + void iterateSchedules() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add( + client + .schedules() + .create(ScheduleIntegrationTest.scheduleDef(uniqueName("it-sch-" + i))) + .getId()); + } + Iterator it = client.schedules().iterate(new ListOptions().desc(true), 1L); + assertTrue(findsAll(it, Schedule::getId, ids)); + } finally { + for (String id : ids) { + client.schedule(id).delete(); + } + } + } + + @Test + void iterateWebhooks() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add( + client + .webhooks() + .create(WebhookIntegrationTest.webhookDef("https://example.com/it-wh-" + i)) + .getId()); + } + Iterator it = client.webhooks().iterate(new ListOptions().desc(true), 1L); + assertTrue(findsAll(it, Webhook::getId, ids)); + } finally { + for (String id : ids) { + client.webhook(id).delete(); + } + } + } + + @Test + void iterateActors() { + ApifyClient client = requireClient(); + Set ids = new HashSet<>(); + try { + for (int i = 0; i < 2; i++) { + ids.add( + client + .actors() + .create(ActorIntegrationTest.minimalActor(uniqueName("it-act-" + i))) + .getId()); + } + // Restrict to the current user's Actors so iteration finds the freshly-created ones quickly. + Iterator it = client.actors().iterate(new ActorListOptions().my(true).desc(true), 1L); + assertTrue(findsAll(it, Actor::getId, ids)); + } finally { + for (String id : ids) { + client.actor(id).delete(); + } + } + } + + @Test + void iterateActorVersionsAndEnvVars() { + ApifyClient client = requireClient(); + Actor actor = client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("it-ver"))); + try { + var actorClient = client.actor(actor.getId()); + // The minimal Actor ships with version 0.0, so iteration must yield at least one version. + Iterator versions = + actorClient.versions().iterate(new ListOptions(), 1L); + assertTrue(versions.hasNext(), "expected at least the initial version"); + + var envVars = actorClient.version("0.0").envVars(); + envVars.create(new ActorEnvVar("IT_VAR_A", "a")); + envVars.create(new ActorEnvVar("IT_VAR_B", "b")); + Set seen = new HashSet<>(); + Iterator it = envVars.iterate(); + while (it.hasNext()) { + seen.add(it.next().getName()); + } + assertTrue(seen.contains("IT_VAR_A") && seen.contains("IT_VAR_B"), "saw " + seen); + } finally { + client.actor(actor.getId()).delete(); + } + } + + @Test + void iterateBuildsBounded() { + ApifyClient client = requireClient(); + // Builds require building an Actor (expensive); assert a bounded slice iterates cleanly. + Iterator it = client.builds().iterate(new ListOptions().limit(5L), 2L); + int count = 0; + while (it.hasNext()) { + assertTrue(it.next().getId() != null); + count++; + } + assertTrue(count <= 5, "the total-cap limit must bound iteration; got " + count); + } + + @Test + void iterateRunsBounded() { + ApifyClient client = requireClient(); + Iterator it = + client.runs().iterate(new ListOptions().limit(5L), new RunListOptions(), 2L); + int count = 0; + while (it.hasNext()) { + assertTrue(it.next().getId() != null); + count++; + } + assertTrue(count <= 5); + } + + @Test + void iterateWebhookDispatchesBounded() { + ApifyClient client = requireClient(); + Iterator it = + client.webhookDispatches().iterate(new ListOptions().limit(5L), 2L); + int count = 0; + while (it.hasNext()) { + it.next(); + count++; + } + assertTrue(count <= 5); + } +} diff --git a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java index 08d6abd..66a19ce 100644 --- a/src/test/java/com/apify/client/integration/StoreIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/StoreIntegrationTest.java @@ -20,7 +20,7 @@ void listStore() { @Test void iterateStore() { ApifyClient client = requireClient(); - Iterator it = client.store().iterate(new StoreListOptions().limit(5L)); + Iterator it = client.store().iterate(new StoreListOptions().limit(20L), 5L); int count = 0; while (count < 12 && it.hasNext()) { ActorStoreListItem item = it.next(); From e365533f1ee439506964fbaf931b2933bbd5296b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 23:36:14 +0000 Subject: [PATCH 04/37] fix: terminate iteration on empty page, not short page or total The API clamps over-large page sizes and some endpoints under-report (dataset items total=0), so short-page and total-based termination both truncate results. Advance offset by actual count and stop only on an empty page or the caller cap. Hardens hermetic tests with server-clamp scenarios and a terminating empty page. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- cp.txt | 1 + .../com/apify/client/PaginatedIterator.java | 26 ++++---- .../apify/client/PaginatedIteratorTest.java | 63 ++++++++++++------- .../com/apify/client/ReviewFixesTest.java | 14 +++-- 4 files changed, 64 insertions(+), 40 deletions(-) create mode 100644 cp.txt diff --git a/cp.txt b/cp.txt new file mode 100644 index 0000000..b1e4eb4 --- /dev/null +++ b/cp.txt @@ -0,0 +1 @@ +/root/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.17.2/jackson-databind-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.17.2/jackson-annotations-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.17.2/jackson-core-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.17.2/jackson-datatype-jsr310-2.17.2.jar:/root/.m2/repository/com/aayushatharva/brotli4j/brotli4j/1.23.0/brotli4j-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/service/1.23.0/service-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-linux-x86_64/1.23.0/native-linux-x86_64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-linux-aarch64/1.23.0/native-linux-aarch64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-osx-x86_64/1.23.0/native-osx-x86_64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-osx-aarch64/1.23.0/native-osx-aarch64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-windows-x86_64/1.23.0/native-windows-x86_64-1.23.0.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter/5.10.2/junit-jupiter-5.10.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-api/5.10.2/junit-jupiter-api-5.10.2.jar:/root/.m2/repository/org/opentest4j/opentest4j/1.3.0/opentest4j-1.3.0.jar:/root/.m2/repository/org/junit/platform/junit-platform-commons/1.10.2/junit-platform-commons-1.10.2.jar:/root/.m2/repository/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-params/5.10.2/junit-jupiter-params-5.10.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-engine/5.10.2/junit-jupiter-engine-5.10.2.jar:/root/.m2/repository/org/junit/platform/junit-platform-engine/1.10.2/junit-platform-engine-1.10.2.jar \ No newline at end of file diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java index a74f34e..2cda88e 100644 --- a/src/main/java/com/apify/client/PaginatedIterator.java +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -9,7 +9,8 @@ * time. Internal reusable engine shared by every collection client's {@code iterate} method, so the * paging arithmetic lives in one place (DRY). * - *

The end-user behaviour matches the reference JS client's {@code _listPaginatedFromCallback}: + *

Behaviour, mirroring the end-user contract of the reference JS client's iterable {@code + * list()}: * *

    *
  • {@code totalLimit} caps the total number of items yielded across all pages ({@code @@ -19,12 +20,13 @@ * overshot. *
* - *

Iteration stops when the cap is reached, the API returns an empty page, or a page comes back - * shorter than requested (the last page). The short-page check — rather than relying solely on the - * reported {@code total} — makes iteration robust to a momentarily under-reported {@code total} - * (the count can lag right after a write), which would otherwise truncate a full page. The reported - * total is only consulted to terminate the "unbounded, server-chosen page size" case, where no page - * size is known to compare against. + *

Each page advances the offset by the number of items actually returned, and iteration stops + * only when the cap is reached or the API returns an empty page. It deliberately does not + * stop on a short page or a reported {@code total}: the API clamps an over-large requested page + * size to its own maximum (so a "short" page is common mid-collection), and some endpoints report a + * {@code total} of {@code 0} or a value that lags right after a write (e.g. dataset items) — either + * signal would truncate iteration. Terminating on an empty page costs one extra request at the end + * but yields the complete result in every case. */ final class PaginatedIterator implements Iterator { @@ -79,13 +81,9 @@ private void fetchNextPage() { int count = buffer.size(); offset += count; yielded += count; - - boolean capReached = totalLimit != null && yielded >= totalLimit; - boolean shortPage = pageLimit != null && count < pageLimit; - // With no explicit page size the API returns its default page, so there is nothing to compare a - // short page against — fall back to the reported total to detect the end of the collection. - boolean totalReached = pageLimit == null && offset >= page.getTotal(); - if (count == 0 || capReached || shortPage || totalReached) { + // Stop on the caller's cap or an empty page; never on a short page (the API clamps large page + // sizes) or the reported total (unreliable on some endpoints — see class doc). + if (count == 0 || (totalLimit != null && yielded >= totalLimit)) { exhausted = true; } } diff --git a/src/test/java/com/apify/client/PaginatedIteratorTest.java b/src/test/java/com/apify/client/PaginatedIteratorTest.java index 53da9ed..b80f72d 100644 --- a/src/test/java/com/apify/client/PaginatedIteratorTest.java +++ b/src/test/java/com/apify/client/PaginatedIteratorTest.java @@ -12,37 +12,43 @@ /** * Hermetic (token-free) tests for the offset/limit iteration engine. They drive {@link * PaginatedIterator} with a stub page fetcher over a synthetic collection, pinning the total-cap / - * chunk-size arithmetic and termination that mirror the reference JS {@code - * _listPaginatedFromCallback} — without any network or {@code APIFY_TOKEN}. + * chunk-size arithmetic, server-side page-size clamping, and termination — without any network or + * {@code APIFY_TOKEN}. */ class PaginatedIteratorTest { - /** A fake paginated endpoint over integers {@code [0, available)} reporting {@code total}. */ + /** + * A fake paginated endpoint over integers {@code [0, available)}. Like the real API, it clamps a + * requested page size (or an unset one) to {@code maxPageSize} ({@code 0} = no clamp). + */ private static final class StubFetcher implements PaginatedIterator.PageFetcher { - final long total; final long available; + final long maxPageSize; final List requests = new ArrayList<>(); - StubFetcher(long total) { - this(total, total); + StubFetcher(long available) { + this(available, 0); } - StubFetcher(long total, long available) { - this.total = total; + StubFetcher(long available, long maxPageSize) { this.available = available; + this.maxPageSize = maxPageSize; } @Override public PaginationList fetch(long offset, Long limit) { requests.add(new long[] {offset, limit == null ? -1 : limit}); - long take = limit == null ? Long.MAX_VALUE : limit; + long requested = limit == null ? Long.MAX_VALUE : limit; + long take = maxPageSize > 0 ? Math.min(requested, maxPageSize) : requested; List items = new ArrayList<>(); for (long i = offset; i < available && items.size() < take; i++) { items.add((int) i); } PaginationList page = new PaginationList<>(); page.setItems(items); - page.setTotal(total); + // Some endpoints (e.g. dataset items) report a total of 0 or a lagging value; the engine must + // not depend on it, so the stub deliberately reports an unhelpful total. + page.setTotal(0); page.setOffset(offset); page.setCount(items.size()); return page; @@ -62,7 +68,8 @@ void totalCapTrimsLastPage() { StubFetcher f = new StubFetcher(10); List got = drain(new PaginatedIterator<>(3L, 2L, null, f)); assertEquals(List.of(0, 1, 2), got, "limit=3 caps the total yielded across pages"); - // First page requests min(cap,chunk)=2; second page requests min(remaining=1,chunk=2)=1. + // First page requests min(cap,chunk)=2; second requests min(remaining=1,chunk=2)=1; the cap is + // reached exactly, so no trailing empty request is made. assertEquals(2, f.requests.size()); assertEquals(0L, f.requests.get(0)[0]); assertEquals(2L, f.requests.get(0)[1]); @@ -71,32 +78,44 @@ void totalCapTrimsLastPage() { } @Test - void chunkSizePagesAll() { + void chunkSizePagesAllThenStopsOnEmptyPage() { StubFetcher f = new StubFetcher(5); List got = drain(new PaginatedIterator<>(null, 2L, null, f)); assertEquals(List.of(0, 1, 2, 3, 4), got); - assertEquals(3, f.requests.size(), "5 items / page size 2 => 3 pages"); + // 5 items / page size 2 => pages of 2,2,1 then a trailing empty page confirms the end. + assertEquals(4, f.requests.size()); + assertEquals(5L, f.requests.get(3)[0], "trailing request pages past the last item"); + assertEquals(2L, f.requests.get(3)[1], "each page still requests the chunk size"); } @Test - void noCapNoChunkUsesServerDefault() { - StubFetcher f = new StubFetcher(4); - List got = drain(new PaginatedIterator<>(null, null, null, f)); - assertEquals(List.of(0, 1, 2, 3), got); - assertEquals(1, f.requests.size()); - assertEquals(-1L, f.requests.get(0)[1], "no cap and no chunk => null limit (server default)"); + void serverClampsLargePageSizeSoShortPageIsNotTheEnd() { + // Server caps every page at 2 items; the caller asks for far more. A page shorter than the + // request must NOT be treated as end-of-collection. + StubFetcher f = new StubFetcher(5, 2); + List got = drain(new PaginatedIterator<>(null, 100L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got, "clamped short pages must keep paging to the end"); + } + + @Test + void capIsHonoredEvenWhenServerClampsPages() { + StubFetcher f = new StubFetcher(100, 2); + List got = drain(new PaginatedIterator<>(5L, 100L, null, f)); + assertEquals(List.of(0, 1, 2, 3, 4), got, "the total cap holds despite server page clamping"); } @Test - void capLargerThanTotalYieldsAll() { + void noChunkUsesServerDefaultAndPagesToEmpty() { StubFetcher f = new StubFetcher(4); - assertEquals(List.of(0, 1, 2, 3), drain(new PaginatedIterator<>(100L, null, null, f))); + List got = drain(new PaginatedIterator<>(null, null, null, f)); + assertEquals(List.of(0, 1, 2, 3), got); + assertEquals(-1L, f.requests.get(0)[1], "no cap and no chunk => null limit (server default)"); } @Test void startOffsetIsHonored() { StubFetcher f = new StubFetcher(10); - List got = drain(new PaginatedIterator<>(null, null, 7L, f)); + List got = drain(new PaginatedIterator<>(null, 5L, 7L, f)); assertEquals(List.of(7, 8, 9), got, "iteration starts at the requested offset"); assertEquals(7L, f.requests.get(0)[0]); } diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ReviewFixesTest.java index 0575db4..9bebbaf 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ReviewFixesTest.java @@ -283,8 +283,14 @@ void storeIterateWalksMultiplePages() { "{\"data\":{\"items\":[{},{}],\"total\":3,\"offset\":0,\"limit\":2,\"count\":2}}"), MockBackend.ok( 200, - "{\"data\":{\"items\":[{}],\"total\":3,\"offset\":2,\"limit\":2,\"count\":1}}"))); - // No total cap; page size 2 drives paging across the reported total of 3. + "{\"data\":{\"items\":[{}],\"total\":3,\"offset\":2,\"limit\":2,\"count\":1}}"), + // Trailing empty page: the iterator stops on an empty page (it does not trust the + // reported total, which some endpoints under-report), so a final empty page is + // required to terminate an uncapped walk. + MockBackend.ok( + 200, + "{\"data\":{\"items\":[],\"total\":3,\"offset\":3,\"limit\":2,\"count\":0}}"))); + // No total cap; page size 2 drives paging until the empty page. java.util.Iterator it = client(backend).store().iterate(new StoreListOptions(), 2L); int count = 0; @@ -292,8 +298,8 @@ void storeIterateWalksMultiplePages() { it.next(); count++; } - assertEquals(3, count, "iteration should walk across both pages"); - assertEquals(2, backend.calls, "one API call per page"); + assertEquals(3, count, "iteration should walk across both non-empty pages"); + assertEquals(3, backend.calls, "two data pages plus the terminating empty page"); } @Test From 3a556600f76e7982283c1c9394ae2667a9ab5be3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 23:55:18 +0000 Subject: [PATCH 05/37] chore: address review round 4 (hygiene, tests, docs) Remove stray cp.txt + gitignore it; add hermetic KeyIteratorTest; pattern-based offline CI test selection; client-side cap trim on both iterators; doc fixes (nested webhook list+iterate, paginateRequests exception, iterateKeys limit meaning, iterateItems filter caveat, normalized null-semantics, quick-start pom.xml, ActorRun statuses, write return types). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .github/workflows/java-integration-tests.yml | 7 +- .gitignore | 3 + README.md | 28 ++++++- cp.txt | 1 - docs/actors.md | 2 +- docs/builds.md | 2 +- docs/runs.md | 5 +- docs/schedules.md | 2 +- docs/storages.md | 30 ++++--- docs/tasks.md | 4 +- docs/webhooks.md | 12 +-- .../java/com/apify/client/DatasetClient.java | 5 ++ .../com/apify/client/KeyValueStoreClient.java | 5 ++ .../com/apify/client/PaginatedIterator.java | 6 ++ .../com/apify/client/KeyIteratorTest.java | 80 +++++++++++++++++++ 15 files changed, 164 insertions(+), 28 deletions(-) delete mode 100644 cp.txt create mode 100644 src/test/java/com/apify/client/KeyIteratorTest.java diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index 3257ab1..49ac3cc 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -45,9 +45,12 @@ jobs: - name: Static analysis (SpotBugs) run: mvn -B -DskipTests compile spotbugs:check - # Offline unit tests (mock HTTP backend): prove the retry/error/signature logic without the API. + # Offline unit tests (mock HTTP backend): prove the retry/error/signature/pagination logic + # without the API. Selected by pattern — every hermetic test in the base package runs, and the + # token-gated integration/example/doc-snippet suites are excluded — so new hermetic tests are + # covered here automatically without editing this list. - name: Unit tests - run: mvn -B test -Dtest='UnitHttpTest,ReviewFixesTest,ClientMetaTest,SignatureTest,ConfigTest,CompressionTest' -DfailIfNoTests=true + run: mvn -B test -Dtest='!*IntegrationTest,!ExamplesTest,!DocSnippetsTest' -DfailIfNoTests=true # Fail fast if the integration-test secret is missing or empty. Without this guard the # integration tests silently "pass" (JUnit assumptions skip them when APIFY_TOKEN is unset), diff --git a/.gitignore b/.gitignore index 07ad7f2..4f958b3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,9 @@ replay_pid* # Maven build output target/ +# Local build-classpath dumps (e.g. from `mvn dependency:build-classpath`) +cp.txt + # IDE .idea/ *.iml diff --git a/README.md b/README.md index 5c6fd27..4c6d32f 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,30 @@ implementation 'com.apify:apify-client:0.2.0' ## Quick start -A complete, copy-pasteable first program (save as `HelloApify.java`). Populate a `lib/` directory -with the client and its runtime dependencies (from a directory whose `pom.xml` declares the -dependency shown in [Installation](#installation) above), then compile and run against the JVM's -`lib/*` classpath wildcard — quote it so the shell does not expand it: +A complete, copy-pasteable first program (save as `HelloApify.java`). First scaffold a minimal +`pom.xml` next to it so Maven can resolve the client and its runtime dependencies: + +```xml + + 4.0.0 + com.example + hello-apify + 1.0.0 + + 17 + + + + com.apify + apify-client + 0.2.0 + + + +``` + +Then populate a `lib/` directory with the client and its runtime dependencies, and compile and run +against the JVM's `lib/*` classpath wildcard — quote it so the shell does not expand it: ```bash # 1. Collect apify-client and its runtime dependencies (Jackson, brotli4j codecs, …) into lib/. diff --git a/cp.txt b/cp.txt deleted file mode 100644 index b1e4eb4..0000000 --- a/cp.txt +++ /dev/null @@ -1 +0,0 @@ -/root/.m2/repository/com/fasterxml/jackson/core/jackson-databind/2.17.2/jackson-databind-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/core/jackson-annotations/2.17.2/jackson-annotations-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/core/jackson-core/2.17.2/jackson-core-2.17.2.jar:/root/.m2/repository/com/fasterxml/jackson/datatype/jackson-datatype-jsr310/2.17.2/jackson-datatype-jsr310-2.17.2.jar:/root/.m2/repository/com/aayushatharva/brotli4j/brotli4j/1.23.0/brotli4j-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/service/1.23.0/service-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-linux-x86_64/1.23.0/native-linux-x86_64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-linux-aarch64/1.23.0/native-linux-aarch64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-osx-x86_64/1.23.0/native-osx-x86_64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-osx-aarch64/1.23.0/native-osx-aarch64-1.23.0.jar:/root/.m2/repository/com/aayushatharva/brotli4j/native-windows-x86_64/1.23.0/native-windows-x86_64-1.23.0.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter/5.10.2/junit-jupiter-5.10.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-api/5.10.2/junit-jupiter-api-5.10.2.jar:/root/.m2/repository/org/opentest4j/opentest4j/1.3.0/opentest4j-1.3.0.jar:/root/.m2/repository/org/junit/platform/junit-platform-commons/1.10.2/junit-platform-commons-1.10.2.jar:/root/.m2/repository/org/apiguardian/apiguardian-api/1.1.2/apiguardian-api-1.1.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-params/5.10.2/junit-jupiter-params-5.10.2.jar:/root/.m2/repository/org/junit/jupiter/junit-jupiter-engine/5.10.2/junit-jupiter-engine-5.10.2.jar:/root/.m2/repository/org/junit/platform/junit-platform-engine/1.10.2/junit-platform-engine-1.10.2.jar \ No newline at end of file diff --git a/docs/actors.md b/docs/actors.md index 95d6839..68c6efd 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -53,7 +53,7 @@ Actor created = client.actors().create(Map.of( | `defaultBuild(Long waitForFinish)` | Resolve the default build. Returns `BuildClient`. | | `lastRun(String status)` / `lastRun(LastRunOptions)` | A `RunClient` for the last run. | | `builds()` / `runs()` / `versions()` | Nested collection clients. | -| `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, list only). | +| `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, `list` + `iterate`, no `create`). | | `version(String)` | An `ActorVersionClient`. | `ActorStartOptions` fields (all optional): `build`, `memoryMbytes`, `timeoutSecs`, `waitForFinish`, diff --git a/docs/builds.md b/docs/builds.md index 7930888..2e6417d 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -8,7 +8,7 @@ builds) and a single build with `client.build(id)`. | Method | Description | |---|---| | `list(ListOptions)` | List builds. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | ## `BuildClient` diff --git a/docs/runs.md b/docs/runs.md index e1a17b5..7d75bb4 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -8,7 +8,7 @@ Access the run collection with `client.runs()` (or `client.actor(id).runs()` / | Method | Description | |---|---| | `list(ListOptions, RunListOptions)` | List runs. Returns `PaginationList`. | -| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | `RunListOptions` adds `status(List)` (e.g. `SUCCEEDED`, `RUNNING`; sent comma-separated) and, for Actor/task-scoped collections, `startedAfter(String)` / `startedBefore(String)` (ISO-8601). @@ -46,6 +46,9 @@ To set the current run's status message from inside an Actor, use the top-level `ActorRun` fields include `getId()`, `getActId()`, `getUserId()`, `getStatus()`, `getStatusMessage()`, `getStartedAt()`, `getFinishedAt()`, `getBuildId()`, `getDefaultDatasetId()`, `getDefaultKeyValueStoreId()`, `getDefaultRequestQueueId()`, `getContainerUrl()`, plus `isTerminal()`. +The status is one of `READY`, `RUNNING`, `SUCCEEDED`, `FAILED`, `TIMING-OUT`, `TIMED-OUT`, +`ABORTING`, `ABORTED`; `isTerminal()` is true for the finished states (`SUCCEEDED`, `FAILED`, +`TIMED-OUT`, `ABORTED`). `RunChargeOptions` (constructed with the required event name) uses plain values: `count(Long)` and `idempotencyKey(String)` — the latter is auto-generated when unset so a transport-retried charge is diff --git a/docs/schedules.md b/docs/schedules.md index 6512fa0..d57a30b 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -8,7 +8,7 @@ Schedules automatically start Actor or task runs at specified times. Access the | Method | Description | |---|---| | `list(ListOptions)` | List schedules. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `create(Object)` | Create a schedule from a JSON-serializable definition. Returns `Schedule`. | The `actions` array describes what the schedule runs (e.g. a `RUN_ACTOR` action): diff --git a/docs/storages.md b/docs/storages.md index dbc0af9..f244e86 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -24,7 +24,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | Method | Description | |---|---| | `list(StorageListOptions)` | List datasets. Returns `PaginationList`. | -| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `getOrCreate(String name)` | Get or create a named dataset (empty name → unnamed). Returns `Dataset`. | | `getOrCreate(String name, Object schema)` | As above, sending a creation-time dataset `schema` when a new dataset is created. Returns `Dataset`. | @@ -37,10 +37,17 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | -| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; `limit` caps the total, `chunkSize` sets the page size. | +| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | + +> **Server-side item filters and iteration.** The dataset-items endpoint applies `offset`/`limit` to +> the raw items and then drops those removed by a server-side filter (`skipEmpty`, `skipHidden`, +> `clean`, `simplified`), so a page can contain fewer items than requested. Because `iterateItems` +> advances the offset by the number of items actually returned, combining it with those filters over a +> multi-page dataset can make page windows overlap and repeat items. Prefer paging without server-side +> item filters when iterating, or de-duplicate on the client. | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | -| `pushItems(Object)` | Push a single item or a list of items. | +| `pushItems(Object)` | Push a single item or a list of items. No return value. | | `getStatistics()` | Dataset statistics. Returns `Optional`. | | `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. | @@ -87,13 +94,13 @@ datasets. |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listKeys(ListKeysOptions)` | List keys. Returns `KeyValueStoreKeysPage`. | -| `iterateKeys(ListKeysOptions)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`); the options' `limit` caps the total number of keys yielded. | +| `iterateKeys(ListKeysOptions)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null` = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. Key iteration is cursor-based, so it has no separate `chunkSize`. | | `recordExists(String key)` | Whether a record exists. | | `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Returns `Optional`. | -| `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. | -| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`timeoutSecs`, `doNotRetryTimeouts`). | -| `setRecordJson(String key, Object value)` | Store JSON. | -| `deleteRecord(String key)` | Delete a record. | +| `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. No return value. | +| `setRecord(String key, byte[] value, String contentType, SetRecordOptions)` | Store raw bytes with write options (`timeoutSecs`, `doNotRetryTimeouts`). No return value. | +| `setRecordJson(String key, Object value)` | Store JSON. No return value. | +| `deleteRecord(String key)` | Delete a record. No return value. | | `getRecordPublicUrl(String key)` | A public (optionally signed) record URL. | | `createKeysPublicUrl(Long expiresInSecs)` | A public (optionally signed) key-list URL. | | `createKeysPublicUrl(ListKeysOptions, Long expiresInSecs)` | As above, forwarding key-listing filters (`limit`, `prefix`, `collection`, `exclusiveStartKey`). | @@ -144,7 +151,12 @@ as for datasets. | `prolongRequestLock(String id, long lockSecs, boolean forefront)` | Extend a lock. Returns `JsonNode`. | | `deleteRequestLock(String id, boolean forefront)` | Release a lock. No return value. | | `unlockRequests()` | Release all the client's locks. Returns `JsonNode`. | -| `paginateRequests(Long pageLimit)` | A lazy `Iterator` over all requests. | +| `paginateRequests(Long pageLimit)` | A lazy `Iterator` over all requests, paging with the queue's forward cursor. | + +> **Naming exception.** Request-queue *requests* are iterated with `paginateRequests(Long pageLimit)` +> — not an `iterate(...)` method — because the request-queue listing is cursor-based rather than +> offset/limit. Its single argument is the per-request page size; there is no total-items cap. Every +> other resource uses the `iterate`/`iterateItems`/`iterateKeys` family. ```java RequestQueue rq = client.requestQueues().getOrCreate("my-queue"); diff --git a/docs/tasks.md b/docs/tasks.md index 61d6f60..fb0b94d 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -8,7 +8,7 @@ Tasks are pre-configured Actor runs with stored input. Access the task collectio | Method | Description | |---|---| | `list(ListOptions)` | List tasks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `create(Object)` | Create a task from a JSON-serializable definition. Returns `Task`. | ```java @@ -30,7 +30,7 @@ Task task = client.tasks().create(Map.of( | `updateInput(Object)` | Replace the stored input. Returns `JsonNode`. | | `lastRun(String status)` / `lastRun(LastRunOptions)` | A `RunClient` for the last run (see [`LastRunOptions`](actors.md#actorclient)). | | `runs()` | Nested run collection client. | -| `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, list only). | +| `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, `list` + `iterate`, no `create`). | `TaskStartOptions` mirrors `ActorStartOptions` but omits the Actor-only `contentType` and `forcePermissionLevel`: `build`, `memoryMbytes`, `timeoutSecs`, `waitForFinish`, `maxItems`, diff --git a/docs/webhooks.md b/docs/webhooks.md index d2a69be..e2c63b8 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -12,17 +12,17 @@ The account-wide collection supports both listing and creation. | Method | Description | |---|---| | `list(ListOptions)` | List webhooks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; `limit` caps the total, `chunkSize` sets the page size. Also available on the read-only nested collections. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). Also available on the read-only nested collections. | | `create(Object)` | Create a webhook. Returns `Webhook`. | ### Nested webhook collections (read-only) `client.actor(id).webhooks()` and `client.task(id).webhooks()` return a -`NestedWebhookCollectionClient`. The Apify API only supports **listing** webhooks on those nested +`NestedWebhookCollectionClient`. The Apify API only supports **reading** webhooks on those nested paths (`GET /v2/acts/{id}/webhooks`, `GET /v2/actor-tasks/{id}/webhooks`), so this read-only type -exposes `list(ListOptions)` only — it has no `create(...)`. To create a webhook targeting a specific -Actor or task, use `client.webhooks().create(...)` and set the Actor/task in the webhook's -`condition`. +exposes `list(ListOptions)` and `iterate(ListOptions, Long chunkSize)` — it has no `create(...)`. To +create a webhook targeting a specific Actor or task, use `client.webhooks().create(...)` and set the +Actor/task in the webhook's `condition`. A webhook definition supplies `eventTypes` (a list of event-type strings such as `ACTOR.RUN.SUCCEEDED`, `ACTOR.RUN.FAILED`, `ACTOR.RUN.ABORTED`, `ACTOR.RUN.TIMED_OUT`, @@ -57,7 +57,7 @@ System.out.println(dispatch.getId()); | Method | Description | |---|---| | `webhookDispatches().list(ListOptions)` | List dispatches. Returns `PaginationList`. | -| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; `limit` caps the total, `chunkSize` sets the page size. | +| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `webhookDispatch(id).get()` | Fetch a dispatch. Returns `Optional`. | `WebhookDispatch` fields: `getId()`, `getWebhookId()`. diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 6b0eae5..7eb01b0 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -87,6 +87,11 @@ public Iterator iterateItems(DatasetListItemsOptions options, Long chu * fetching pages on demand. The options' {@code limit} caps the total number of items yielded * ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = server * default). + * + *

Note: server-side item filters ({@code skipEmpty}, {@code skipHidden}, {@code clean}, {@code + * simplified}) are applied after {@code offset}/{@code limit}, so a page can return fewer items + * than requested and paging windows may overlap — combining those filters with iteration can + * repeat items. Iterate without server-side item filters, or de-duplicate on the client. */ public Iterator iterateItems( DatasetListItemsOptions options, Long chunkSize, Class itemClass) { diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 5b8c630..b5a3a16 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -116,6 +116,11 @@ private void fetchPage() { pos = 0; cursor = page.getNextExclusiveStartKey(); if (remaining != null) { + // Defensively trim the last page to the cap in case the server returned more than + // requested. + if (buffer.size() > remaining) { + buffer = buffer.subList(0, remaining.intValue()); + } remaining -= buffer.size(); } // Stop when the page is empty, there is no next cursor, or the total cap is reached. diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java index 2cda88e..965ff5a 100644 --- a/src/main/java/com/apify/client/PaginatedIterator.java +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -81,6 +81,12 @@ private void fetchNextPage() { int count = buffer.size(); offset += count; yielded += count; + // Defensively trim the last page to the cap in case the server returned more than requested, so + // the caller never sees more than `totalLimit` items. + if (totalLimit != null && yielded > totalLimit) { + buffer = buffer.subList(0, count - (int) (yielded - totalLimit)); + yielded = totalLimit; + } // Stop on the caller's cap or an empty page; never on a short page (the API clamps large page // sizes) or the reported total (unreliable on some endpoints — see class doc). if (count == 0 || (totalLimit != null && yielded >= totalLimit)) { diff --git a/src/test/java/com/apify/client/KeyIteratorTest.java b/src/test/java/com/apify/client/KeyIteratorTest.java new file mode 100644 index 0000000..08b2c1c --- /dev/null +++ b/src/test/java/com/apify/client/KeyIteratorTest.java @@ -0,0 +1,80 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import org.junit.jupiter.api.Test; + +/** + * Hermetic (token-free) tests for the cursor-based key iterator ({@link + * KeyValueStoreClient#iterateKeys}), driven by a {@link MockBackend}: cursor chaining across pages, + * the {@code limit} total-cap, and empty-page / empty-cursor termination. + */ +class KeyIteratorTest { + + private static ApifyClient client(MockBackend backend) { + return ApifyClient.builder() + .token("test-token") + .httpBackend(backend) + .maxRetries(0) + .minDelayBetweenRetries(Duration.ofMillis(1)) + .build(); + } + + private static List keys(Iterator it) { + List out = new ArrayList<>(); + while (it.hasNext()) { + out.add(it.next().getKey()); + } + return out; + } + + @Test + void chainsAcrossPagesUsingCursor() { + MockBackend backend = + new MockBackend( + List.of( + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"a\",\"size\":1},{\"key\":\"b\",\"size\":1}]," + + "\"nextExclusiveStartKey\":\"b\",\"isTruncated\":true}}"), + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"c\",\"size\":1}]," + + "\"nextExclusiveStartKey\":null,\"isTruncated\":false}}"))); + List got = keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions())); + assertEquals(List.of("a", "b", "c"), got, "iterator should chain pages via the cursor"); + assertEquals(2, backend.calls, "stops once the next cursor is null"); + assertTrue(backend.lastUrl.contains("exclusiveStartKey=b"), backend.lastUrl); + } + + @Test + void limitCapsTotalKeysYielded() { + MockBackend backend = + MockBackend.ofConstant( + 200, + "{\"data\":{\"items\":[{\"key\":\"a\",\"size\":1},{\"key\":\"b\",\"size\":1}," + + "{\"key\":\"c\",\"size\":1}],\"nextExclusiveStartKey\":\"c\",\"isTruncated\":true}}"); + List got = + keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions().limit(2L))); + assertEquals(List.of("a", "b"), got, "limit caps the total keys yielded"); + assertEquals(1, backend.calls, "the cap is reached within the first page"); + assertTrue(backend.lastUrl.contains("limit=2"), backend.lastUrl); + } + + @Test + void stopsOnEmptyPage() { + MockBackend backend = + MockBackend.ofConstant( + 200, "{\"data\":{\"items\":[],\"nextExclusiveStartKey\":\"z\",\"isTruncated\":true}}"); + Iterator it = + client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions()); + assertFalse(it.hasNext(), "an empty page ends iteration even if a next cursor is reported"); + assertEquals(1, backend.calls); + } +} From 333151d6bdd78336039b36261e75b72ac9226bd7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 00:12:44 +0000 Subject: [PATCH 06/37] docs: fix split DatasetClient table; CI dedupe; comment reflow Move the iterateItems filter caveat blockquote after the DatasetClient table (it had split the table); run only *IntegrationTest in the integration CI step to avoid re-running hermetic tests; reflow a KVS comment. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .github/workflows/java-integration-tests.yml | 5 +++-- docs/storages.md | 8 ++++---- src/main/java/com/apify/client/KeyValueStoreClient.java | 5 ++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/java-integration-tests.yml b/.github/workflows/java-integration-tests.yml index 49ac3cc..e094fa8 100644 --- a/.github/workflows/java-integration-tests.yml +++ b/.github/workflows/java-integration-tests.yml @@ -68,8 +68,9 @@ jobs: env: # The integration-test token is stored as a repository secret. APIFY_TOKEN: ${{ secrets.APIFY_TOKEN }} - # Run every test except the example/doc-snippet harnesses (exercised by the step below). - run: mvn -B test -Dtest='!ExamplesTest,!DocSnippetsTest' -DfailIfNoTests=true + # Run only the live integration suites; the hermetic tests already ran in the offline step + # above and the example/doc-snippet harnesses run in the step below. + run: mvn -B test -Dtest='*IntegrationTest' -DfailIfNoTests=true # Standalone CI step that verifies the documentation examples actually work end-to-end against # the live API (ExamplesTest runs each example's main), and that every in-documentation Java diff --git a/docs/storages.md b/docs/storages.md index f244e86..37d4fff 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -39,6 +39,10 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | | `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | | `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | +| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | +| `pushItems(Object)` | Push a single item or a list of items. No return value. | +| `getStatistics()` | Dataset statistics. Returns `Optional`. | +| `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. | > **Server-side item filters and iteration.** The dataset-items endpoint applies `offset`/`limit` to > the raw items and then drops those removed by a server-side filter (`skipEmpty`, `skipHidden`, @@ -46,10 +50,6 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and > advances the offset by the number of items actually returned, combining it with those filters over a > multi-page dataset can make page windows overlap and repeat items. Prefer paging without server-side > item filters when iterating, or de-duplicate on the client. -| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | -| `pushItems(Object)` | Push a single item or a list of items. No return value. | -| `getStatistics()` | Dataset statistics. Returns `Optional`. | -| `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. | ```java Dataset ds = client.datasets().getOrCreate("my-dataset"); diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index b5a3a16..8e4514b 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -104,9 +104,8 @@ public KeyValueStoreKey next() { private void fetchPage() { QueryParams params = new QueryParams(); - // The API caps the page size; requesting `remaining` keeps the last page from overshooting - // the - // caller's total cap. A null limit lets the server choose its default page size. + // Requesting `remaining` keeps the last page from overshooting the caller's total cap; a null + // limit lets the server choose its default page size. params.addLong("limit", remaining); params.addString("exclusiveStartKey", cursor); options.applyFilters(params); From 0266f6edbb28b1a97de677de3c9aa1d3fa8d990d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 00:24:03 +0000 Subject: [PATCH 07/37] docs: warn iterateItems can stop early under server-side filters Extend the dataset-items iteration caveat (docs + Javadoc) to cover silent early-termination when an offset window is fully filtered out, not just the overlap/repeat mode. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/storages.md | 7 +++++-- src/main/java/com/apify/client/DatasetClient.java | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/storages.md b/docs/storages.md index 37d4fff..7856af3 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -48,8 +48,11 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and > the raw items and then drops those removed by a server-side filter (`skipEmpty`, `skipHidden`, > `clean`, `simplified`), so a page can contain fewer items than requested. Because `iterateItems` > advances the offset by the number of items actually returned, combining it with those filters over a -> multi-page dataset can make page windows overlap and repeat items. Prefer paging without server-side -> item filters when iterating, or de-duplicate on the client. +> multi-page dataset has two failure modes: page windows can overlap and **repeat items**, and — more +> severely — if an entire offset window is filtered out the endpoint returns an empty page, which the +> iterator treats as the end, so iteration **stops early and silently skips the remaining data** (an +> all-filtered first page yields nothing at all). Prefer paging without server-side item filters when +> iterating, or fetch pages explicitly with `listItems` and filter client-side. ```java Dataset ds = client.datasets().getOrCreate("my-dataset"); diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 7eb01b0..e409222 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -90,8 +90,10 @@ public Iterator iterateItems(DatasetListItemsOptions options, Long chu * *

Note: server-side item filters ({@code skipEmpty}, {@code skipHidden}, {@code clean}, {@code * simplified}) are applied after {@code offset}/{@code limit}, so a page can return fewer items - * than requested and paging windows may overlap — combining those filters with iteration can - * repeat items. Iterate without server-side item filters, or de-duplicate on the client. + * than requested. Combining those filters with iteration can repeat items (overlapping windows) + * and, if a whole offset window is filtered out, the endpoint returns an empty page which ends + * iteration early — silently skipping the remaining items. Iterate without server-side item + * filters, or page explicitly with {@link #listItems} and filter client-side. */ public Iterator iterateItems( DatasetListItemsOptions options, Long chunkSize, Class itemClass) { From 38f24e05cf866abb68b7d9ee35b82ca5c9a869a1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 00:37:15 +0000 Subject: [PATCH 08/37] chore: address review round 6 (publish gate, iterateKeys chunkSize, tests, docs) Publish workflow now selects hermetic tests by pattern (auto-includes new ones); add iterateKeys(options, chunkSize) overload for parity; hermetic test for the overshoot-trim branch; README quick start reads the token from the environment. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .github/workflows/java-publish.yml | 5 +++- README.md | 3 ++- docs/storages.md | 2 +- .../com/apify/client/KeyValueStoreClient.java | 26 +++++++++++++++---- .../apify/client/PaginatedIteratorTest.java | 16 ++++++++++++ 5 files changed, 44 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 4ad8af1..e70e984 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -62,8 +62,11 @@ jobs: - name: Static analysis (SpotBugs) run: mvn -B -DskipTests compile spotbugs:check + # Same pattern-based selection as the integration workflow's offline gate: run every hermetic + # test and exclude the token-gated integration/example/doc-snippet suites, so new hermetic + # tests are gated on release automatically without editing this list. - name: Unit tests - run: mvn -B test -Dtest='UnitHttpTest,ReviewFixesTest,ClientMetaTest,SignatureTest,ConfigTest' -DfailIfNoTests=true + run: mvn -B test -Dtest='!*IntegrationTest,!ExamplesTest,!DocSnippetsTest' -DfailIfNoTests=true - name: Resolve version from pom.xml id: version diff --git a/README.md b/README.md index 4c6d32f..86c6f04 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,8 @@ import com.apify.client.ActorStartOptions; class HelloApify { public static void main(String[] args) { - ApifyClient client = ApifyClient.create("my-api-token"); + // Your API token from https://console.apify.com/settings/integrations + ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); ActorRun run = client.actor("apify/hello-world").call(null, new ActorStartOptions(), 120L); System.out.println("Run " + run.getId() + " finished with status " + run.getStatus()); } diff --git a/docs/storages.md b/docs/storages.md index 7856af3..1bc6fb1 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -97,7 +97,7 @@ datasets. |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listKeys(ListKeysOptions)` | List keys. Returns `KeyValueStoreKeysPage`. | -| `iterateKeys(ListKeysOptions)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null` = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. Key iteration is cursor-based, so it has no separate `chunkSize`. | +| `iterateKeys(ListKeysOptions)` / `iterateKeys(ListKeysOptions, Long chunkSize)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null` = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. `chunkSize` sets the per-request page size (`null` = server default). | | `recordExists(String key)` | Whether a record exists. | | `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Returns `Optional`. | | `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. No return value. | diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 8e4514b..7bfb2ba 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -60,9 +60,19 @@ public KeyValueStoreKeysPage listKeys(ListKeysOptions options) { * Returns a lazy iterator over this store's keys, fetching pages on demand via the cursor-based * ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} caps the total number * of keys yielded ({@code null} = all); any {@code exclusiveStartKey} sets the starting point. + * The page size is the server default; use {@link #iterateKeys(ListKeysOptions, Long)} to set it. */ public Iterator iterateKeys(ListKeysOptions options) { - return new KeysIterator(options != null ? options : new ListKeysOptions()); + return iterateKeys(options, null); + } + + /** + * As {@link #iterateKeys(ListKeysOptions)}, but {@code chunkSize} sets the per-request page size + * ({@code null} = server default). Provided for consistency with the collection {@code iterate} + * helpers; key listing is cursor-based, so the options' {@code limit} remains a total-items cap. + */ + public Iterator iterateKeys(ListKeysOptions options, Long chunkSize) { + return new KeysIterator(options != null ? options : new ListKeysOptions(), chunkSize); } /** @@ -70,14 +80,16 @@ public Iterator iterateKeys(ListKeysOptions options) { */ private final class KeysIterator implements Iterator { private final ListKeysOptions options; + private final Long chunkSize; private List buffer = List.of(); private int pos; private String cursor; private Long remaining; private boolean exhausted; - KeysIterator(ListKeysOptions options) { + KeysIterator(ListKeysOptions options, Long chunkSize) { this.options = options; + this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; this.cursor = options.exclusiveStartKeyValue(); Long limit = options.limitValue(); this.remaining = limit != null && limit > 0 ? limit : null; @@ -104,9 +116,13 @@ public KeyValueStoreKey next() { private void fetchPage() { QueryParams params = new QueryParams(); - // Requesting `remaining` keeps the last page from overshooting the caller's total cap; a null - // limit lets the server choose its default page size. - params.addLong("limit", remaining); + // Request the smaller of the remaining cap and the page size, so the last page never + // overshoots the caller's total cap; a null limit lets the server choose its default page. + Long pageLimit = remaining; + if (chunkSize != null && (pageLimit == null || chunkSize < pageLimit)) { + pageLimit = chunkSize; + } + params.addLong("limit", pageLimit); params.addString("exclusiveStartKey", cursor); options.applyFilters(params); KeyValueStoreKeysPage page = diff --git a/src/test/java/com/apify/client/PaginatedIteratorTest.java b/src/test/java/com/apify/client/PaginatedIteratorTest.java index b80f72d..d77461d 100644 --- a/src/test/java/com/apify/client/PaginatedIteratorTest.java +++ b/src/test/java/com/apify/client/PaginatedIteratorTest.java @@ -120,6 +120,22 @@ void startOffsetIsHonored() { assertEquals(7L, f.requests.get(0)[0]); } + @Test + void trimsOverReturnedPageToCap() { + // A misbehaving server that returns more items than requested must not make the iterator + // overshoot the caller's total cap. + PaginatedIterator.PageFetcher overReturning = + (offset, limit) -> { + PaginationList page = new PaginationList<>(); + page.setItems(List.of(1, 2, 3, 4, 5)); // ignores the requested limit + page.setTotal(5); + page.setCount(5); + return page; + }; + List got = drain(new PaginatedIterator<>(3L, 2L, null, overReturning)); + assertEquals(List.of(1, 2, 3), got, "the last page is trimmed to the cap"); + } + @Test void emptyCollectionYieldsNothing() { StubFetcher f = new StubFetcher(0); From 594620b743a2fc4c149b89310f65504613eeb7ef Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 01:08:41 +0000 Subject: [PATCH 09/37] test+feat: cover iterateKeys chunkSize; add single-arg iterate overloads Add hermetic tests for the iterateKeys(options, chunkSize) page-size and cap-combine paths; add convenience iterate(options) overloads across all collection clients and iterateItems for parity with iterateKeys; document the two-form iteration convention in the docs index. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/README.md | 9 ++++ .../AbstractWebhookCollectionClient.java | 5 ++ .../apify/client/ActorCollectionClient.java | 13 ++++-- .../client/ActorVersionCollectionClient.java | 5 ++ .../apify/client/BuildCollectionClient.java | 5 ++ .../java/com/apify/client/DatasetClient.java | 13 ++++++ .../apify/client/DatasetCollectionClient.java | 7 +++ .../client/KeyValueStoreCollectionClient.java | 7 +++ .../client/RequestQueueCollectionClient.java | 7 +++ .../com/apify/client/RunCollectionClient.java | 8 ++++ .../client/ScheduleCollectionClient.java | 5 ++ .../apify/client/StoreCollectionClient.java | 7 +++ .../apify/client/TaskCollectionClient.java | 5 ++ .../WebhookDispatchCollectionClient.java | 5 ++ .../com/apify/client/KeyIteratorTest.java | 46 +++++++++++++++++++ 15 files changed, 144 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 0f6cfbe..ea71523 100644 --- a/docs/README.md +++ b/docs/README.md @@ -103,6 +103,15 @@ PaginationList builds = client.builds().list(new ListOptions().limit(50L) `getCount()`, `isDesc()` and `getItems()`. Within-storage listers (`listKeys`, `listHead`) return their own page/head containers instead. +## Iteration — `iterate` / `iterateItems` / `iterateKeys` + +Each paginated collection also offers a lazy `Iterator` that fetches pages on demand: `iterate(...)` +on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStoreClient.iterateKeys(...)` +(request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the +**total** number of items yielded (`null`/unset = all). The per-request page size is an optional +trailing `chunkSize` argument, so every iterator has two forms — e.g. `iterate(options)` (server +default page size) and `iterate(options, chunkSize)`. Per-page tuning aside, both yield the same items. + ## Resource pages - [Actors, versions & environment variables](actors.md) diff --git a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java index e16a626..23e1153 100644 --- a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java +++ b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java @@ -27,6 +27,11 @@ public PaginationList list(ListOptions options) { * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = * server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/ActorCollectionClient.java b/src/main/java/com/apify/client/ActorCollectionClient.java index 430f92c..cebdc2b 100644 --- a/src/main/java/com/apify/client/ActorCollectionClient.java +++ b/src/main/java/com/apify/client/ActorCollectionClient.java @@ -18,9 +18,16 @@ public PaginationList list(ActorListOptions options) { } /** - * Returns a lazy iterator over the account's Actors, fetching pages on demand. The options' - * {@code limit} caps the total number of Actors yielded ({@code null} = all); {@code chunkSize} - * is the per-request page size ({@code null} = server default). + * Returns a lazy iterator over the account's Actors, fetching pages on demand at the server's + * default page size. The options' {@code limit} caps the total number of Actors yielded ({@code + * null} = all). + */ + public Iterator iterate(ActorListOptions options) { + return iterate(options, null); + } + + /** + * As {@link #iterate(ActorListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ActorListOptions options, Long chunkSize) { ActorListOptions opts = options != null ? options : new ActorListOptions(); diff --git a/src/main/java/com/apify/client/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/ActorVersionCollectionClient.java index 1072364..ebb3ad3 100644 --- a/src/main/java/com/apify/client/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/ActorVersionCollectionClient.java @@ -22,6 +22,11 @@ public PaginationList list(ListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/BuildCollectionClient.java b/src/main/java/com/apify/client/BuildCollectionClient.java index 86a36b5..61f74d5 100644 --- a/src/main/java/com/apify/client/BuildCollectionClient.java +++ b/src/main/java/com/apify/client/BuildCollectionClient.java @@ -25,6 +25,11 @@ public PaginationList list(ListOptions options) { * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = * server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index e409222..10b7896 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -82,6 +82,19 @@ public Iterator iterateItems(DatasetListItemsOptions options, Long chu return iterateItems(options, chunkSize, JsonNode.class); } + /** As {@link #iterateItems(DatasetListItemsOptions, Long)} with the server-default page size. */ + public Iterator iterateItems(DatasetListItemsOptions options) { + return iterateItems(options, null, JsonNode.class); + } + + /** + * As {@link #iterateItems(DatasetListItemsOptions, Long, Class)} with the server-default page + * size. + */ + public Iterator iterateItems(DatasetListItemsOptions options, Class itemClass) { + return iterateItems(options, null, itemClass); + } + /** * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, * fetching pages on demand. The options' {@code limit} caps the total number of items yielded diff --git a/src/main/java/com/apify/client/DatasetCollectionClient.java b/src/main/java/com/apify/client/DatasetCollectionClient.java index 2dea4ee..692362b 100644 --- a/src/main/java/com/apify/client/DatasetCollectionClient.java +++ b/src/main/java/com/apify/client/DatasetCollectionClient.java @@ -22,6 +22,13 @@ public PaginationList list(StorageListOptions options) { * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = * server default). */ + public Iterator iterate(StorageListOptions options) { + return iterate(options, null); + } + + /** + * As {@link #iterate(StorageListOptions)}, but {@code chunkSize} sets the per-request page size. + */ public Iterator iterate(StorageListOptions options, Long chunkSize) { StorageListOptions opts = options != null ? options : new StorageListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java index 6140d71..3b04cc7 100644 --- a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java @@ -22,6 +22,13 @@ public PaginationList list(StorageListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(StorageListOptions options) { + return iterate(options, null); + } + + /** + * As {@link #iterate(StorageListOptions)}, but {@code chunkSize} sets the per-request page size. + */ public Iterator iterate(StorageListOptions options, Long chunkSize) { StorageListOptions opts = options != null ? options : new StorageListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/RequestQueueCollectionClient.java b/src/main/java/com/apify/client/RequestQueueCollectionClient.java index 235f612..fef3aad 100644 --- a/src/main/java/com/apify/client/RequestQueueCollectionClient.java +++ b/src/main/java/com/apify/client/RequestQueueCollectionClient.java @@ -22,6 +22,13 @@ public PaginationList list(StorageListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(StorageListOptions options) { + return iterate(options, null); + } + + /** + * As {@link #iterate(StorageListOptions)}, but {@code chunkSize} sets the per-request page size. + */ public Iterator iterate(StorageListOptions options, Long chunkSize) { StorageListOptions opts = options != null ? options : new StorageListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/RunCollectionClient.java b/src/main/java/com/apify/client/RunCollectionClient.java index 10d0980..a3d935d 100644 --- a/src/main/java/com/apify/client/RunCollectionClient.java +++ b/src/main/java/com/apify/client/RunCollectionClient.java @@ -35,6 +35,14 @@ public PaginationList list(ListOptions options, RunListOptions filter) * chunkSize} is the per-request page size ({@code null} = server default). Both {@code options} * and {@code filter} may be {@code null}. */ + public Iterator iterate(ListOptions options, RunListOptions filter) { + return iterate(options, filter, null); + } + + /** + * As {@link #iterate(ListOptions, RunListOptions)}, but {@code chunkSize} sets the per-request + * page size. + */ public Iterator iterate(ListOptions options, RunListOptions filter, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/ScheduleCollectionClient.java b/src/main/java/com/apify/client/ScheduleCollectionClient.java index 5914c0d..55da65a 100644 --- a/src/main/java/com/apify/client/ScheduleCollectionClient.java +++ b/src/main/java/com/apify/client/ScheduleCollectionClient.java @@ -22,6 +22,11 @@ public PaginationList list(ListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/StoreCollectionClient.java b/src/main/java/com/apify/client/StoreCollectionClient.java index 5add221..831c4e6 100644 --- a/src/main/java/com/apify/client/StoreCollectionClient.java +++ b/src/main/java/com/apify/client/StoreCollectionClient.java @@ -22,6 +22,13 @@ public PaginationList list(StoreListOptions options) { * options' {@code limit} caps the total number of Actors yielded ({@code null} = all); {@code * chunkSize} is the per-request page size ({@code null} = server default). */ + public Iterator iterate(StoreListOptions options) { + return iterate(options, null); + } + + /** + * As {@link #iterate(StoreListOptions)}, but {@code chunkSize} sets the per-request page size. + */ public Iterator iterate(StoreListOptions options, Long chunkSize) { StoreListOptions opts = options != null ? options : new StoreListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/TaskCollectionClient.java b/src/main/java/com/apify/client/TaskCollectionClient.java index 99ac196..698f0bd 100644 --- a/src/main/java/com/apify/client/TaskCollectionClient.java +++ b/src/main/java/com/apify/client/TaskCollectionClient.java @@ -22,6 +22,11 @@ public PaginationList list(ListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java index 76056a7..2e49fd4 100644 --- a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java +++ b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java @@ -25,6 +25,11 @@ public PaginationList list(ListOptions options) { * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code * null} = server default). */ + public Iterator iterate(ListOptions options) { + return iterate(options, null); + } + + /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); return ctx.iterateResource( diff --git a/src/test/java/com/apify/client/KeyIteratorTest.java b/src/test/java/com/apify/client/KeyIteratorTest.java index 08b2c1c..c92b7bb 100644 --- a/src/test/java/com/apify/client/KeyIteratorTest.java +++ b/src/test/java/com/apify/client/KeyIteratorTest.java @@ -67,6 +67,52 @@ void limitCapsTotalKeysYielded() { assertTrue(backend.lastUrl.contains("limit=2"), backend.lastUrl); } + @Test + void chunkSizeSetsPerRequestPageSize() { + MockBackend backend = + new MockBackend( + List.of( + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"a\",\"size\":1},{\"key\":\"b\",\"size\":1}]," + + "\"nextExclusiveStartKey\":\"b\",\"isTruncated\":true}}"), + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"c\",\"size\":1},{\"key\":\"d\",\"size\":1}]," + + "\"nextExclusiveStartKey\":\"d\",\"isTruncated\":true}}"), + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"e\",\"size\":1}]," + + "\"nextExclusiveStartKey\":null,\"isTruncated\":false}}"))); + List got = + keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions(), 2L)); + assertEquals(List.of("a", "b", "c", "d", "e"), got, "chunkSize pages the full collection"); + assertEquals(3, backend.calls); + assertTrue(backend.lastUrl.contains("limit=2"), "chunkSize is sent as the page limit"); + } + + @Test + void chunkSizeAndLimitCombine() { + // limit=3 (total cap) with chunkSize=2: first page requests 2, second requests min(1,2)=1. + MockBackend backend = + new MockBackend( + List.of( + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"a\",\"size\":1},{\"key\":\"b\",\"size\":1}]," + + "\"nextExclusiveStartKey\":\"b\",\"isTruncated\":true}}"), + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{\"key\":\"c\",\"size\":1},{\"key\":\"d\",\"size\":1}]," + + "\"nextExclusiveStartKey\":\"d\",\"isTruncated\":true}}"))); + List got = + keys(client(backend).keyValueStore("s").iterateKeys(new ListKeysOptions().limit(3L), 2L)); + assertEquals(List.of("a", "b", "c"), got, "the total cap trims across chunked pages"); + assertEquals(2, backend.calls); + assertTrue( + backend.lastUrl.contains("limit=1"), "the last page requests only the remaining cap"); + } + @Test void stopsOnEmptyPage() { MockBackend backend = From 02c977e71a63b45a6b1df09c4a665ddc3594fd2e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 01:20:34 +0000 Subject: [PATCH 10/37] fix: remove ambiguous iterateItems(options, Class) overload The (options, Class) overload clashed with (options, Long chunkSize), making iterateItems(opts, null) ambiguous. Drop it; typed iteration uses the 3-arg form. Add a hermetic test for the single-arg iterate delegation. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../java/com/apify/client/DatasetClient.java | 8 ------- .../com/apify/client/ReviewFixesTest.java | 23 +++++++++++++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 10b7896..06b2eae 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -87,14 +87,6 @@ public Iterator iterateItems(DatasetListItemsOptions options) { return iterateItems(options, null, JsonNode.class); } - /** - * As {@link #iterateItems(DatasetListItemsOptions, Long, Class)} with the server-default page - * size. - */ - public Iterator iterateItems(DatasetListItemsOptions options, Class itemClass) { - return iterateItems(options, null, itemClass); - } - /** * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, * fetching pages on demand. The options' {@code limit} caps the total number of items yielded diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ReviewFixesTest.java index 9bebbaf..c3c1ed9 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ReviewFixesTest.java @@ -302,6 +302,29 @@ void storeIterateWalksMultiplePages() { assertEquals(3, backend.calls, "two data pages plus the terminating empty page"); } + @Test + void collectionIterateSingleArgDelegatesToServerDefaultPageSize() { + // The arg-less-chunkSize convenience overload must page correctly (delegates with null chunk). + MockBackend backend = + new MockBackend( + List.of( + MockBackend.ok( + 200, + "{\"data\":{\"items\":[{},{}],\"total\":2,\"offset\":0,\"limit\":2,\"count\":2}}"), + MockBackend.ok( + 200, + "{\"data\":{\"items\":[],\"total\":2,\"offset\":2,\"limit\":2,\"count\":0}}"))); + java.util.Iterator it = client(backend).actors().iterate(new ActorListOptions()); + int count = 0; + while (it.hasNext()) { + it.next(); + count++; + } + assertEquals(2, count, "single-arg iterate should yield every item"); + assertFalse( + backend.lastUrl.contains("limit="), "no chunkSize => no limit param (server default)"); + } + @Test void runCollectionListToleratesNullOptionsAndFilter() { MockBackend backend = From a930fe9efeb49674ee820dcb69320fcb19ef8e77 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 01:32:27 +0000 Subject: [PATCH 11/37] test: hermetic coverage for dataset-items iteration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add DatasetItemsIteratorTest (MockBackend) covering iterateItems / fetchItemsPage: bare-array paging with empty-page termination, total-cap trim, and typed decoding — so the dataset-items path is gated offline. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../client/DatasetItemsIteratorTest.java | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/test/java/com/apify/client/DatasetItemsIteratorTest.java diff --git a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java new file mode 100644 index 0000000..eea888a --- /dev/null +++ b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java @@ -0,0 +1,78 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +/** + * Hermetic (token-free) tests for {@link DatasetClient#iterateItems} and its {@code fetchItemsPage} + * helper — the distinct dataset-items path that parses a bare JSON array body (not a {@code data} + * envelope) and terminates on an empty page. Driven by a {@link MockBackend}, no {@code + * APIFY_TOKEN}. + */ +class DatasetItemsIteratorTest { + + private static ApifyClient client(MockBackend backend) { + return ApifyClient.builder() + .token("test-token") + .httpBackend(backend) + .maxRetries(0) + .minDelayBetweenRetries(Duration.ofMillis(1)) + .build(); + } + + @Test + void pagesBareArrayBodyAndStopsOnEmptyPage() { + MockBackend backend = + new MockBackend( + List.of( + MockBackend.ok(200, "[{\"n\":1},{\"n\":2}]"), + MockBackend.ok(200, "[{\"n\":3}]"), + MockBackend.ok(200, "[]"))); + List seen = new ArrayList<>(); + Iterator it = + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L); + while (it.hasNext()) { + seen.add(it.next().get("n").asInt()); + } + assertEquals(List.of(1, 2, 3), seen, "iterateItems should page the bare-array endpoint"); + assertEquals(3, backend.calls, "two data pages plus the terminating empty page"); + assertTrue(backend.lastUrl.contains("datasets/d1/items"), backend.lastUrl); + assertTrue(backend.lastUrl.contains("limit=2"), "chunkSize drives the per-request page size"); + } + + @Test + void totalCapTrimsDatasetItems() { + // The cap wins even though the server would return more; only the first page is requested. + MockBackend backend = MockBackend.ofConstant(200, "[{\"n\":1},{\"n\":2},{\"n\":3}]"); + List seen = new ArrayList<>(); + Iterator it = + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions().limit(2L), 5L); + while (it.hasNext()) { + seen.add(it.next().get("n").asInt()); + } + assertEquals(List.of(1, 2), seen, "limit caps the total items yielded"); + assertEquals(1, backend.calls); + } + + @Test + void decodesIntoRequestedType() { + MockBackend backend = + new MockBackend(List.of(MockBackend.ok(200, "[{\"n\":7}]"), MockBackend.ok(200, "[]"))); + List seen = new ArrayList<>(); + Iterator it = + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L, Map.class); + while (it.hasNext()) { + seen.add(it.next().get("n")); + } + assertEquals(1, seen.size(), "typed iteration should decode and yield the item"); + assertEquals(7, ((Number) seen.get(0)).intValue()); + } +} From 2f142cf89d9a6b3df75d358810cb1598e9df2081 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 01:43:48 +0000 Subject: [PATCH 12/37] test: use a typed record instead of raw Map in dataset-items test Avoids the raw-type in decodesIntoRequestedType. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../com/apify/client/DatasetItemsIteratorTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java index eea888a..f5e4a7c 100644 --- a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java +++ b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java @@ -8,7 +8,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; import org.junit.jupiter.api.Test; /** @@ -62,17 +61,20 @@ void totalCapTrimsDatasetItems() { assertEquals(1, backend.calls); } + /** A typed item for {@link #decodesIntoRequestedType()}. */ + public record Item(int n) {} + @Test void decodesIntoRequestedType() { MockBackend backend = new MockBackend(List.of(MockBackend.ok(200, "[{\"n\":7}]"), MockBackend.ok(200, "[]"))); - List seen = new ArrayList<>(); - Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L, Map.class); + List seen = new ArrayList<>(); + Iterator it = + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), 2L, Item.class); while (it.hasNext()) { - seen.add(it.next().get("n")); + seen.add(it.next()); } assertEquals(1, seen.size(), "typed iteration should decode and yield the item"); - assertEquals(7, ((Number) seen.get(0)).intValue()); + assertEquals(7, seen.get(0).n()); } } From e7697a19a2cc5f5b4dc2dff12dd034f31adeb6c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 01:58:43 +0000 Subject: [PATCH 13/37] polish: KeysIterator honors isTruncated; normalize chunkSize; doc note Terminate key iteration on the API's isTruncated flag; normalize chunkSize in the PaginatedIterator constructor; reconcile the docs iteration note with the per-resource tables. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/README.md | 5 +++-- src/main/java/com/apify/client/KeyValueStoreClient.java | 7 +++++-- src/main/java/com/apify/client/PaginatedIterator.java | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index ea71523..4c9bee2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -109,8 +109,9 @@ Each paginated collection also offers a lazy `Iterator` that fetches pages on de on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStoreClient.iterateKeys(...)` (request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the **total** number of items yielded (`null`/unset = all). The per-request page size is an optional -trailing `chunkSize` argument, so every iterator has two forms — e.g. `iterate(options)` (server -default page size) and `iterate(options, chunkSize)`. Per-page tuning aside, both yield the same items. +trailing `chunkSize` argument: the per-resource tables below show the `chunkSize` form, and each +iterator also has an overload that omits it (using the server's default page size). Per-page tuning +aside, both forms yield the same items. ## Resource pages diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 7bfb2ba..34d6ac6 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -130,6 +130,7 @@ private void fetchPage() { buffer = page.getItems(); pos = 0; cursor = page.getNextExclusiveStartKey(); + boolean truncated = page.isTruncated(); if (remaining != null) { // Defensively trim the last page to the cap in case the server returned more than // requested. @@ -138,10 +139,12 @@ private void fetchPage() { } remaining -= buffer.size(); } - // Stop when the page is empty, there is no next cursor, or the total cap is reached. - if (buffer.isEmpty() + // Stop when the API reports the listing is not truncated (no more keys), there is no next + // cursor, the page is empty, or the total cap is reached. + if (!truncated || cursor == null || cursor.isEmpty() + || buffer.isEmpty() || (remaining != null && remaining <= 0)) { exhausted = true; } diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java index 965ff5a..7108e27 100644 --- a/src/main/java/com/apify/client/PaginatedIterator.java +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -48,7 +48,7 @@ interface PageFetcher { PaginatedIterator(Long totalLimit, Long chunkSize, Long startOffset, PageFetcher fetcher) { this.totalLimit = totalLimit != null && totalLimit > 0 ? totalLimit : null; - this.chunkSize = chunkSize; + this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; this.offset = startOffset != null && startOffset > 0 ? startOffset : 0; this.fetcher = fetcher; } From 1efd73362f03ca01f1f784878425c71c2d7c31f9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 02:47:19 +0000 Subject: [PATCH 14/37] fix: snapshot iterator filters for full options isolation Iterators now snapshot the caller's filters at call time (matching the old copy() behavior), so mutating the options object mid-iteration no longer leaks into later pages. Align docs storage-name snippet with the runnable program's unique-name isolation. Add an isolation regression test. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/examples.md | 9 +++++---- src/main/java/com/apify/client/DatasetClient.java | 9 +++++++-- .../com/apify/client/KeyValueStoreClient.java | 9 ++++++--- .../java/com/apify/client/ResourceContext.java | 6 +++++- .../java/com/apify/client/ReviewFixesTest.java | 15 +++++++++++++++ 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index c7dcae4..ffa7adb 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -20,10 +20,11 @@ System.out.println("Items in this page: " + items.getCount()); ```java // Named storages persist on your account; each block deletes its storage in a finally so the -// example does not leak them. +// example does not leak them. A unique suffix keeps the names from colliding across parallel runs. +String suffix = "-" + System.currentTimeMillis(); // Dataset: create, push items, read them back. -Dataset dataset = client.datasets().getOrCreate("example-ds"); +Dataset dataset = client.datasets().getOrCreate("example-ds" + suffix); try { client.dataset(dataset.getId()).pushItems(List.of(Map.of("hello", "world"))); PaginationList items = client.dataset(dataset.getId()).listItems(new DatasetListItemsOptions()); @@ -33,7 +34,7 @@ try { } // Key-value store: create, set a record, read it back. -KeyValueStore store = client.keyValueStores().getOrCreate("example-kvs"); +KeyValueStore store = client.keyValueStores().getOrCreate("example-kvs" + suffix); try { client.keyValueStore(store.getId()).setRecordJson("OUTPUT", Map.of("answer", 42)); Optional record = client.keyValueStore(store.getId()).getRecord("OUTPUT"); @@ -43,7 +44,7 @@ try { } // Request queue: create, add a request, read the head. -RequestQueue queue = client.requestQueues().getOrCreate("example-rq"); +RequestQueue queue = client.requestQueues().getOrCreate("example-rq" + suffix); try { client.requestQueue(queue.getId()).addRequest(new RequestQueueRequest("https://example.com", "example"), false); RequestQueueHead head = client.requestQueue(queue.getId()).listHead(10L); diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 06b2eae..7e7ba51 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -103,14 +103,19 @@ public Iterator iterateItems(DatasetListItemsOptions options) { public Iterator iterateItems( DatasetListItemsOptions options, Long chunkSize, Class itemClass) { DatasetListItemsOptions opts = options != null ? options : new DatasetListItemsOptions(); + // Snapshot the filters/offset/limit/desc once so mutating the options mid-iteration cannot + // leak. + QueryParams filters = new QueryParams(); + opts.applyFilters(filters); + Boolean desc = opts.descValue(); return new PaginatedIterator<>( opts.limitValue(), chunkSize, opts.offsetValue(), (offset, pageLimit) -> { QueryParams p = new QueryParams().addLong("offset", offset).addLong("limit", pageLimit); - opts.applyFilters(p); - return fetchItemsPage(p, opts.descValue(), itemClass); + p.extend(filters); + return fetchItemsPage(p, desc, itemClass); }); } diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 34d6ac6..3b4e3b3 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -79,8 +79,8 @@ public Iterator iterateKeys(ListKeysOptions options, Long chun * Lazily iterates over a store's keys via the cursor-based ({@code exclusiveStartKey}) listing. */ private final class KeysIterator implements Iterator { - private final ListKeysOptions options; private final Long chunkSize; + private final QueryParams filters; private List buffer = List.of(); private int pos; private String cursor; @@ -88,11 +88,14 @@ private final class KeysIterator implements Iterator { private boolean exhausted; KeysIterator(ListKeysOptions options, Long chunkSize) { - this.options = options; this.chunkSize = chunkSize != null && chunkSize > 0 ? chunkSize : null; this.cursor = options.exclusiveStartKeyValue(); Long limit = options.limitValue(); this.remaining = limit != null && limit > 0 ? limit : null; + // Snapshot the filters once so mutating the options mid-iteration cannot leak into later + // pages. + this.filters = new QueryParams(); + options.applyFilters(this.filters); } @Override @@ -124,7 +127,7 @@ private void fetchPage() { } params.addLong("limit", pageLimit); params.addString("exclusiveStartKey", cursor); - options.applyFilters(params); + params.extend(filters); KeyValueStoreKeysPage page = ctx.getResourceRequired("keys", params, KeyValueStoreKeysPage.class); buffer = page.getItems(); diff --git a/src/main/java/com/apify/client/ResourceContext.java b/src/main/java/com/apify/client/ResourceContext.java index 9d47622..1c2e8d7 100644 --- a/src/main/java/com/apify/client/ResourceContext.java +++ b/src/main/java/com/apify/client/ResourceContext.java @@ -182,13 +182,17 @@ Iterator iterateResource( Long startOffset, Consumer applyFilters, Class itemClass) { + // Snapshot the caller's filters once, so mutating the options object mid-iteration cannot leak + // into later pages (the iterator owns an independent copy of every filter, offset and limit). + QueryParams filters = new QueryParams(); + applyFilters.accept(filters); return new PaginatedIterator<>( totalLimit, chunkSize, startOffset, (offset, pageLimit) -> { QueryParams p = new QueryParams().addLong("offset", offset).addLong("limit", pageLimit); - applyFilters.accept(p); + p.extend(filters); return listResource(subPath, p, itemClass); }); } diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ReviewFixesTest.java index c3c1ed9..782786f 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ReviewFixesTest.java @@ -325,6 +325,21 @@ void collectionIterateSingleArgDelegatesToServerDefaultPageSize() { backend.lastUrl.contains("limit="), "no chunkSize => no limit param (server default)"); } + @Test + void iterateSnapshotsOptionsSoLaterMutationsDoNotLeak() { + // The iterator must capture the options (offset/limit AND filters) at call time; mutating the + // caller's options object afterwards must not change subsequent page requests. + MockBackend backend = + MockBackend.ofConstant( + 200, "{\"data\":{\"items\":[{}],\"total\":1,\"offset\":0,\"limit\":0,\"count\":1}}"); + ActorListOptions options = new ActorListOptions().sortBy("createdAt"); + java.util.Iterator it = client(backend).actors().iterate(options); + options.sortBy("modifiedAt"); // mutate after obtaining the iterator + it.hasNext(); // triggers the first page fetch + assertTrue(backend.lastUrl.contains("sortBy=createdAt"), backend.lastUrl); + assertFalse(backend.lastUrl.contains("modifiedAt"), backend.lastUrl); + } + @Test void runCollectionListToleratesNullOptionsAndFilter() { MockBackend backend = From 9c95d61e7a178b4be986ac4b3800fc9eb0e24234 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 03:07:35 +0000 Subject: [PATCH 15/37] docs: unique actor name in example snippet; clarify iterator equivalence Actor-creation snippet uses a unique name (matching the storage snippet and runnable program); README no longer claims blanket item-equivalence across iterator forms, pointing to the iterateItems+filter caveat. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/README.md | 6 ++++-- docs/examples.md | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/README.md b/docs/README.md index 4c9bee2..3de825a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -110,8 +110,10 @@ on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStore (request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the **total** number of items yielded (`null`/unset = all). The per-request page size is an optional trailing `chunkSize` argument: the per-resource tables below show the `chunkSize` form, and each -iterator also has an overload that omits it (using the server's default page size). Per-page tuning -aside, both forms yield the same items. +iterator also has an overload that omits it (using the server's default page size). The page size +does not change which items a collection iterator yields; note the one exception in +[Storages](storages.md) — `iterateItems` combined with server-side item filters, where the page size +can affect the result. ## Resource pages diff --git a/docs/examples.md b/docs/examples.md index ffa7adb..4b01af5 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -65,7 +65,8 @@ user.ifPresent(u -> System.out.println("Account " + u.getId() + " / " + u.getUse ```java Actor created = client.actors().create(Map.of( - "name", "my-example-actor", + // A unique name avoids collisions across parallel runs. + "name", "my-example-actor-" + System.currentTimeMillis(), "isPublic", false, "versions", List.of(Map.of( "versionNumber", "0.0", From 2becb8bcf326aa4bc18468437d271458610f10d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 03:49:38 +0000 Subject: [PATCH 16/37] docs: Maven Central + Gradle repositories note; lastRun() read-only caveat Document that the client is on Maven Central and show mavenCentral() in the Gradle snippet; note that lastRun() RunClients are for reads and the mutating methods need a concrete run id. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- README.md | 14 +++++++++++--- docs/runs.md | 8 ++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 86c6f04..2c11b10 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ queues, tasks, schedules, webhooks, the store, users and logs). ## Installation -Maven: +The client is published to [Maven Central](https://central.sonatype.com/artifact/com.apify/apify-client). + +Maven (Maven Central is a default repository, so no extra configuration is needed): ```xml @@ -25,10 +27,16 @@ Maven: ``` -Gradle: +Gradle — ensure `mavenCentral()` is in your `repositories`, then add the dependency: ```groovy -implementation 'com.apify:apify-client:0.2.0' +repositories { + mavenCentral() +} + +dependencies { + implementation 'com.apify:apify-client:0.2.0' +} ``` ## Quick start diff --git a/docs/runs.md b/docs/runs.md index 7d75bb4..9d0673e 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -43,6 +43,14 @@ use `log()` for the full log text or for non-raw/download options. To set the current run's status message from inside an Actor, use the top-level `client.setStatusMessage(...)` (see [the docs index](README.md#setting-single-resource-status)). +> **`lastRun()` clients are for reads.** A `RunClient` obtained via `actor(id).lastRun(...)` / +> `task(id).lastRun(...)` targets the `runs/last` path and is intended for reading (`get`, +> `dataset()`/`keyValueStore()`/`requestQueue()`, `log()`). The mutating methods (`abort`, `reboot`, +> `resurrect`, `metamorph`, `charge`, `update`, `delete`) require a concrete run and have no +> `runs/last` endpoint — resolve the id first (`lastRun(...).get()` then `client.run(id)`). The type +> exposes them uniformly (matching the reference client's shared `RunClient`), but calling them on a +> last-run client will fail server-side. + `ActorRun` fields include `getId()`, `getActId()`, `getUserId()`, `getStatus()`, `getStatusMessage()`, `getStartedAt()`, `getFinishedAt()`, `getBuildId()`, `getDefaultDatasetId()`, `getDefaultKeyValueStoreId()`, `getDefaultRequestQueueId()`, `getContainerUrl()`, plus `isTerminal()`. From 257f1bfc14a879b0d02b2c63ec01cbf7d5972097 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 05:32:53 +0000 Subject: [PATCH 17/37] refactor: simplify PaginatedIterator.minForLimit to null-aware min Constructor already normalizes totalLimit/chunkSize to positive-or-null and capRemaining is always > 0 at the call site (exhausted guard), so the in-method > 0 re-checks were dead. Collapse to a plain null-aware min and document the input contract. No behavior change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../com/apify/client/PaginatedIterator.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java index 7108e27..9f37440 100644 --- a/src/main/java/com/apify/client/PaginatedIterator.java +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -95,18 +95,18 @@ private void fetchNextPage() { } /** - * The API treats {@code 0} as "unset" for a limit, so this returns the smaller of the two - * positive bounds, or {@code null} (server default) when neither is a positive value. + * Returns the smaller of the two per-page bounds, or {@code null} (server default) when neither + * is set. Both inputs are already positive-or-{@code null}: {@code chunkSize} is normalized in + * the constructor and {@code capRemaining} is only ever {@code > 0} here (the {@code exhausted} + * guard blocks fetching once the cap is reached). */ private static Long minForLimit(Long a, Long b) { - Long x = a != null && a > 0 ? a : null; - Long y = b != null && b > 0 ? b : null; - if (x == null) { - return y; + if (a == null) { + return b; } - if (y == null) { - return x; + if (b == null) { + return a; } - return Math.min(x, y); + return Math.min(a, b); } } From faf8c20c01e0dcaac212c9b63bcacd0b14808ef5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 06:47:02 +0000 Subject: [PATCH 18/37] fix: versions().iterate() single-fetch on non-paginated endpoint GET /v2/actors/{actorId}/versions takes no pagination params and returns the full {total, items} list on every request (server ignores offset), identical to the env-vars collection. Routing it through the offset/limit PaginatedIterator looped forever (no limit) or duplicated items (with a limit), since that engine only terminates on an empty page or the total cap. Make iterate(ListOptions) a single fetch mirroring the env-vars sibling, honoring the limit as a client-side cap and dropping the meaningless chunkSize overload. Correct CHANGELOG (Actor-version regrouped with env-vars as non-paginated) and docs/actors.md. Integration test now fully drains the versions iterator; add hermetic ReviewFixesTest regression tests that reproduce the endpoint with a constant mock backend and assert single-fetch, exactly-once iteration, and the limit cap. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 11 +++-- docs/actors.md | 2 +- .../client/ActorVersionCollectionClient.java | 32 +++++++------ .../com/apify/client/ReviewFixesTest.java | 46 +++++++++++++++++++ .../integration/IterationIntegrationTest.java | 17 +++++-- 5 files changed, 84 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79cfb4b..e2ef5db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added - Lazy iteration helpers over every paginated collection, matching the reference JS client's - iterable `list()`: `iterate(options, chunkSize)` on the Actor, Actor-version, build, run, dataset, + iterable `list()`: `iterate(options, chunkSize)` on the Actor, build, run, dataset, key-value-store, request-queue, task, schedule, webhook, and webhook-dispatch collection clients; - `ActorEnvVarCollectionClient.iterate()` (the non-paginated env-var collection); and - `DatasetClient.iterateItems(...)` for dataset items and `KeyValueStoreClient.iterateKeys(...)` for - store keys. The options' `limit` caps the total number of items yielded and `chunkSize` sets the - per-request page size. + and `DatasetClient.iterateItems(...)` for dataset items and `KeyValueStoreClient.iterateKeys(...)` + for store keys. The options' `limit` caps the total number of items yielded and `chunkSize` sets + the per-request page size. The non-paginated collections — `ActorVersionCollectionClient.iterate(options)` + and `ActorEnvVarCollectionClient.iterate()` — return the full list in a single fetch (no page size + to tune). ### Changed diff --git a/docs/actors.md b/docs/actors.md index 68c6efd..8b3baef 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -91,7 +91,7 @@ and deletes a single version and exposes its environment variables. | Method | Description | |---|---| | `list(ListOptions)` | List the Actor's versions. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all versions; `limit` caps the total, `chunkSize` sets the page size. | +| `iterate(ListOptions)` | Lazy `Iterator` over all versions; `limit` caps the total. The versions endpoint is not paginated (one fetch returns every version), so there is no page size to tune. | | `create(Object version)` | Create a version. Returns `ActorVersion`. | ### `ActorVersionClient` — `client.actor(id).version(v)` diff --git a/src/main/java/com/apify/client/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/ActorVersionCollectionClient.java index ebb3ad3..213c51a 100644 --- a/src/main/java/com/apify/client/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/ActorVersionCollectionClient.java @@ -1,6 +1,7 @@ package com.apify.client; import java.util.Iterator; +import java.util.List; /** A client for an Actor's version collection ({@code GET/POST /v2/actors/{actorId}/versions}). */ public final class ActorVersionCollectionClient { @@ -18,24 +19,25 @@ public PaginationList list(ListOptions options) { } /** - * Returns a lazy iterator over the Actor's versions. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * Returns a lazy iterator over the Actor's versions. + * + *

{@code GET /v2/actors/{actorId}/versions} is not offset/limit paginated: it takes + * no pagination parameters and returns the full version list in a single {@code {total, items}} + * response (the server ignores {@code offset}). This iterates that one fetched page, so draining + * the iterator always terminates and never re-yields a version. (Routing it through the offset/ + * limit paging engine would loop forever, since the server returns the same non-empty page at + * every offset — this is why the sibling non-paginated {@code env-vars} collection is also a + * single-fetch iterator.) The options' {@code limit} still caps the number yielded ({@code null} + * = all); there is no page size to tune. */ public Iterator iterate(ListOptions options) { - return iterate(options, null); - } - - /** As {@link #iterate(ListOptions)}, but {@code chunkSize} sets the per-request page size. */ - public Iterator iterate(ListOptions options, Long chunkSize) { ListOptions opts = options != null ? options : new ListOptions(); - return ctx.iterateResource( - "", - opts.limitValue(), - chunkSize, - opts.offsetValue(), - opts::applyFilters, - ActorVersion.class); + List items = list(opts).getItems(); + Long limit = opts.limitValue(); + if (limit != null && limit > 0 && items.size() > limit) { + items = items.subList(0, (int) (long) limit); + } + return items.iterator(); } /** Creates a new Actor version. {@code version} is any JSON-serializable version definition. */ diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ReviewFixesTest.java index 782786f..b3a1637 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ReviewFixesTest.java @@ -449,4 +449,50 @@ void logStreamThrowsOnNonSuccessStatus() { assertThrows(ApifyApiException.class, () -> client(backend).log("run1").stream()); assertEquals(403, ex.getStatusCode()); } + + @Test + void versionsIterateSingleFetchTerminatesAndDoesNotDuplicate() { + // GET /v2/actors/{actorId}/versions is NOT offset/limit paginated: the server ignores `offset` + // and returns the full {total, items} list on every request. `ofConstant` reproduces exactly + // that (same non-empty page for every call). Draining the iterator must terminate and yield + // each version once. Routing this endpoint through the offset/limit paging engine looped + // forever (empty-page termination never triggers), so this pins the single-fetch behaviour. + MockBackend backend = + MockBackend.ofConstant( + 200, + "{\"data\":{\"total\":3,\"items\":[" + + "{\"versionNumber\":\"0.1\"}," + + "{\"versionNumber\":\"0.2\"}," + + "{\"versionNumber\":\"0.3\"}]}}"); + java.util.Iterator it = + client(backend).actor("me/actor").versions().iterate(new ListOptions()); + List yielded = new ArrayList<>(); + int guard = 0; + while (it.hasNext()) { + // Guard converts a termination regression (infinite loop) into a clean failure, not a hang. + assertTrue( + ++guard <= 100, "versions iterator did not terminate (paged a non-paginated endpoint)"); + yielded.add(it.next().getVersionNumber()); + } + assertEquals(List.of("0.1", "0.2", "0.3"), yielded, "each version yielded once, in order"); + assertEquals(1, backend.calls, "non-paginated versions endpoint must be fetched exactly once"); + } + + @Test + void versionsIterateHonorsTotalLimitCap() { + MockBackend backend = + MockBackend.ofConstant( + 200, + "{\"data\":{\"total\":3,\"items\":[" + + "{\"versionNumber\":\"0.1\"}," + + "{\"versionNumber\":\"0.2\"}," + + "{\"versionNumber\":\"0.3\"}]}}"); + java.util.Iterator it = + client(backend).actor("me/actor").versions().iterate(new ListOptions().limit(2L)); + List yielded = new ArrayList<>(); + while (it.hasNext()) { + yielded.add(it.next().getVersionNumber()); + } + assertEquals(List.of("0.1", "0.2"), yielded, "limit caps the number yielded"); + } } diff --git a/src/test/java/com/apify/client/integration/IterationIntegrationTest.java b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java index 1fb140a..d84e349 100644 --- a/src/test/java/com/apify/client/integration/IterationIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/IterationIntegrationTest.java @@ -261,10 +261,21 @@ void iterateActorVersionsAndEnvVars() { Actor actor = client.actors().create(ActorIntegrationTest.minimalActor(uniqueName("it-ver"))); try { var actorClient = client.actor(actor.getId()); - // The minimal Actor ships with version 0.0, so iteration must yield at least one version. + // The versions endpoint is not paginated (one fetch returns every version); fully draining + // the iterator must terminate and must not re-yield a version. The minimal Actor ships with + // version 0.0, so iteration yields at least one version, each exactly once. + Set versionNumbers = new HashSet<>(); + int versionCount = 0; Iterator versions = - actorClient.versions().iterate(new ListOptions(), 1L); - assertTrue(versions.hasNext(), "expected at least the initial version"); + actorClient.versions().iterate(new ListOptions()); + while (versions.hasNext()) { + versionCount++; + versionNumbers.add(versions.next().getVersionNumber()); + } + assertTrue(versionCount >= 1, "expected at least the initial version"); + assertEquals(versionCount, versionNumbers.size(), "versions iterator re-yielded a version"); + assertTrue( + versionNumbers.contains("0.0"), "expected initial version 0.0, saw " + versionNumbers); var envVars = actorClient.version("0.0").envVars(); envVars.create(new ActorEnvVar("IT_VAR_A", "a")); From 4c5f46980812323ccca140c554bd4b65102ce4f0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 07:47:41 +0000 Subject: [PATCH 19/37] docs: clarify iterate limit(0) semantics and log-redirection exception Add a parity note (docs/README.md Iteration section + ListOptions.limit Javadoc) that during iterate() a non-positive/zero limit means no cap, unlike list() which sends limit=0 verbatim. Note that the log-redirection example fragment throws a checked IOException and link the runnable LogRedirection.java. Documentation only; no behavior change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/README.md | 4 +++- docs/examples.md | 5 +++++ src/main/java/com/apify/client/ListOptions.java | 7 ++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 3de825a..9644749 100644 --- a/docs/README.md +++ b/docs/README.md @@ -108,7 +108,9 @@ their own page/head containers instead. Each paginated collection also offers a lazy `Iterator` that fetches pages on demand: `iterate(...)` on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStoreClient.iterateKeys(...)` (request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the -**total** number of items yielded (`null`/unset = all). The per-request page size is an optional +**total** number of items yielded; `null`/unset — or a non-positive value such as `0` — means no cap, +so every item is yielded. (This differs from `list(...)`, which sends `limit=0` verbatim and returns +zero items; during iteration a falsy `limit` means "unbounded", matching the reference JS client.) The per-request page size is an optional trailing `chunkSize` argument: the per-resource tables below show the `chunkSize` form, and each iterator also has an overload that omits it (using the server's default page size). The page size does not change which items a collection iterator yields; note the one exception in diff --git a/docs/examples.md b/docs/examples.md index 4b01af5..b60dfb0 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -112,6 +112,11 @@ while (shown < 5 && it.hasNext()) { ## Run an Actor with log redirection +`getStreamedLog()` and `InputStream.transferTo(...)` throw a checked `IOException`, so run this where +that exception is handled — the complete +[`LogRedirection.java`](../src/test/java/com/apify/client/examples/LogRedirection.java) program declares +`throws Exception`. + ```java ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); try (InputStream stream = client.run(run.getId()).getStreamedLog()) { diff --git a/src/main/java/com/apify/client/ListOptions.java b/src/main/java/com/apify/client/ListOptions.java index 4c3e5de..6e952c1 100644 --- a/src/main/java/com/apify/client/ListOptions.java +++ b/src/main/java/com/apify/client/ListOptions.java @@ -16,7 +16,12 @@ public ListOptions offset(Long offset) { return this; } - /** Maximum number of items to return. */ + /** + * Maximum number of items to return. In {@code list(...)} this is sent verbatim (so {@code 0} + * returns zero items); in {@code iterate(...)} a non-positive/zero {@code limit} means "no cap" — + * iteration yields every item (matching the reference client's falsy-limit-means-unbounded + * intent). + */ public ListOptions limit(Long limit) { this.limit = limit; return this; From 8da26463b918b9a219c2726e7d0769fbbe595796 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 08:20:55 +0000 Subject: [PATCH 20/37] docs: iterate limit(0) parity note on all offset options classes Extend the list-vs-iterate limit(0) clarification to every options class that feeds an iterate helper (StorageListOptions, ActorListOptions, StoreListOptions, DatasetListItemsOptions, ListKeysOptions), since these are independent classes and do not inherit ListOptions' Javadoc. ListRequestsOptions is unchanged (it has no iterate overload). Docs only. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/main/java/com/apify/client/ActorListOptions.java | 5 ++++- src/main/java/com/apify/client/DatasetListItemsOptions.java | 6 +++++- src/main/java/com/apify/client/ListKeysOptions.java | 5 ++++- src/main/java/com/apify/client/StorageListOptions.java | 6 +++++- src/main/java/com/apify/client/StoreListOptions.java | 5 ++++- 5 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/apify/client/ActorListOptions.java b/src/main/java/com/apify/client/ActorListOptions.java index 88496fc..cdf9c57 100644 --- a/src/main/java/com/apify/client/ActorListOptions.java +++ b/src/main/java/com/apify/client/ActorListOptions.java @@ -14,7 +14,10 @@ public ActorListOptions offset(Long offset) { return this; } - /** Maximum number of Actors to return. */ + /** + * Maximum number of Actors to return. Sent verbatim by {@code list(...)} (so {@code 0} returns + * zero Actors); in {@code iterate(...)} a non-positive/zero {@code limit} means "no cap" (all). + */ public ActorListOptions limit(Long limit) { this.limit = limit; return this; diff --git a/src/main/java/com/apify/client/DatasetListItemsOptions.java b/src/main/java/com/apify/client/DatasetListItemsOptions.java index ad01795..94bc0ab 100644 --- a/src/main/java/com/apify/client/DatasetListItemsOptions.java +++ b/src/main/java/com/apify/client/DatasetListItemsOptions.java @@ -29,7 +29,11 @@ public DatasetListItemsOptions offset(Long offset) { return this; } - /** Maximum number of items to return. */ + /** + * Maximum number of items to return. Sent verbatim by {@code listItems(...)} (so {@code 0} + * returns zero items); in {@code iterateItems(...)} a non-positive/zero {@code limit} means "no + * cap" (all). + */ public DatasetListItemsOptions limit(Long limit) { this.limit = limit; return this; diff --git a/src/main/java/com/apify/client/ListKeysOptions.java b/src/main/java/com/apify/client/ListKeysOptions.java index 7044243..ca2d1d7 100644 --- a/src/main/java/com/apify/client/ListKeysOptions.java +++ b/src/main/java/com/apify/client/ListKeysOptions.java @@ -8,7 +8,10 @@ public final class ListKeysOptions { private String collection; private String signature; - /** Maximum number of keys to return. */ + /** + * Maximum number of keys to return. Sent verbatim by {@code listKeys(...)} (so {@code 0} returns + * zero keys); in {@code iterateKeys(...)} a non-positive/zero {@code limit} means "no cap" (all). + */ public ListKeysOptions limit(Long limit) { this.limit = limit; return this; diff --git a/src/main/java/com/apify/client/StorageListOptions.java b/src/main/java/com/apify/client/StorageListOptions.java index ab14c21..f9b9e6e 100644 --- a/src/main/java/com/apify/client/StorageListOptions.java +++ b/src/main/java/com/apify/client/StorageListOptions.java @@ -18,7 +18,11 @@ public StorageListOptions offset(Long offset) { return this; } - /** Maximum number of items to return. */ + /** + * Maximum number of items to return. Sent verbatim by {@code list(...)} (so {@code 0} returns + * zero items); in {@code iterate(...)} a non-positive/zero {@code limit} means "no cap" (all + * items). + */ public StorageListOptions limit(Long limit) { this.limit = limit; return this; diff --git a/src/main/java/com/apify/client/StoreListOptions.java b/src/main/java/com/apify/client/StoreListOptions.java index d2947ed..9187464 100644 --- a/src/main/java/com/apify/client/StoreListOptions.java +++ b/src/main/java/com/apify/client/StoreListOptions.java @@ -19,7 +19,10 @@ public StoreListOptions offset(Long offset) { return this; } - /** Maximum number of Actors to return. */ + /** + * Maximum number of Actors to return. Sent verbatim by {@code list(...)} (so {@code 0} returns + * zero Actors); in {@code iterate(...)} a non-positive/zero {@code limit} means "no cap" (all). + */ public StoreListOptions limit(Long limit) { this.limit = limit; return this; From a701fc607d9fe31ff494b93ac9cf438a7d67ed2e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 09:11:49 +0000 Subject: [PATCH 21/37] docs: resolve 8 documentation review items at 8da2646 - Attribute checked IOException to transferTo(), not getStreamedLog() - Add "non-positive = no cap" nuance to all iterate Javadocs + doc tables - Reword README iteration note (drop "falsy"/untested server outcome) - Fix runs list signature note (RunListOptions) in README - Use absolute GitHub links for runnable example programs - Add return types to the TaskClient docs table - Note versions iterate ignores offset - Replace non-standard CHANGELOG "### Breaking" with Changed + bold prefix Documentation/Javadoc only; no runtime or public-interface change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 11 ++++------- docs/README.md | 12 +++++++----- docs/actors.md | 4 ++-- docs/builds.md | 2 +- docs/examples.md | 17 +++++++++-------- docs/misc.md | 2 +- docs/runs.md | 2 +- docs/schedules.md | 2 +- docs/storages.md | 6 +++--- docs/tasks.md | 8 ++++---- docs/webhooks.md | 4 ++-- .../client/AbstractWebhookCollectionClient.java | 4 ++-- .../com/apify/client/ActorCollectionClient.java | 2 +- .../client/ActorVersionCollectionClient.java | 3 ++- .../com/apify/client/BuildCollectionClient.java | 4 ++-- .../java/com/apify/client/DatasetClient.java | 4 ++-- .../apify/client/DatasetCollectionClient.java | 4 ++-- .../com/apify/client/KeyValueStoreClient.java | 5 +++-- .../client/KeyValueStoreCollectionClient.java | 4 ++-- .../client/RequestQueueCollectionClient.java | 4 ++-- .../com/apify/client/RunCollectionClient.java | 6 +++--- .../apify/client/ScheduleCollectionClient.java | 4 ++-- .../com/apify/client/StoreCollectionClient.java | 4 ++-- .../com/apify/client/TaskCollectionClient.java | 4 ++-- .../client/WebhookDispatchCollectionClient.java | 4 ++-- 25 files changed, 64 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ef5db..ebf012a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,13 +26,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). nullability/optionality on some response fields — already tolerated because the models use nullable boxed field types (a JSON `null` deserializes to `null`) and an optional field simply stays unset. - -### Breaking - -- `StoreCollectionClient.iterate` now takes `iterate(StoreListOptions, Long chunkSize)`, where the - options' `limit` is the total-items cap and `chunkSize` is the page size. Previously `limit` was - the per-page size. This aligns Store iteration with the reference client and the new collection - iterators. +- **Breaking:** `StoreCollectionClient.iterate` now takes `iterate(StoreListOptions, Long chunkSize)`, + where the options' `limit` is the total-items cap and `chunkSize` is the page size. Previously + `limit` was the per-page size. This aligns Store iteration with the reference client and the new + collection iterators. ## [0.1.3] - 2026-07-10 diff --git a/docs/README.md b/docs/README.md index 9644749..48ff665 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ This directory documents the public API of the Apify Java client, organized by r lists the available methods with their parameters and short snippets. The snippets are code fragments that assume a configured `client` and the imports listed below, not standalone `main` programs; for complete, runnable programs see [examples.md](examples.md) and -[`src/test/java/com/apify/client/examples/`](../src/test/java/com/apify/client/examples). For an +[`src/test/java/com/apify/client/examples/`](https://github.com/apify/apify-client-java/tree/master/src/test/java/com/apify/client/examples). For an overview, configuration, error handling and the full resource table, see the [top-level README](../README.md). @@ -84,8 +84,9 @@ PaginationList page = client.actors().list(options); ## Common list options — `ListOptions` -Most `list` methods (builds, runs, tasks, schedules, webhooks, Actor versions) take the shared -`ListOptions`, which carries the standard pagination/ordering controls. +Most `list` methods (builds, tasks, schedules, webhooks, Actor versions) take the shared +`ListOptions`, which carries the standard pagination/ordering controls. Runs additionally take a +`RunListOptions` status filter — `runs().list(ListOptions, RunListOptions)`; see [Runs](runs.md). | Method | Type | Meaning | |---|---|---| @@ -109,8 +110,9 @@ Each paginated collection also offers a lazy `Iterator` that fetches pages on de on the collection clients, `DatasetClient.iterateItems(...)`, and `KeyValueStoreClient.iterateKeys(...)` (request-queue requests use `RequestQueueClient.paginateRequests(...)`). The options' `limit` caps the **total** number of items yielded; `null`/unset — or a non-positive value such as `0` — means no cap, -so every item is yielded. (This differs from `list(...)`, which sends `limit=0` verbatim and returns -zero items; during iteration a falsy `limit` means "unbounded", matching the reference JS client.) The per-request page size is an optional +so every item is yielded. (This differs from `list(...)`, which sends `limit=0` to the server +verbatim rather than treating it as unbounded — the iteration behavior matches the reference JS +client.) The per-request page size is an optional trailing `chunkSize` argument: the per-resource tables below show the `chunkSize` form, and each iterator also has an overload that omits it (using the server's default page size). The page size does not change which items a collection iterator yields; note the one exception in diff --git a/docs/actors.md b/docs/actors.md index 8b3baef..711cc95 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -8,7 +8,7 @@ where `id` is an Actor ID or `username~name` (a `/` in the id is accepted and no | Method | Description | |---|---| | `list(ActorListOptions)` | List the account's Actors. Returns `PaginationList`. | -| `iterate(ActorListOptions, Long chunkSize)` | Lazy `Iterator` over all matches; the options' `limit` caps the total yielded, `chunkSize` sets the page size (both `null` = all / server default). | +| `iterate(ActorListOptions, Long chunkSize)` | Lazy `Iterator` over all matches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the page size (`null` = server default). | | `create(Object)` | Create a new Actor from a JSON-serializable definition. Returns `Actor`. | `ActorListOptions` adds `my(Boolean)` (only Actors owned by the current user) and @@ -91,7 +91,7 @@ and deletes a single version and exposes its environment variables. | Method | Description | |---|---| | `list(ListOptions)` | List the Actor's versions. Returns `PaginationList`. | -| `iterate(ListOptions)` | Lazy `Iterator` over all versions; `limit` caps the total. The versions endpoint is not paginated (one fetch returns every version), so there is no page size to tune. | +| `iterate(ListOptions)` | Lazy `Iterator` over all versions; `limit` caps the total (`null`/unset or non-positive = all). The versions endpoint is not paginated (one fetch returns every version), so `offset` has no effect and there is no page size to tune. | | `create(Object version)` | Create a version. Returns `ActorVersion`. | ### `ActorVersionClient` — `client.actor(id).version(v)` diff --git a/docs/builds.md b/docs/builds.md index 2e6417d..bf2d821 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -8,7 +8,7 @@ builds) and a single build with `client.build(id)`. | Method | Description | |---|---| | `list(ListOptions)` | List builds. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all builds; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | ## `BuildClient` diff --git a/docs/examples.md b/docs/examples.md index b60dfb0..31fbffb 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -2,10 +2,11 @@ Each example below is a code fragment that assumes a configured `client` and the imports listed in the [documentation index](README.md#imports-and-dependencies); it is not a standalone `main`. The -complete, runnable programs live under -[`src/test/java/com/apify/client/examples/`](../src/test/java/com/apify/client/examples) and are -executed end-to-end against the live API by the `Test examples` CI step (see `ExamplesTest`), so -they are guaranteed to stay runnable. +complete, standalone programs (with imports and a `main`) live under +[`src/test/java/com/apify/client/examples/`](https://github.com/apify/apify-client-java/tree/master/src/test/java/com/apify/client/examples) +and are executed end-to-end against the live API by the `Test examples` CI step (see `ExamplesTest`), +so they are guaranteed to stay runnable. The link points to the GitHub source so it resolves from the +rendered docs as well as the repository. ## Run a store Actor and read its default dataset @@ -112,10 +113,10 @@ while (shown < 5 && it.hasNext()) { ## Run an Actor with log redirection -`getStreamedLog()` and `InputStream.transferTo(...)` throw a checked `IOException`, so run this where -that exception is handled — the complete -[`LogRedirection.java`](../src/test/java/com/apify/client/examples/LogRedirection.java) program declares -`throws Exception`. +`InputStream.transferTo(...)` throws a checked `IOException`, so run this where that exception is +handled — the complete +[`LogRedirection.java`](https://github.com/apify/apify-client-java/blob/master/src/test/java/com/apify/client/examples/LogRedirection.java) +program declares `throws Exception`. ```java ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); diff --git a/docs/misc.md b/docs/misc.md index 3d6ee4c..3b8754f 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -7,7 +7,7 @@ Browse public Actors in the Apify Store. | Method | Description | |---|---| | `list(StoreListOptions)` | A page of Store Actors. Returns `PaginationList`. | -| `iterate(StoreListOptions, Long chunkSize)` | A lazy `Iterator` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null` = all); `chunkSize` is the per-request page size (`null` = server default). | +| `iterate(StoreListOptions, Long chunkSize)` | A lazy `Iterator` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null`/unset or non-positive = all); `chunkSize` is the per-request page size (`null` = server default). | `StoreListOptions` fields: `offset`, `limit`, `search`, `sortBy`, `category`, `username`, `pricingModel` (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`), diff --git a/docs/runs.md b/docs/runs.md index 9d0673e..4434de7 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -8,7 +8,7 @@ Access the run collection with `client.runs()` (or `client.actor(id).runs()` / | Method | Description | |---|---| | `list(ListOptions, RunListOptions)` | List runs. Returns `PaginationList`. | -| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterate(ListOptions, RunListOptions, Long chunkSize)` | Lazy `Iterator` over all matching runs; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | `RunListOptions` adds `status(List)` (e.g. `SUCCEEDED`, `RUNNING`; sent comma-separated) and, for Actor/task-scoped collections, `startedAfter(String)` / `startedBefore(String)` (ISO-8601). diff --git a/docs/schedules.md b/docs/schedules.md index d57a30b..97a64cb 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -8,7 +8,7 @@ Schedules automatically start Actor or task runs at specified times. Access the | Method | Description | |---|---| | `list(ListOptions)` | List schedules. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all schedules; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | | `create(Object)` | Create a schedule from a JSON-serializable definition. Returns `Schedule`. | The `actions` array describes what the schedule runs (e.g. a `RUN_ACTOR` action): diff --git a/docs/storages.md b/docs/storages.md index 1bc6fb1..fdff969 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -24,7 +24,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | Method | Description | |---|---| | `list(StorageListOptions)` | List datasets. Returns `PaginationList`. | -| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterate(StorageListOptions, Long chunkSize)` | Lazy `Iterator` over all datasets; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | | `getOrCreate(String name)` | Get or create a named dataset (empty name → unnamed). Returns `Dataset`. | | `getOrCreate(String name, Object schema)` | As above, sending a creation-time dataset `schema` when a new dataset is created. Returns `Dataset`. | @@ -37,7 +37,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | -| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | | `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | | `pushItems(Object)` | Push a single item or a list of items. No return value. | @@ -97,7 +97,7 @@ datasets. |---|---| | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listKeys(ListKeysOptions)` | List keys. Returns `KeyValueStoreKeysPage`. | -| `iterateKeys(ListKeysOptions)` / `iterateKeys(ListKeysOptions, Long chunkSize)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null` = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. `chunkSize` sets the per-request page size (`null` = server default). | +| `iterateKeys(ListKeysOptions)` / `iterateKeys(ListKeysOptions, Long chunkSize)` | Lazy `Iterator` over all keys, paging with the cursor (`exclusiveStartKey`). Note: here the options' `limit` caps the **total** number of keys yielded (`null`/unset or non-positive = all), whereas for `listKeys`/`createKeysPublicUrl` the same `ListKeysOptions.limit` is a single-request page size. `chunkSize` sets the per-request page size (`null` = server default). | | `recordExists(String key)` | Whether a record exists. | | `getRecord(String key)` / `getRecord(String key, GetRecordOptions)` | Fetch a record. Returns `Optional`. | | `setRecord(String key, byte[] value, String contentType)` | Store raw bytes. No return value. | diff --git a/docs/tasks.md b/docs/tasks.md index fb0b94d..56aa635 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -8,7 +8,7 @@ Tasks are pre-configured Actor runs with stored input. Access the task collectio | Method | Description | |---|---| | `list(ListOptions)` | List tasks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all tasks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | | `create(Object)` | Create a task from a JSON-serializable definition. Returns `Task`. | ```java @@ -23,9 +23,9 @@ Task task = client.tasks().create(Map.of( | Method | Description | |---|---| -| `get()` / `update(Object)` / `delete()` | CRUD. | -| `start(Object input, TaskStartOptions)` | Start a task run (input overrides stored input; `null` uses it). | -| `call(Object input, TaskStartOptions, Long waitSecs)` | Start and poll until finished. | +| `get()` / `update(Object)` / `delete()` | CRUD. Return `Optional` / `Task` / `void`. | +| `start(Object input, TaskStartOptions)` | Start a task run (input overrides stored input; `null` uses it). Returns `ActorRun`. | +| `call(Object input, TaskStartOptions, Long waitSecs)` | Start and poll until finished. Returns `ActorRun`. | | `getInput()` | The stored input. Returns `Optional`. | | `updateInput(Object)` | Replace the stored input. Returns `JsonNode`. | | `lastRun(String status)` / `lastRun(LastRunOptions)` | A `RunClient` for the last run (see [`LastRunOptions`](actors.md#actorclient)). | diff --git a/docs/webhooks.md b/docs/webhooks.md index e2c63b8..9aa51b1 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -12,7 +12,7 @@ The account-wide collection supports both listing and creation. | Method | Description | |---|---| | `list(ListOptions)` | List webhooks. Returns `PaginationList`. | -| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). Also available on the read-only nested collections. | +| `iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all webhooks; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). Also available on the read-only nested collections. | | `create(Object)` | Create a webhook. Returns `Webhook`. | ### Nested webhook collections (read-only) @@ -57,7 +57,7 @@ System.out.println(dispatch.getId()); | Method | Description | |---|---| | `webhookDispatches().list(ListOptions)` | List dispatches. Returns `PaginationList`. | -| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; the options' `limit` caps the total yielded (`null` = all), `chunkSize` sets the per-request page size (`null` = server default). | +| `webhookDispatches().iterate(ListOptions, Long chunkSize)` | Lazy `Iterator` over all dispatches; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | | `webhookDispatch(id).get()` | Fetch a dispatch. Returns `Optional`. | `WebhookDispatch` fields: `getId()`, `getWebhookId()`. diff --git a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java index 23e1153..0c88298 100644 --- a/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java +++ b/src/main/java/com/apify/client/AbstractWebhookCollectionClient.java @@ -24,8 +24,8 @@ public PaginationList list(ListOptions options) { /** * Returns a lazy iterator over the webhooks. The options' {@code limit} caps the total number - * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = - * server default). + * yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size + * ({@code null} = server default). */ public Iterator iterate(ListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/ActorCollectionClient.java b/src/main/java/com/apify/client/ActorCollectionClient.java index cebdc2b..799e2f2 100644 --- a/src/main/java/com/apify/client/ActorCollectionClient.java +++ b/src/main/java/com/apify/client/ActorCollectionClient.java @@ -20,7 +20,7 @@ public PaginationList list(ActorListOptions options) { /** * Returns a lazy iterator over the account's Actors, fetching pages on demand at the server's * default page size. The options' {@code limit} caps the total number of Actors yielded ({@code - * null} = all). + * null} or non-positive = all). */ public Iterator iterate(ActorListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/ActorVersionCollectionClient.java index 213c51a..14c2fc6 100644 --- a/src/main/java/com/apify/client/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/ActorVersionCollectionClient.java @@ -28,7 +28,8 @@ public PaginationList list(ListOptions options) { * limit paging engine would loop forever, since the server returns the same non-empty page at * every offset — this is why the sibling non-paginated {@code env-vars} collection is also a * single-fetch iterator.) The options' {@code limit} still caps the number yielded ({@code null} - * = all); there is no page size to tune. + * or non-positive = all); {@code offset} has no effect (the server ignores it) and there is no + * page size to tune. */ public Iterator iterate(ListOptions options) { ListOptions opts = options != null ? options : new ListOptions(); diff --git a/src/main/java/com/apify/client/BuildCollectionClient.java b/src/main/java/com/apify/client/BuildCollectionClient.java index 61f74d5..91d7656 100644 --- a/src/main/java/com/apify/client/BuildCollectionClient.java +++ b/src/main/java/com/apify/client/BuildCollectionClient.java @@ -22,8 +22,8 @@ public PaginationList list(ListOptions options) { /** * Returns a lazy iterator over the builds. The options' {@code limit} caps the total number - * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = - * server default). + * yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size + * ({@code null} = server default). */ public Iterator iterate(ListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 7e7ba51..00f2132 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -90,8 +90,8 @@ public Iterator iterateItems(DatasetListItemsOptions options) { /** * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, * fetching pages on demand. The options' {@code limit} caps the total number of items yielded - * ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = server - * default). + * ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size ({@code + * null} = server default). * *

Note: server-side item filters ({@code skipEmpty}, {@code skipHidden}, {@code clean}, {@code * simplified}) are applied after {@code offset}/{@code limit}, so a page can return fewer items diff --git a/src/main/java/com/apify/client/DatasetCollectionClient.java b/src/main/java/com/apify/client/DatasetCollectionClient.java index 692362b..bdd4aaa 100644 --- a/src/main/java/com/apify/client/DatasetCollectionClient.java +++ b/src/main/java/com/apify/client/DatasetCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(StorageListOptions options) { /** * Returns a lazy iterator over the datasets. The options' {@code limit} caps the total number - * yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code null} = - * server default). + * yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page size + * ({@code null} = server default). */ public Iterator iterate(StorageListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 3b4e3b3..839f656 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -59,8 +59,9 @@ public KeyValueStoreKeysPage listKeys(ListKeysOptions options) { /** * Returns a lazy iterator over this store's keys, fetching pages on demand via the cursor-based * ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} caps the total number - * of keys yielded ({@code null} = all); any {@code exclusiveStartKey} sets the starting point. - * The page size is the server default; use {@link #iterateKeys(ListKeysOptions, Long)} to set it. + * of keys yielded ({@code null} or non-positive = all); any {@code exclusiveStartKey} sets the + * starting point. The page size is the server default; use {@link #iterateKeys(ListKeysOptions, + * Long)} to set it. */ public Iterator iterateKeys(ListKeysOptions options) { return iterateKeys(options, null); diff --git a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java index 3b04cc7..7066cb0 100644 --- a/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(StorageListOptions options) { /** * Returns a lazy iterator over the key-value stores. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * number yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page + * size ({@code null} = server default). */ public Iterator iterate(StorageListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/RequestQueueCollectionClient.java b/src/main/java/com/apify/client/RequestQueueCollectionClient.java index fef3aad..b0ca489 100644 --- a/src/main/java/com/apify/client/RequestQueueCollectionClient.java +++ b/src/main/java/com/apify/client/RequestQueueCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(StorageListOptions options) { /** * Returns a lazy iterator over the request queues. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * number yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page + * size ({@code null} = server default). */ public Iterator iterate(StorageListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/RunCollectionClient.java b/src/main/java/com/apify/client/RunCollectionClient.java index a3d935d..101a5a2 100644 --- a/src/main/java/com/apify/client/RunCollectionClient.java +++ b/src/main/java/com/apify/client/RunCollectionClient.java @@ -31,9 +31,9 @@ public PaginationList list(ListOptions options, RunListOptions filter) /** * Returns a lazy iterator over the runs, applying the standard pagination and run-specific - * filters. The options' {@code limit} caps the total number yielded ({@code null} = all); {@code - * chunkSize} is the per-request page size ({@code null} = server default). Both {@code options} - * and {@code filter} may be {@code null}. + * filters. The options' {@code limit} caps the total number yielded ({@code null} or non-positive + * = all); {@code chunkSize} is the per-request page size ({@code null} = server default). Both + * {@code options} and {@code filter} may be {@code null}. */ public Iterator iterate(ListOptions options, RunListOptions filter) { return iterate(options, filter, null); diff --git a/src/main/java/com/apify/client/ScheduleCollectionClient.java b/src/main/java/com/apify/client/ScheduleCollectionClient.java index 55da65a..bcd3d61 100644 --- a/src/main/java/com/apify/client/ScheduleCollectionClient.java +++ b/src/main/java/com/apify/client/ScheduleCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(ListOptions options) { /** * Returns a lazy iterator over the account's schedules. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * number yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page + * size ({@code null} = server default). */ public Iterator iterate(ListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/StoreCollectionClient.java b/src/main/java/com/apify/client/StoreCollectionClient.java index 831c4e6..56cb5d5 100644 --- a/src/main/java/com/apify/client/StoreCollectionClient.java +++ b/src/main/java/com/apify/client/StoreCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(StoreListOptions options) { /** * Returns a lazy iterator over Store Actors matching the options, fetching pages on demand. The - * options' {@code limit} caps the total number of Actors yielded ({@code null} = all); {@code - * chunkSize} is the per-request page size ({@code null} = server default). + * options' {@code limit} caps the total number of Actors yielded ({@code null} or non-positive = + * all); {@code chunkSize} is the per-request page size ({@code null} = server default). */ public Iterator iterate(StoreListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/TaskCollectionClient.java b/src/main/java/com/apify/client/TaskCollectionClient.java index 698f0bd..392b9c5 100644 --- a/src/main/java/com/apify/client/TaskCollectionClient.java +++ b/src/main/java/com/apify/client/TaskCollectionClient.java @@ -19,8 +19,8 @@ public PaginationList list(ListOptions options) { /** * Returns a lazy iterator over the account's tasks. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * number yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page + * size ({@code null} = server default). */ public Iterator iterate(ListOptions options) { return iterate(options, null); diff --git a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java index 2e49fd4..2aa3ede 100644 --- a/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java +++ b/src/main/java/com/apify/client/WebhookDispatchCollectionClient.java @@ -22,8 +22,8 @@ public PaginationList list(ListOptions options) { /** * Returns a lazy iterator over the webhook dispatches. The options' {@code limit} caps the total - * number yielded ({@code null} = all); {@code chunkSize} is the per-request page size ({@code - * null} = server default). + * number yielded ({@code null} or non-positive = all); {@code chunkSize} is the per-request page + * size ({@code null} = server default). */ public Iterator iterate(ListOptions options) { return iterate(options, null); From 09b39b631278548cea94b4a5159bdfb7f85652cc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 10:03:00 +0000 Subject: [PATCH 22/37] docs: resolve documentation items from a701fc6 reviews - ActorVersionCollectionClient: iterator is eager (single fetch), not lazy - RunListOptions: correct startedAfter/startedBefore scoping per spec - Add experimental/AI-generated disclaimer to all resource doc pages - Retitle examples.md ("Examples") and stop overclaiming runnable - KeyValueStoreClient.iterateKeys: page size left to server, bounded by limit - Add DatasetClient.iterateItems(options, Class) convenience overload + docs - Move README imports note above the first quick-start fragment Docs/Javadoc plus one additive (non-breaking) overload; no behavior change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- README.md | 12 +++++----- docs/actors.md | 2 ++ docs/builds.md | 2 ++ docs/examples.md | 6 ++--- docs/misc.md | 2 ++ docs/runs.md | 2 ++ docs/schedules.md | 2 ++ docs/storages.md | 6 +++-- docs/tasks.md | 2 ++ docs/webhooks.md | 2 ++ .../client/ActorVersionCollectionClient.java | 3 ++- .../java/com/apify/client/DatasetClient.java | 8 +++++++ .../com/apify/client/KeyValueStoreClient.java | 4 ++-- .../java/com/apify/client/RunListOptions.java | 5 +++-- .../client/DatasetItemsIteratorTest.java | 22 +++++++++++++++++++ 15 files changed, 63 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 2c11b10..d1b91db 100644 --- a/README.md +++ b/README.md @@ -90,8 +90,11 @@ class HelloApify { } ``` -The remaining snippets below are fragments that assume a configured `client` (see the imports note -after the next block): +The remaining snippets below are fragments that assume a configured `client` and these imports: all +public client types live in the `com.apify.client` package (e.g. `import com.apify.client.*;`); the +snippets also use `com.fasterxml.jackson.databind.JsonNode` (from the Jackson dependency) for untyped +data, `java.time.Duration` in the configuration examples, and standard JDK types such as +`java.util.Optional` and `java.util.Map` (`import java.util.*;`). ```java ApifyClient client = ApifyClient.create("my-api-token"); @@ -106,11 +109,6 @@ PaginationList items = System.out.println("Items in this page: " + items.getCount()); ``` -All public client types live in the `com.apify.client` package (e.g. `import com.apify.client.*;`). -The snippets also use `com.fasterxml.jackson.databind.JsonNode` (from the Jackson dependency) for -untyped data, `java.time.Duration` in the configuration examples, and standard JDK types such as -`java.util.Optional` and `java.util.Map` (`import java.util.*;`). - `ApifyClient.create` takes the token as an explicit argument — it does **not** read `APIFY_TOKEN` (or any other environment variable) automatically. Read it yourself if you want that, e.g. `ApifyClient.create(System.getenv("APIFY_TOKEN"))`. diff --git a/docs/actors.md b/docs/actors.md index 711cc95..b6634d9 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -1,5 +1,7 @@ # Actors, versions & environment variables +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Access the Actor collection with `client.actors()` and a single Actor with `client.actor(id)`, where `id` is an Actor ID or `username~name` (a `/` in the id is accepted and normalized). diff --git a/docs/builds.md b/docs/builds.md index bf2d821..08de142 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -1,5 +1,7 @@ # Builds +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Access the build collection with `client.builds()` (or `client.actor(id).builds()` for an Actor's builds) and a single build with `client.build(id)`. diff --git a/docs/examples.md b/docs/examples.md index 31fbffb..47aa6a7 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,7 +1,7 @@ -# Runnable examples +# Examples -Each example below is a code fragment that assumes a configured `client` and the imports listed in -the [documentation index](README.md#imports-and-dependencies); it is not a standalone `main`. The +Each example below is a code fragment (not a standalone `main`) that assumes a configured `client` +and the imports listed in the [documentation index](README.md#imports-and-dependencies). The complete, standalone programs (with imports and a `main`) live under [`src/test/java/com/apify/client/examples/`](https://github.com/apify/apify-client-java/tree/master/src/test/java/com/apify/client/examples) and are executed end-to-end against the live API by the `Test examples` CI step (see `ExamplesTest`), diff --git a/docs/misc.md b/docs/misc.md index 3b8754f..ba9a098 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -1,5 +1,7 @@ # Store, users & logs +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + ## Apify Store — `client.store()` Browse public Actors in the Apify Store. diff --git a/docs/runs.md b/docs/runs.md index 4434de7..c1fdc75 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -1,5 +1,7 @@ # Runs +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Access the run collection with `client.runs()` (or `client.actor(id).runs()` / `client.task(id).runs()`) and a single run with `client.run(id)`. diff --git a/docs/schedules.md b/docs/schedules.md index 97a64cb..bb2b700 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -1,5 +1,7 @@ # Schedules +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Schedules automatically start Actor or task runs at specified times. Access the collection with `client.schedules()` and a single schedule with `client.schedule(id)`. diff --git a/docs/storages.md b/docs/storages.md index fdff969..928b0cf 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -1,5 +1,7 @@ # Storages: datasets, key-value stores, request queues +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + The three storage types share a consistent shape: a collection client (`list`, `getOrCreate`) and a single-resource client (`get`, `update`, `delete`, plus storage-specific operations). Run-nested default storages are reachable via `client.run(id).dataset()` / `.keyValueStore()` / @@ -37,8 +39,8 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `get()` / `update(Object)` / `delete()` | Metadata CRUD. | | `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | -| `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), `chunkSize` sets the per-request page size (`null` = server default). | -| `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | +| `iterateItems(DatasetListItemsOptions)` / `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), the optional `chunkSize` sets the per-request page size (omitted/`null` = server default). | +| `iterateItems(DatasetListItemsOptions, Class)` / `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | | `pushItems(Object)` | Push a single item or a list of items. No return value. | | `getStatistics()` | Dataset statistics. Returns `Optional`. | diff --git a/docs/tasks.md b/docs/tasks.md index 56aa635..a36905b 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -1,5 +1,7 @@ # Tasks +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Tasks are pre-configured Actor runs with stored input. Access the task collection with `client.tasks()` and a single task with `client.task(id)`. diff --git a/docs/webhooks.md b/docs/webhooks.md index 9aa51b1..014e0f1 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -1,5 +1,7 @@ # Webhooks & dispatches +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Webhooks notify an external service when specific events occur. Access the collection with `client.webhooks()` and a single webhook with `client.webhook(id)`. Dispatches (individual invocations) are available account-wide via `client.webhookDispatches()` / diff --git a/src/main/java/com/apify/client/ActorVersionCollectionClient.java b/src/main/java/com/apify/client/ActorVersionCollectionClient.java index 14c2fc6..5c4b936 100644 --- a/src/main/java/com/apify/client/ActorVersionCollectionClient.java +++ b/src/main/java/com/apify/client/ActorVersionCollectionClient.java @@ -19,7 +19,8 @@ public PaginationList list(ListOptions options) { } /** - * Returns a lazy iterator over the Actor's versions. + * Returns an iterator over the Actor's versions. Unlike the paginated collection iterators, this + * fetches eagerly — the single request runs when {@code iterate} is called (see below). * *

{@code GET /v2/actors/{actorId}/versions} is not offset/limit paginated: it takes * no pagination parameters and returns the full version list in a single {@code {total, items}} diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index 00f2132..e40432d 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -87,6 +87,14 @@ public Iterator iterateItems(DatasetListItemsOptions options) { return iterateItems(options, null, JsonNode.class); } + /** + * As {@link #iterateItems(DatasetListItemsOptions, Long, Class)} with the server-default page + * size — typed iteration without having to pass an explicit {@code null} chunk size. + */ + public Iterator iterateItems(DatasetListItemsOptions options, Class itemClass) { + return iterateItems(options, null, itemClass); + } + /** * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, * fetching pages on demand. The options' {@code limit} caps the total number of items yielded diff --git a/src/main/java/com/apify/client/KeyValueStoreClient.java b/src/main/java/com/apify/client/KeyValueStoreClient.java index 839f656..85c22fa 100644 --- a/src/main/java/com/apify/client/KeyValueStoreClient.java +++ b/src/main/java/com/apify/client/KeyValueStoreClient.java @@ -60,8 +60,8 @@ public KeyValueStoreKeysPage listKeys(ListKeysOptions options) { * Returns a lazy iterator over this store's keys, fetching pages on demand via the cursor-based * ({@code exclusiveStartKey}) listing endpoint. The options' {@code limit} caps the total number * of keys yielded ({@code null} or non-positive = all); any {@code exclusiveStartKey} sets the - * starting point. The page size is the server default; use {@link #iterateKeys(ListKeysOptions, - * Long)} to set it. + * starting point. The per-request page size is left to the server (bounded by any {@code limit} + * cap); use {@link #iterateKeys(ListKeysOptions, Long)} to set it explicitly. */ public Iterator iterateKeys(ListKeysOptions options) { return iterateKeys(options, null); diff --git a/src/main/java/com/apify/client/RunListOptions.java b/src/main/java/com/apify/client/RunListOptions.java index 3f21d70..6dbf16f 100644 --- a/src/main/java/com/apify/client/RunListOptions.java +++ b/src/main/java/com/apify/client/RunListOptions.java @@ -4,8 +4,9 @@ /** * Run-specific filters for {@link RunCollectionClient#list(ListOptions, RunListOptions)}. The - * {@code startedAfter}/{@code startedBefore} filters are only honoured by the Actor-scoped and - * task-scoped run collections. + * {@code startedAfter}/{@code startedBefore} filters are honoured by the top-level ({@code + * /v2/actor-runs}) and Actor-scoped ({@code /v2/acts/{actorId}/runs}) run collections; the + * task-scoped run collection accepts only {@code status}. */ public final class RunListOptions { private List status; diff --git a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java index f5e4a7c..6026dc6 100644 --- a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java +++ b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java @@ -47,6 +47,28 @@ void pagesBareArrayBodyAndStopsOnEmptyPage() { assertTrue(backend.lastUrl.contains("limit=2"), "chunkSize drives the per-request page size"); } + @Test + void typedIterateItemsDefaultPageSizeOverload() { + // iterateItems(options, Class) — typed iteration at the server-default page size, without + // having to pass an explicit null chunkSize. Decodes each item into the requested type. + MockBackend backend = + new MockBackend( + List.of(MockBackend.ok(200, "[{\"n\":1},{\"n\":2}]"), MockBackend.ok(200, "[]"))); + List seen = new ArrayList<>(); + Iterator it = + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), Row.class); + while (it.hasNext()) { + seen.add(it.next().n); + } + assertEquals(List.of(1, 2), seen, "typed default-page-size overload yields decoded items"); + assertTrue(backend.lastUrl.contains("datasets/d1/items"), backend.lastUrl); + } + + /** Minimal typed row for the typed-iteration overload test. */ + static final class Row { + public int n; + } + @Test void totalCapTrimsDatasetItems() { // The cap wins even though the server would return more; only the first page is requested. From b01549b46eb22b1715177b52d89bc489666a304f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 10:31:02 +0000 Subject: [PATCH 23/37] fix: revert ambiguous iterateItems overload; correct actor paths Revert the iterateItems(options, Class) overload re-added in 09b39b6: it reintroduces the iterateItems(opts, null) ambiguity that commit 02c977e removed. Typed default-page-size iteration uses the 3-arg form with a null chunk size (documented). Fix Javadoc/doc paths /v2/acts -> /v2/actors in RunListOptions and docs/webhooks.md. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/storages.md | 2 +- docs/webhooks.md | 2 +- src/main/java/com/apify/client/DatasetClient.java | 8 -------- src/main/java/com/apify/client/RunListOptions.java | 2 +- .../com/apify/client/DatasetItemsIteratorTest.java | 11 ++++++----- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/storages.md b/docs/storages.md index 928b0cf..be8a606 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -40,7 +40,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `listItems(DatasetListItemsOptions)` | List items as `PaginationList`. | | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | | `iterateItems(DatasetListItemsOptions)` / `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), the optional `chunkSize` sets the per-request page size (omitted/`null` = server default). | -| `iterateItems(DatasetListItemsOptions, Class)` / `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. | +| `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. For typed iteration at the server-default page size, pass a `null` chunk size: `iterateItems(opts, null, T.class)`. | | `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | | `pushItems(Object)` | Push a single item or a list of items. No return value. | | `getStatistics()` | Dataset statistics. Returns `Optional`. | diff --git a/docs/webhooks.md b/docs/webhooks.md index 014e0f1..edebc51 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -21,7 +21,7 @@ The account-wide collection supports both listing and creation. `client.actor(id).webhooks()` and `client.task(id).webhooks()` return a `NestedWebhookCollectionClient`. The Apify API only supports **reading** webhooks on those nested -paths (`GET /v2/acts/{id}/webhooks`, `GET /v2/actor-tasks/{id}/webhooks`), so this read-only type +paths (`GET /v2/actors/{id}/webhooks`, `GET /v2/actor-tasks/{id}/webhooks`), so this read-only type exposes `list(ListOptions)` and `iterate(ListOptions, Long chunkSize)` — it has no `create(...)`. To create a webhook targeting a specific Actor or task, use `client.webhooks().create(...)` and set the Actor/task in the webhook's `condition`. diff --git a/src/main/java/com/apify/client/DatasetClient.java b/src/main/java/com/apify/client/DatasetClient.java index e40432d..00f2132 100644 --- a/src/main/java/com/apify/client/DatasetClient.java +++ b/src/main/java/com/apify/client/DatasetClient.java @@ -87,14 +87,6 @@ public Iterator iterateItems(DatasetListItemsOptions options) { return iterateItems(options, null, JsonNode.class); } - /** - * As {@link #iterateItems(DatasetListItemsOptions, Long, Class)} with the server-default page - * size — typed iteration without having to pass an explicit {@code null} chunk size. - */ - public Iterator iterateItems(DatasetListItemsOptions options, Class itemClass) { - return iterateItems(options, null, itemClass); - } - /** * Returns a lazy iterator over the dataset's items, decoding each into {@code itemClass}, * fetching pages on demand. The options' {@code limit} caps the total number of items yielded diff --git a/src/main/java/com/apify/client/RunListOptions.java b/src/main/java/com/apify/client/RunListOptions.java index 6dbf16f..1322548 100644 --- a/src/main/java/com/apify/client/RunListOptions.java +++ b/src/main/java/com/apify/client/RunListOptions.java @@ -5,7 +5,7 @@ /** * Run-specific filters for {@link RunCollectionClient#list(ListOptions, RunListOptions)}. The * {@code startedAfter}/{@code startedBefore} filters are honoured by the top-level ({@code - * /v2/actor-runs}) and Actor-scoped ({@code /v2/acts/{actorId}/runs}) run collections; the + * /v2/actor-runs}) and Actor-scoped ({@code /v2/actors/{actorId}/runs}) run collections; the * task-scoped run collection accepts only {@code status}. */ public final class RunListOptions { diff --git a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java index 6026dc6..d51f1b7 100644 --- a/src/test/java/com/apify/client/DatasetItemsIteratorTest.java +++ b/src/test/java/com/apify/client/DatasetItemsIteratorTest.java @@ -48,19 +48,20 @@ void pagesBareArrayBodyAndStopsOnEmptyPage() { } @Test - void typedIterateItemsDefaultPageSizeOverload() { - // iterateItems(options, Class) — typed iteration at the server-default page size, without - // having to pass an explicit null chunkSize. Decodes each item into the requested type. + void typedIterateItemsAtDefaultPageSize() { + // Typed iteration at the server-default page size uses the 3-arg form with a null chunkSize. + // (No (options, Class) overload exists: it would make iterateItems(opts, null) ambiguous + // with (options, Long) — see commit 02c977e.) Decodes each item into the requested type. MockBackend backend = new MockBackend( List.of(MockBackend.ok(200, "[{\"n\":1},{\"n\":2}]"), MockBackend.ok(200, "[]"))); List seen = new ArrayList<>(); Iterator it = - client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), Row.class); + client(backend).dataset("d1").iterateItems(new DatasetListItemsOptions(), null, Row.class); while (it.hasNext()) { seen.add(it.next().n); } - assertEquals(List.of(1, 2), seen, "typed default-page-size overload yields decoded items"); + assertEquals(List.of(1, 2), seen, "typed iteration at default page size yields decoded items"); assertTrue(backend.lastUrl.contains("datasets/d1/items"), backend.lastUrl); } From 191c7701fe854374a8d7e6dc5392bcb829c739ca Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 11:40:51 +0000 Subject: [PATCH 24/37] docs: resolve documentation items from b01549b reviews - Add experimental/AI-generated disclaimer to docs/examples.md - Update inbound link text "Runnable examples" -> "Examples" - Reorder README quick-start (source before compile) and align classpaths - Enumerate DownloadItemsFormat constants in docs/storages.md - Name the public com.apify.client.Version class in README Versioning - Add first-mention pointer for types used in the README quick-start Documentation only; no code or public-interface change. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- README.md | 33 +++++++++++++++++++++------------ docs/README.md | 2 +- docs/examples.md | 2 ++ docs/storages.md | 2 +- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d1b91db..2cb004f 100644 --- a/README.md +++ b/README.md @@ -63,17 +63,7 @@ A complete, copy-pasteable first program (save as `HelloApify.java`). First scaf ``` -Then populate a `lib/` directory with the client and its runtime dependencies, and compile and run -against the JVM's `lib/*` classpath wildcard — quote it so the shell does not expand it: - -```bash -# 1. Collect apify-client and its runtime dependencies (Jackson, brotli4j codecs, …) into lib/. -mvn dependency:copy-dependencies -DoutputDirectory=lib -DincludeScope=runtime - -# 2. Compile and run. '.' is for the compiled HelloApify.class; lib/* is the JVM classpath wildcard. -javac -cp 'lib/*' HelloApify.java -java -cp '.:lib/*' HelloApify # Windows: java -cp ".;lib/*" HelloApify -``` +Create `HelloApify.java`: ```java import com.apify.client.ApifyClient; @@ -90,6 +80,18 @@ class HelloApify { } ``` +Then populate a `lib/` directory with the client and its runtime dependencies, and compile and run +against the JVM's `lib/*` classpath wildcard — quote it so the shell does not expand it: + +```bash +# 1. Collect apify-client and its runtime dependencies (Jackson, brotli4j codecs, …) into lib/. +mvn dependency:copy-dependencies -DoutputDirectory=lib -DincludeScope=runtime + +# 2. Compile and run. '.' is for the compiled HelloApify.class; lib/* is the JVM classpath wildcard. +javac -cp '.:lib/*' HelloApify.java # Windows: javac -cp ".;lib/*" HelloApify.java +java -cp '.:lib/*' HelloApify # Windows: java -cp ".;lib/*" HelloApify +``` + The remaining snippets below are fragments that assume a configured `client` and these imports: all public client types live in the `com.apify.client` package (e.g. `import com.apify.client.*;`); the snippets also use `com.fasterxml.jackson.databind.JsonNode` (from the Jackson dependency) for untyped @@ -109,6 +111,10 @@ PaginationList items = System.out.println("Items in this page: " + items.getCount()); ``` +The types used above — `PaginationList`, `DatasetListItemsOptions`, and the per-resource clients — +are documented on the [resource pages](docs/README.md); `ApifyApiException` is covered under +[Error handling](#error-handling) below. + `ApifyClient.create` takes the token as an explicit argument — it does **not** read `APIFY_TOKEN` (or any other environment variable) automatically. Read it yourself if you want that, e.g. `ApifyClient.create(System.getenv("APIFY_TOKEN"))`. @@ -180,6 +186,9 @@ try { ## Versioning +The public `com.apify.client.Version` class (`import com.apify.client.Version;`) exposes two +constants: + - `Version.CLIENT_VERSION` — the semantic version of this client (`0.2.0`). - `Version.API_SPEC_VERSION` — the Apify OpenAPI specification version this client was verified against (`v2-2026-07-10T105921Z`). @@ -221,7 +230,7 @@ Full documentation is in the [`docs/`](docs/README.md) directory, organized by r - [Schedules](docs/schedules.md) - [Webhooks & dispatches](docs/webhooks.md) - [Store, users & logs](docs/misc.md) -- [Runnable examples](docs/examples.md) +- [Examples](docs/examples.md) ## Resources diff --git a/docs/README.md b/docs/README.md index 48ff665..91e407e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -129,4 +129,4 @@ can affect the result. - [Schedules](schedules.md) - [Webhooks & dispatches](webhooks.md) - [Store, users & logs](misc.md) -- [Runnable examples](examples.md) +- [Examples](examples.md) diff --git a/docs/examples.md b/docs/examples.md index 47aa6a7..c9cc3a6 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,5 +1,7 @@ # Examples +> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. + Each example below is a code fragment (not a standalone `main`) that assumes a configured `client` and the imports listed in the [documentation index](README.md#imports-and-dependencies). The complete, standalone programs (with imports and a `main`) live under diff --git a/docs/storages.md b/docs/storages.md index be8a606..1036c06 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -41,7 +41,7 @@ getters `getId()`, `getName()`, `getUserId()`, `getCreatedAt()` (`Instant`), and | `listItems(DatasetListItemsOptions, Class)` | List items decoded into `T`. Returns `PaginationList`. | | `iterateItems(DatasetListItemsOptions)` / `iterateItems(DatasetListItemsOptions, Long chunkSize)` | Lazy `Iterator` over all items; the options' `limit` caps the total yielded (`null`/unset or non-positive = all), the optional `chunkSize` sets the per-request page size (omitted/`null` = server default). | | `iterateItems(DatasetListItemsOptions, Long chunkSize, Class)` | As above, decoded into `T`. Returns `Iterator`. For typed iteration at the server-default page size, pass a `null` chunk size: `iterateItems(opts, null, T.class)`. | -| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes (JSON/JSONL/CSV/XLSX/XML/RSS/HTML). | +| `downloadItems(DownloadItemsFormat, DatasetDownloadOptions)` | Serialized bytes. `DownloadItemsFormat` is one of `JSON`, `JSONL`, `CSV`, `XLSX`, `XML`, `RSS`, `HTML`. | | `pushItems(Object)` | Push a single item or a list of items. No return value. | | `getStatistics()` | Dataset statistics. Returns `Optional`. | | `createItemsPublicUrl(DatasetListItemsOptions, Long expiresInSecs)` | A public (optionally signed) items URL. | From 8a08622b67e4569b1bb8c75cca1d2925d243b3d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 12:16:25 +0000 Subject: [PATCH 25/37] docs: scope PaginatedIterator class-doc reuse claim accurately The engine backs every offset/limit-paginated collection's iterate method; the cursor iterateKeys and single-fetch versions/env-vars iterators do not use it. Comment only. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/main/java/com/apify/client/PaginatedIterator.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/apify/client/PaginatedIterator.java b/src/main/java/com/apify/client/PaginatedIterator.java index 9f37440..dd990ba 100644 --- a/src/main/java/com/apify/client/PaginatedIterator.java +++ b/src/main/java/com/apify/client/PaginatedIterator.java @@ -6,8 +6,9 @@ /** * A lazy {@link Iterator} over an offset/limit-paginated list endpoint, fetching one page at a - * time. Internal reusable engine shared by every collection client's {@code iterate} method, so the - * paging arithmetic lives in one place (DRY). + * time. Internal reusable engine shared by every offset/limit-paginated collection's {@code + * iterate} method, so the paging arithmetic lives in one place (DRY). (The cursor-based {@code + * iterateKeys} and the single-fetch {@code versions}/{@code env-vars} iterators do not use it.) * *

Behaviour, mirroring the end-user contract of the reference JS client's iterable {@code * list()}: From 640402fdafadf8b23aa8d9a0f07cb51557c8a3bc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 13:12:10 +0000 Subject: [PATCH 26/37] feat: add StreamedLog log-redirection helper and standardize doc disclaimer Implement a StreamedLog helper (start/stop lifecycle, line-aware buffering, fromStart/toLog options, default per-run prefixed logger) matching the reference JS run.getStreamedLog(), returned by RunClient.getStreamedLog(). Update example (g), docs, and add hermetic + integration test coverage. Standardize the "official, but experimental" disclaimer across all doc pages. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 19 ++ README.md | 8 +- docs/actors.md | 4 +- docs/builds.md | 4 +- docs/examples.md | 19 +- docs/misc.md | 4 +- docs/runs.md | 37 ++- docs/schedules.md | 4 +- docs/storages.md | 4 +- docs/tasks.md | 4 +- docs/webhooks.md | 4 +- pom.xml | 2 +- src/main/java/com/apify/client/RunClient.java | 68 ++++- .../java/com/apify/client/StreamedLog.java | 250 ++++++++++++++++++ .../com/apify/client/StreamedLogOptions.java | 58 ++++ src/main/java/com/apify/client/Version.java | 2 +- .../com/apify/client/StreamedLogTest.java | 134 ++++++++++ .../apify/client/examples/LogRedirection.java | 17 +- .../integration/ActorRunIntegrationTest.java | 15 ++ 19 files changed, 618 insertions(+), 39 deletions(-) create mode 100644 src/main/java/com/apify/client/StreamedLog.java create mode 100644 src/main/java/com/apify/client/StreamedLogOptions.java create mode 100644 src/test/java/com/apify/client/StreamedLogTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf012a..65673fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ All notable changes to the Apify Java client are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] - 2026-07-11 + +### Added + +- `StreamedLog` log-redirection helper (matching the reference client's `getStreamedLog`): + `RunClient.getStreamedLog()` / `getStreamedLog(StreamedLogOptions)` return a `StreamedLog` that + follows the run's live log in a background thread and redirects each complete, timestamped message + to a destination. `StreamedLog` is `AutoCloseable` with `start()`/`stop()` lifecycle. Options: + `toLog(Consumer)` (custom destination; default is a per-run prefixed `java.util.logging` + logger), `prefix(String)`, and `fromStart(boolean)` (skip pre-redirection log lines when false). + +### Changed + +- **Breaking:** `RunClient.getStreamedLog()` now returns a `StreamedLog` redirection helper instead + of a raw `InputStream`, aligning its public interface with the reference client. For raw stream + access use `run(id).log().stream(new LogOptions().raw(true))`. +- Standardized the "official, but experimental" disclaimer wording across the README and all + documentation pages. + ## [0.2.0] - 2026-07-10 ### Added diff --git a/README.md b/README.md index 2cb004f..12aabfc 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Maven (Maven Central is a default repository, so no extra configuration is neede com.apify apify-client - 0.2.0 + 0.3.0 ``` @@ -35,7 +35,7 @@ repositories { } dependencies { - implementation 'com.apify:apify-client:0.2.0' + implementation 'com.apify:apify-client:0.3.0' } ``` @@ -57,7 +57,7 @@ A complete, copy-pasteable first program (save as `HelloApify.java`). First scaf com.apify apify-client - 0.2.0 + 0.3.0 @@ -189,7 +189,7 @@ try { The public `com.apify.client.Version` class (`import com.apify.client.Version;`) exposes two constants: -- `Version.CLIENT_VERSION` — the semantic version of this client (`0.2.0`). +- `Version.CLIENT_VERSION` — the semantic version of this client (`0.3.0`). - `Version.API_SPEC_VERSION` — the Apify OpenAPI specification version this client was verified against (`v2-2026-07-10T105921Z`). diff --git a/docs/actors.md b/docs/actors.md index b6634d9..7d5cd16 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -1,6 +1,8 @@ # Actors, versions & environment variables -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Access the Actor collection with `client.actors()` and a single Actor with `client.actor(id)`, where `id` is an Actor ID or `username~name` (a `/` in the id is accepted and normalized). diff --git a/docs/builds.md b/docs/builds.md index 08de142..8854024 100644 --- a/docs/builds.md +++ b/docs/builds.md @@ -1,6 +1,8 @@ # Builds -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Access the build collection with `client.builds()` (or `client.actor(id).builds()` for an Actor's builds) and a single build with `client.build(id)`. diff --git a/docs/examples.md b/docs/examples.md index c9cc3a6..a84a6ab 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -1,6 +1,8 @@ # Examples -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Each example below is a code fragment (not a standalone `main`) that assumes a configured `client` and the imports listed in the [documentation index](README.md#imports-and-dependencies). The @@ -115,14 +117,17 @@ while (shown < 5 && it.hasNext()) { ## Run an Actor with log redirection -`InputStream.transferTo(...)` throws a checked `IOException`, so run this where that exception is -handled — the complete -[`LogRedirection.java`](https://github.com/apify/apify-client-java/blob/master/src/test/java/com/apify/client/examples/LogRedirection.java) -program declares `throws Exception`. +`getStreamedLog()` returns a [`StreamedLog`](runs.md#streamed-log-redirection) that follows the +run's live log and redirects each message to a destination (by default a per-run prefixed logger). +It is `AutoCloseable`, so a try-with-resources block stops redirection when the run finishes. The +complete program lives in +[`LogRedirection.java`](https://github.com/apify/apify-client-java/blob/master/src/test/java/com/apify/client/examples/LogRedirection.java). ```java ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); -try (InputStream stream = client.run(run.getId()).getStreamedLog()) { - stream.transferTo(System.out); +RunClient runClient = client.run(run.getId()); +try (StreamedLog streamedLog = runClient.getStreamedLog()) { + streamedLog.start(); + runClient.waitForFinish(120L); } ``` diff --git a/docs/misc.md b/docs/misc.md index ba9a098..be90ba6 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -1,6 +1,8 @@ # Store, users & logs -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. ## Apify Store — `client.store()` diff --git a/docs/runs.md b/docs/runs.md index c1fdc75..9f25d9c 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -1,6 +1,8 @@ # Runs -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Access the run collection with `client.runs()` (or `client.actor(id).runs()` / `client.task(id).runs()`) and a single run with `client.run(id)`. @@ -37,10 +39,37 @@ PaginationList runs = client.runs().list( | `waitForFinish(Long waitSecs)` | Poll until the run finishes (`null` waits indefinitely). Returns `ActorRun`. | | `dataset()` / `keyValueStore()` / `requestQueue()` | Clients for the run's default storages. | | `log()` | A `LogClient` for the run's log (see [Store, users & logs](misc.md#logs--clientlogid)). | -| `getStreamedLog()` | A live raw log `InputStream` (for log redirection). | +| `getStreamedLog()` | A `StreamedLog` that redirects the live log to a default per-run logger. | +| `getStreamedLog(StreamedLogOptions)` | As above, with a custom destination / options. | -`getStreamedLog()` is a convenience equivalent to `run(id).log().stream(new LogOptions().raw(true))`; -use `log()` for the full log text or for non-raw/download options. +### Streamed log redirection + +`getStreamedLog()` returns a `StreamedLog`: a helper that follows the run's live raw log in a +background thread and redirects each complete, timestamped message to a destination. It is +`AutoCloseable` — call `start()` to begin redirection and `stop()` (or `close()`, via +try-with-resources) to end it. + +With no options, messages go to a `java.util.logging.Logger` at `INFO` level, prefixed with the +Actor name and run id (looked up automatically). `StreamedLogOptions` customizes it: + +- `toLog(Consumer)` — send each complete message to your own consumer instead of the + default logger. +- `prefix(String)` — override the auto-built prefix used by the default logger. +- `fromStart(boolean)` — when `false`, skip log lines produced before redirection started + (default `true`). + +```java +RunClient runClient = client.run("run-id"); +List collected = new ArrayList<>(); +try (StreamedLog streamedLog = + runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add).fromStart(true))) { + streamedLog.start(); + runClient.waitForFinish(120L); +} +``` + +For raw stream access without redirection, use `log().stream(new LogOptions().raw(true))`; use +`log()` for the full log text or for non-raw/download options. To set the current run's status message from inside an Actor, use the top-level `client.setStatusMessage(...)` (see [the docs index](README.md#setting-single-resource-status)). diff --git a/docs/schedules.md b/docs/schedules.md index bb2b700..d820b6d 100644 --- a/docs/schedules.md +++ b/docs/schedules.md @@ -1,6 +1,8 @@ # Schedules -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Schedules automatically start Actor or task runs at specified times. Access the collection with `client.schedules()` and a single schedule with `client.schedule(id)`. diff --git a/docs/storages.md b/docs/storages.md index 1036c06..54c981d 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -1,6 +1,8 @@ # Storages: datasets, key-value stores, request queues -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. The three storage types share a consistent shape: a collection client (`list`, `getOrCreate`) and a single-resource client (`get`, `update`, `delete`, plus storage-specific operations). Run-nested diff --git a/docs/tasks.md b/docs/tasks.md index a36905b..9fe6f19 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -1,6 +1,8 @@ # Tasks -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Tasks are pre-configured Actor runs with stored input. Access the task collection with `client.tasks()` and a single task with `client.task(id)`. diff --git a/docs/webhooks.md b/docs/webhooks.md index edebc51..44236e3 100644 --- a/docs/webhooks.md +++ b/docs/webhooks.md @@ -1,6 +1,8 @@ # Webhooks & dispatches -> **Official but experimental — AI-generated and AI-maintained.** Review the code before relying on it in production. +> **Official, but experimental — AI-generated and AI-maintained.** This is an official Apify client, +> but it is experimental: it is generated and maintained by AI. Review the code before relying on it +> in production and report issues on the repository. Webhooks notify an external service when specific events occur. Access the collection with `client.webhooks()` and a single webhook with `client.webhook(id)`. Dispatches (individual diff --git a/pom.xml b/pom.xml index 7f831ed..91e1476 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ com.apify apify-client - 0.2.0 + 0.3.0 jar Apify Java Client diff --git a/src/main/java/com/apify/client/RunClient.java b/src/main/java/com/apify/client/RunClient.java index 374e77f..37709dd 100644 --- a/src/main/java/com/apify/client/RunClient.java +++ b/src/main/java/com/apify/client/RunClient.java @@ -1,10 +1,11 @@ package com.apify.client; -import java.io.InputStream; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Consumer; +import java.util.logging.Logger; /** * A client for a specific Actor run. @@ -182,15 +183,66 @@ public LogClient log() { return LogClient.nested(ctx.http, ctx.subUrl(""), ctx.baseParams); } + /** Logger name used by the default log-redirection destination. */ + private static final String REDIRECT_LOGGER_NAME = "com.apify.client.ActorRunLog"; + + /** + * Returns a {@link StreamedLog} that redirects this run's live log to a default per-run logger. + * + *

The default destination is a {@link java.util.logging.Logger} that receives each message at + * {@code INFO} level, prefixed with the Actor name and run id (built by fetching the run and its + * Actor). Call {@link StreamedLog#start()} to begin redirection and {@link StreamedLog#stop()} + * (or {@link StreamedLog#close()}) to end it. + * + *

For a custom destination or to skip historical log lines, use {@link + * #getStreamedLog(StreamedLogOptions)}. For raw stream access without redirection, use {@link + * #log()}{@code .stream(...)}. + */ + public StreamedLog getStreamedLog() { + return getStreamedLog(new StreamedLogOptions()); + } + /** - * Opens a live stream of this run's raw log, for convenient log redirection. The caller must - * close the returned stream. + * Returns a {@link StreamedLog} that redirects this run's live log according to {@code options}. * - *

This is a shorthand for the common raw-log case. For full control over the streamed log - * (e.g. non-raw content or a download disposition), use {@link #log()}{@code .stream(options)} - * directly with a {@link LogOptions}. + *

If {@link StreamedLogOptions#toLog(Consumer)} is set, each complete log message is passed to + * that consumer; otherwise a default {@link java.util.logging.Logger} destination is used with a + * per-run prefix (an explicit {@link StreamedLogOptions#prefix(String)} overrides the auto-built + * one). {@link StreamedLogOptions#fromStart(boolean)} controls whether log lines produced before + * redirection started are included. */ - public InputStream getStreamedLog() { - return log().stream(new LogOptions().raw(true)); + public StreamedLog getStreamedLog(StreamedLogOptions options) { + Consumer destination = options.destination(); + if (destination == null) { + String prefix = + options.prefixValue() != null ? options.prefixValue() : buildDefaultLogPrefix(); + destination = defaultLogDestination(prefix); + } + return new StreamedLog(log(), destination, options.fromStartValue()); + } + + /** + * Builds the per-run prefix used by the default redirection destination, mirroring the reference + * client: the Actor name (looked up from the run) and {@code runId:{id}}, joined with a space and + * followed by {@code " -> "}. Falls back to just the run id when the Actor name is unavailable. + */ + private String buildDefaultLogPrefix() { + String actorName = ""; + Optional run = get(); + if (run.isPresent() && run.get().getActId() != null && !run.get().getActId().isEmpty()) { + Optional actor = root.actor(run.get().getActId()).get(); + if (actor.isPresent() && actor.get().getName() != null) { + actorName = actor.get().getName(); + } + } + String runPart = "runId:" + id; + String name = actorName.isEmpty() ? runPart : actorName + " " + runPart; + return name + " -> "; + } + + /** A destination that logs each message at {@code INFO} level, prefixed with {@code prefix}. */ + private static Consumer defaultLogDestination(String prefix) { + Logger logger = Logger.getLogger(REDIRECT_LOGGER_NAME); + return message -> logger.info(prefix + message); } } diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java new file mode 100644 index 0000000..2de5a78 --- /dev/null +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -0,0 +1,250 @@ +package com.apify.client; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Helper for redirecting a run's (or build's) streamed log to another destination, mirroring the + * reference client's {@code StreamedLog}. + * + *

It follows the live raw log stream in the background and forwards each complete, timestamped + * log message to a destination {@link Consumer} (by default a per-run prefixed {@link Logger}). + * Messages are parsed on log-line boundaries (the Apify platform prefixes every line with an + * ISO-8601 timestamp), so multi-line messages are kept intact across stream chunks. + * + *

Lifecycle: call {@link #start()} to begin redirection and {@link #stop()} to end it. The class + * is {@link AutoCloseable}, so it can be used in a try-with-resources block; {@link #close()} stops + * redirection if it is still running. + * + *

Typical use: + * + * 

{@code
+ * RunClient runClient = client.run(runId);
+ * try (StreamedLog streamedLog = runClient.getStreamedLog()) {
+ *   streamedLog.start();
+ *   runClient.waitForFinish(120L);
+ * }
+ * }
+ */ +public final class StreamedLog implements AutoCloseable { + + /** + * Marks the start of a log message: an ISO-8601 timestamp at the beginning of a line (or of the + * stream). The platform prefixes every log line with such a timestamp, so this reliably splits + * concatenated messages while keeping multi-line messages together. + */ + private static final Pattern MESSAGE_MARKER = + Pattern.compile("(?:\\n|^)(\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z)"); + + /** Read buffer size for pulling bytes off the live log stream. */ + private static final int READ_BUFFER_BYTES = 8192; + + private final LogClient logClient; + private final Consumer destination; + + /** + * Messages older than this instant are dropped ({@code null} means "redirect from the start", so + * nothing is dropped). Set to construction time when {@code fromStart} is {@code false}. + */ + private final Instant relevancyTimeLimit; + + /** Decoded log text seen so far but not yet split into complete messages. */ + private final StringBuilder pending = new StringBuilder(); + + private volatile boolean stopLogging; + private volatile InputStream activeStream; + private Thread streamingThread; + + StreamedLog(LogClient logClient, Consumer destination, boolean fromStart) { + this.logClient = logClient; + this.destination = destination; + this.relevancyTimeLimit = fromStart ? null : Instant.now(); + } + + /** + * Starts redirecting the log in a background daemon thread. + * + * @throws IllegalStateException if redirection is already running + */ + public synchronized void start() { + if (streamingThread != null) { + throw new IllegalStateException("Streaming task already active"); + } + stopLogging = false; + pending.setLength(0); + Thread thread = new Thread(this::streamLog, "apify-streamed-log"); + thread.setDaemon(true); + streamingThread = thread; + thread.start(); + } + + /** + * Stops log redirection and waits for the background thread to finish. + * + * @throws IllegalStateException if redirection is not running + */ + public synchronized void stop() { + if (streamingThread == null) { + throw new IllegalStateException("Streaming task is not active"); + } + stopLogging = true; + // Close the live stream so a blocked read returns promptly instead of waiting for more bytes. + closeQuietly(activeStream); + try { + streamingThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + streamingThread = null; + } + } + + /** Stops redirection if it is still running; a no-op otherwise. */ + @Override + public void close() { + synchronized (this) { + if (streamingThread == null) { + return; + } + } + stop(); + } + + /** Reads the live raw log stream and forwards complete messages to the destination. */ + private void streamLog() { + try (InputStream stream = logClient.stream(new LogOptions().raw(true))) { + activeStream = stream; + byte[] readBuffer = new byte[READ_BUFFER_BYTES]; + // Bytes after the last newline: an incomplete trailing line kept for the next read so a + // message split across chunk boundaries (including a partial UTF-8 sequence) is not + // corrupted. + byte[] lineRemainder = new byte[0]; + int read; + // Process each chunk before honouring a stop request, so a stop issued right after start + // still redirects whatever the run has already produced (matching the reference client). + while ((read = stream.read(readBuffer)) != -1) { + byte[] combined = new byte[lineRemainder.length + read]; + System.arraycopy(lineRemainder, 0, combined, 0, lineRemainder.length); + System.arraycopy(readBuffer, 0, combined, lineRemainder.length, read); + + int lastNewline = lastIndexOf(combined, (byte) '\n'); + if (lastNewline >= 0) { + String completeText = new String(combined, 0, lastNewline + 1, StandardCharsets.UTF_8); + lineRemainder = new byte[combined.length - (lastNewline + 1)]; + System.arraycopy(combined, lastNewline + 1, lineRemainder, 0, lineRemainder.length); + emitMessages(completeText, false); + } else { + lineRemainder = combined; + } + if (stopLogging) { + break; + } + } + // Flush whatever is left when the stream ends (or redirection is stopped): the last message + // may have no trailing newline. + emitMessages(new String(lineRemainder, StandardCharsets.UTF_8), true); + } catch (IOException e) { + // A read error after an explicit stop is expected (the stream was closed under us). Surface + // only genuine, unsolicited failures, and do so without throwing from the background thread. + if (!stopLogging) { + Logger.getLogger(StreamedLog.class.getName()) + .log(Level.WARNING, "Log redirection stopped due to error", e); + } + } finally { + closeQuietly(activeStream); + } + } + + /** + * Appends {@code text} to the pending buffer, then forwards every complete message it contains. + * When {@code flush} is {@code false} the last message is held back (it may still be growing); + * when {@code true} everything remaining is emitted. + */ + private void emitMessages(String text, boolean flush) { + pending.append(text); + String buffered = pending.toString(); + + Matcher matcher = MESSAGE_MARKER.matcher(buffered); + List messageStarts = new ArrayList<>(); + List timestamps = new ArrayList<>(); + while (matcher.find()) { + messageStarts.add(matcher.start(1)); + timestamps.add(matcher.group(1)); + } + + if (messageStarts.isEmpty()) { + // No complete message yet; keep buffering unless this is the final flush. + if (flush) { + emitIfRelevant(buffered.trim(), null); + pending.setLength(0); + } + return; + } + + int completeCount = flush ? messageStarts.size() : messageStarts.size() - 1; + for (int i = 0; i < completeCount; i++) { + int from = messageStarts.get(i); + int to = (i + 1 < messageStarts.size()) ? messageStarts.get(i + 1) : buffered.length(); + emitIfRelevant(buffered.substring(from, to).trim(), timestamps.get(i)); + } + + if (flush) { + pending.setLength(0); + } else { + // Retain the last (possibly still-growing) message for the next round. + String rest = buffered.substring(messageStarts.get(messageStarts.size() - 1)); + pending.setLength(0); + pending.append(rest); + } + } + + /** + * Forwards a trimmed message to the destination, dropping it if it is empty or (when a relevancy + * limit is set) older than that limit. + */ + private void emitIfRelevant(String message, String timestamp) { + if (message.isEmpty()) { + return; + } + if (relevancyTimeLimit != null && timestamp != null) { + try { + if (Instant.parse(timestamp).isBefore(relevancyTimeLimit)) { + return; + } + } catch (DateTimeParseException ignored) { + // Unparseable timestamp: keep the message rather than silently dropping log output. + } + } + destination.accept(message); + } + + private static int lastIndexOf(byte[] bytes, byte target) { + for (int i = bytes.length - 1; i >= 0; i--) { + if (bytes[i] == target) { + return i; + } + } + return -1; + } + + private static void closeQuietly(InputStream stream) { + if (stream == null) { + return; + } + try { + stream.close(); + } catch (IOException ignored) { + // Best-effort close; nothing actionable if it fails. + } + } +} diff --git a/src/main/java/com/apify/client/StreamedLogOptions.java b/src/main/java/com/apify/client/StreamedLogOptions.java new file mode 100644 index 0000000..8d1c8d0 --- /dev/null +++ b/src/main/java/com/apify/client/StreamedLogOptions.java @@ -0,0 +1,58 @@ +package com.apify.client; + +import java.util.function.Consumer; + +/** + * Configures {@link RunClient#getStreamedLog(StreamedLogOptions)} log redirection. + * + *

Mirrors the reference client's {@code getStreamedLog} options: a destination log ({@code + * toLog}) and whether to include messages produced before redirection started ({@code fromStart}). + */ +public final class StreamedLogOptions { + + private Consumer toLog; + private String prefix; + private boolean fromStart = true; + + /** + * Destination for redirected log messages. Each complete log message is passed to the consumer. + * When left unset, messages go to a {@link java.util.logging.Logger} at {@code INFO} level with + * an auto-built per-run prefix. + */ + public StreamedLogOptions toLog(Consumer toLog) { + this.toLog = toLog; + return this; + } + + /** + * Prefix prepended to each message by the default destination logger. Ignored when a custom + * {@link #toLog(Consumer)} destination is set. When unset, a per-run prefix (Actor name and run + * id) is built automatically. + */ + public StreamedLogOptions prefix(String prefix) { + this.prefix = prefix; + return this; + } + + /** + * Whether to redirect all of the run's logs, including those produced before redirection started + * ({@code true}, the default). When {@code false}, only messages timestamped at or after the + * moment redirection is created are redirected. + */ + public StreamedLogOptions fromStart(boolean fromStart) { + this.fromStart = fromStart; + return this; + } + + Consumer destination() { + return toLog; + } + + String prefixValue() { + return prefix; + } + + boolean fromStartValue() { + return fromStart; + } +} diff --git a/src/main/java/com/apify/client/Version.java b/src/main/java/com/apify/client/Version.java index aa938d4..c3333a0 100644 --- a/src/main/java/com/apify/client/Version.java +++ b/src/main/java/com/apify/client/Version.java @@ -13,7 +13,7 @@ public final class Version { * The semantic version of this client library (see SemVer). * Changes to the public interface other than additive ones are considered breaking changes. */ - public static final String CLIENT_VERSION = "0.2.0"; + public static final String CLIENT_VERSION = "0.3.0"; /** * The version of the Apify OpenAPI specification this client was generated and verified against. diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java new file mode 100644 index 0000000..b1dcdb7 --- /dev/null +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -0,0 +1,134 @@ +package com.apify.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.junit.jupiter.api.Test; + +/** + * Offline tests for the {@link StreamedLog} log-redirection helper: it must forward complete + * timestamped messages to the destination, drop messages older than the relevancy limit when {@code + * fromStart} is false, flush a final unterminated line, and enforce its start/stop lifecycle. + */ +class StreamedLogTest { + + private static ApifyClient client(MockBackend backend) { + return ApifyClient.builder().token("test-token").httpBackend(backend).maxRetries(0).build(); + } + + /** Drives a run's log stream from a scripted body and collects redirected messages. */ + private static List redirect(String streamBody, StreamedLogOptions options) + throws InterruptedException { + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, streamBody); + List collected = new CopyOnWriteArrayList<>(); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(options.toLog(collected::add)); + streamedLog.start(); + // The scripted stream is finite, so redirection finishes on its own; stop() joins the thread. + streamedLog.stop(); + return collected; + } + + @Test + void redirectsCompleteTimestampedMessages() throws InterruptedException { + List messages = + redirect( + "2999-01-01T00:00:00.000Z line A\n2999-01-01T00:00:01.000Z line B\n", + new StreamedLogOptions()); + assertEquals( + List.of("2999-01-01T00:00:00.000Z line A", "2999-01-01T00:00:01.000Z line B"), messages); + } + + @Test + void keepsMultiLineMessagesTogether() throws InterruptedException { + // A message whose body spans multiple lines must not be split at the inner newline; only a + // leading timestamp starts a new message. + List messages = + redirect( + "2999-01-01T00:00:00.000Z first line\ncontinued\n2999-01-01T00:00:01.000Z second\n", + new StreamedLogOptions()); + assertEquals(2, messages.size()); + assertEquals("2999-01-01T00:00:00.000Z first line\ncontinued", messages.get(0)); + assertEquals("2999-01-01T00:00:01.000Z second", messages.get(1)); + } + + @Test + void flushesFinalUnterminatedMessage() throws InterruptedException { + List messages = + redirect("2999-01-01T00:00:00.000Z only line, no newline", new StreamedLogOptions()); + assertEquals(List.of("2999-01-01T00:00:00.000Z only line, no newline"), messages); + } + + @Test + void fromStartFalseDropsOldMessages() throws InterruptedException { + // relevancyTimeLimit is set to "now" at construction, so the year-2000 line is dropped and the + // year-2999 line is kept. + List messages = + redirect( + "2000-01-01T00:00:00.000Z old\n2999-01-01T00:00:00.000Z new\n", + new StreamedLogOptions().fromStart(false)); + assertEquals(List.of("2999-01-01T00:00:00.000Z new"), messages); + } + + @Test + void defaultDestinationDoesNotThrow() throws InterruptedException { + // With no toLog, getStreamedLog fetches the run + Actor to build the prefix, then logs to JUL. + MockBackend backend = + MockBackend.ofConstant( + 200, "{\"data\":{\"id\":\"run123\",\"actId\":\"act1\",\"name\":\"a\"}}"); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z hello\n"); + StreamedLog streamedLog = client(backend).run("run123").getStreamedLog(); + streamedLog.start(); + streamedLog.stop(); + // No assertion on output (goes to java.util.logging); the point is it runs without throwing. + } + + @Test + void startTwiceThrows() throws InterruptedException { + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + streamedLog.start(); + assertThrows(IllegalStateException.class, streamedLog::start); + streamedLog.stop(); + } + + @Test + void stopWithoutStartThrows() { + MockBackend backend = MockBackend.ofConstant(200, ""); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + assertThrows(IllegalStateException.class, streamedLog::stop); + } + + @Test + void closeWithoutStartIsNoOp() { + MockBackend backend = MockBackend.ofConstant(200, ""); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + // close() on a never-started helper must not throw (supports try-with-resources). + streamedLog.close(); + } + + @Test + void closeStopsActiveRedirection() throws InterruptedException { + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); + List collected = new CopyOnWriteArrayList<>(); + try (StreamedLog streamedLog = + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(collected::add))) { + streamedLog.start(); + // give the finite stream a moment to be consumed + Thread.sleep(Duration.ofMillis(50).toMillis()); + } + assertTrue(collected.contains("2999-01-01T00:00:00.000Z x")); + } +} diff --git a/src/test/java/com/apify/client/examples/LogRedirection.java b/src/test/java/com/apify/client/examples/LogRedirection.java index eff05ab..0acfd1e 100644 --- a/src/test/java/com/apify/client/examples/LogRedirection.java +++ b/src/test/java/com/apify/client/examples/LogRedirection.java @@ -3,24 +3,27 @@ import com.apify.client.ActorRun; import com.apify.client.ActorStartOptions; import com.apify.client.ApifyClient; -import java.io.InputStream; +import com.apify.client.RunClient; +import com.apify.client.StreamedLog; /** - * Starts an Actor without waiting, then streams its log output to stdout in real time (log - * redirection). + * Starts an Actor without waiting, then redirects its live log to a logger in real time (log + * redirection) until the run finishes. */ public final class LogRedirection { private LogRedirection() {} - public static void main(String[] args) throws Exception { + public static void main(String[] args) { ApifyClient client = ApifyClient.create(System.getenv("APIFY_TOKEN")); // Start the Actor and return immediately (do not wait for it to finish). ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); + RunClient runClient = client.run(run.getId()); - // Open a live (raw) log stream and copy it to stdout as the run produces output. - try (InputStream stream = client.run(run.getId()).getStreamedLog()) { - stream.transferTo(System.out); + // Redirect the run's live log to the default per-run logger while we wait for it to finish. + try (StreamedLog streamedLog = runClient.getStreamedLog()) { + streamedLog.start(); + runClient.waitForFinish(120L); } } } diff --git a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java index 6bcb9ce..b7a8f3d 100644 --- a/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/ActorRunIntegrationTest.java @@ -53,4 +53,19 @@ void lastRunAccess() { assertTrue(byOrigin.isPresent()); assertEquals("SUCCEEDED", byOrigin.get().getStatus()); } + + @Test + void streamedLogRedirection() { + ApifyClient client = requireClient(); + ActorRun run = client.actor("apify/hello-world").start(null, new ActorStartOptions()); + com.apify.client.RunClient runClient = client.run(run.getId()); + + java.util.List collected = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (com.apify.client.StreamedLog streamedLog = + runClient.getStreamedLog(new com.apify.client.StreamedLogOptions().toLog(collected::add))) { + streamedLog.start(); + runClient.waitForFinish(120L); + } + assertTrue(!collected.isEmpty(), "expected redirected log messages"); + } } From 53b05054c6ba939a1ab8e5aad8e8afb60b9eea12 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 13:44:25 +0000 Subject: [PATCH 27/37] docs: list ArrayList and Consumer imports for streamed-log snippet Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 5 +++++ docs/README.md | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65673fa..0986c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Standardized the "official, but experimental" disclaimer wording across the README and all documentation pages. +### Fixed + +- Documentation: added `java.util.ArrayList` and `java.util.function.Consumer` to the stated + snippet import list in `docs/README.md` so the streamed-log example compiles as written. + ## [0.2.0] - 2026-07-10 ### Added diff --git a/docs/README.md b/docs/README.md index 91e407e..ff80cfe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,8 +32,9 @@ empty `Optional` rather than an exception. API failures are thrown as `ApifyApiE ## Imports and dependencies Snippets in these docs assume the client types are imported from `com.apify.client` (e.g. -`import com.apify.client.*;`) plus standard-library types (`java.util.List`, `java.util.Map`, -`java.util.Optional`, `java.util.Iterator`, `java.time.Duration`, `java.io.InputStream`). +`import com.apify.client.*;`) plus standard-library types (`java.util.List`, `java.util.ArrayList`, +`java.util.Map`, `java.util.Optional`, `java.util.Iterator`, `java.util.function.Consumer`, +`java.time.Duration`, `java.io.InputStream`). Raw-JSON return values use Jackson's `com.fasterxml.jackson.databind.JsonNode`. Jackson is a transitive dependency of this client, so it is already on your classpath. From 7622ab60833399b7aa368975c6484d6fe3b2982c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 14:29:08 +0000 Subject: [PATCH 28/37] fix: deliver last StreamedLog message on stop and prevent stop() hang The final flush that emits StreamedLog's retained last message ran inside the try block, so a stop()-induced IOException from a blocked read() skipped it and the message was lost. Move the flush into finally so a stop still delivers it. Open the log stream in start() before launching the reader thread, eliminating a startup race where stop() closed a null stream and join() hung. Drop the redundant finally close now covered by try-with-resources. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 6 ++ .../java/com/apify/client/StreamedLog.java | 29 ++++--- .../java/com/apify/client/MockBackend.java | 21 +++++ .../com/apify/client/StreamedLogTest.java | 85 +++++++++++++++++++ 4 files changed, 131 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0986c33..6a49969 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed +- `StreamedLog`: the last complete log message is no longer dropped when stopping a live stream. + The final flush now runs in a `finally` block, so a stop that unblocks a blocked read with an + `IOException` still delivers the retained message. +- `StreamedLog`: `stop()`/`close()` can no longer hang. The log stream is now opened in `start()` + before the reader thread is launched, eliminating a startup race where `stop()` closed a + still-null stream and then waited forever on a read that never returned. - Documentation: added `java.util.ArrayList` and `java.util.function.Consumer` to the stated snippet import list in `docs/README.md` so the streamed-log example compiles as written. diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 2de5a78..9c931ce 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -82,6 +82,12 @@ public synchronized void start() { } stopLogging = false; pending.setLength(0); + // Open the live stream here, before launching the reader thread, so activeStream is guaranteed + // to be set once the thread exists. If it were opened inside the thread, a stop() that ran + // during the HTTP round-trip would close a still-null stream, leaving the reader blocked on a + // live read() that never returns and join() hanging forever. Opening it up front also lets a + // failed connection surface to the caller of start() instead of a background-thread warning. + activeStream = logClient.stream(new LogOptions().raw(true)); Thread thread = new Thread(this::streamLog, "apify-streamed-log"); thread.setDaemon(true); streamingThread = thread; @@ -122,13 +128,14 @@ public void close() { /** Reads the live raw log stream and forwards complete messages to the destination. */ private void streamLog() { - try (InputStream stream = logClient.stream(new LogOptions().raw(true))) { - activeStream = stream; + // Bytes after the last newline: an incomplete trailing line kept for the next read so a message + // split across chunk boundaries (including a partial UTF-8 sequence) is not corrupted. Declared + // outside the try so the final flush in finally can still emit it (see below). + byte[] lineRemainder = new byte[0]; + // The stream was opened in start() and stored in activeStream; own its lifecycle here via + // try-with-resources so it is always closed exactly once when the reader exits. + try (InputStream stream = activeStream) { byte[] readBuffer = new byte[READ_BUFFER_BYTES]; - // Bytes after the last newline: an incomplete trailing line kept for the next read so a - // message split across chunk boundaries (including a partial UTF-8 sequence) is not - // corrupted. - byte[] lineRemainder = new byte[0]; int read; // Process each chunk before honouring a stop request, so a stop issued right after start // still redirects whatever the run has already produced (matching the reference client). @@ -150,9 +157,6 @@ private void streamLog() { break; } } - // Flush whatever is left when the stream ends (or redirection is stopped): the last message - // may have no trailing newline. - emitMessages(new String(lineRemainder, StandardCharsets.UTF_8), true); } catch (IOException e) { // A read error after an explicit stop is expected (the stream was closed under us). Surface // only genuine, unsolicited failures, and do so without throwing from the background thread. @@ -161,7 +165,12 @@ private void streamLog() { .log(Level.WARNING, "Log redirection stopped due to error", e); } } finally { - closeQuietly(activeStream); + // Flush whatever is left when the stream ends OR when a stop closes the stream mid-read: the + // last complete message is held back in `pending` by emitMessages(..., false), and a stop + // unblocks the read via an IOException that skips the loop body. Running the final flush in + // finally guarantees that retained message (and any unterminated trailing line) is still + // delivered on stop, matching the reference client. + emitMessages(new String(lineRemainder, StandardCharsets.UTF_8), true); } } diff --git a/src/test/java/com/apify/client/MockBackend.java b/src/test/java/com/apify/client/MockBackend.java index 16dc0d9..e485675 100644 --- a/src/test/java/com/apify/client/MockBackend.java +++ b/src/test/java/com/apify/client/MockBackend.java @@ -96,12 +96,33 @@ public synchronized HttpResponse send(HttpRequest request) throws IOExce /** Scripts the next {@link #sendStreaming} call to return the given status and body. */ void scriptStream(int status, String body) { this.streamResponse = new Scripted(status, body, null); + this.streamBody = null; + } + + /** Optional custom stream body (e.g. a blocking stream that simulates a live, open log). */ + private InputStream streamBody; + + private int streamBodyStatus; + + /** + * Scripts the next {@link #sendStreaming} call to return the given status and an arbitrary, + * possibly blocking, {@link InputStream}. Unlike {@link #scriptStream(int, String)}, the body is + * not a finite in-memory buffer, so {@code read()} does not naturally reach end-of-stream; this + * models a live log that only ends when the stream is closed. + */ + void scriptStream(int status, InputStream body) { + this.streamBodyStatus = status; + this.streamBody = body; + this.streamResponse = null; } @Override public synchronized HttpResponse sendStreaming(HttpRequest request) { calls++; lastUrl = request.uri().toString(); + if (streamBody != null) { + return new FakeStreamResponse(request.uri(), streamBodyStatus, streamBody); + } Scripted r = streamResponse != null ? streamResponse : responses.get(0); return new FakeStreamResponse( request.uri(), r.status, new java.io.ByteArrayInputStream(r.body)); diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index b1dcdb7..b9a4682 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -4,9 +4,13 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; import org.junit.jupiter.api.Test; /** @@ -116,6 +120,42 @@ void closeWithoutStartIsNoOp() { streamedLog.close(); } + @Test + void deliversLastMessageWhenStoppingLiveStream() throws InterruptedException { + // Regression test for a live stream that never reaches end-of-stream on its own. The reader + // buffers three complete lines (a, b, c): a and b are emitted immediately while c is retained + // as the possibly-still-growing last message. Then the read blocks. stop() closes the stream, + // which unblocks the read with an IOException; the retained last message (c) must still be + // flushed and delivered. A finite ByteArrayInputStream cannot exercise this because read() + // returns -1 by itself, so we drive a purpose-built blocking stream instead. + String body = + "2999-01-01T00:00:00.000Z a\n" + + "2999-01-01T00:00:01.000Z b\n" + + "2999-01-01T00:00:02.000Z c\n"; + BlockingStream stream = new BlockingStream(body); + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, stream); + List collected = new CopyOnWriteArrayList<>(); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(collected::add)); + streamedLog.start(); + // Wait until a and b have been redirected, which proves the reader has consumed the body and is + // now blocked with c held back as the pending last message. + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (collected.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(5); + } + assertEquals(2, collected.size(), "a and b should be redirected before stop()"); + streamedLog.stop(); + assertEquals( + List.of( + "2999-01-01T00:00:00.000Z a", + "2999-01-01T00:00:01.000Z b", + "2999-01-01T00:00:02.000Z c"), + collected, + "the retained last message must be delivered after stop()"); + } + @Test void closeStopsActiveRedirection() throws InterruptedException { MockBackend backend = MockBackend.ofConstant(200, ""); @@ -131,4 +171,49 @@ void closeStopsActiveRedirection() throws InterruptedException { } assertTrue(collected.contains("2999-01-01T00:00:00.000Z x")); } + + /** + * An {@link InputStream} that serves a fixed payload once and then blocks on {@code read()} until + * it is closed, throwing an {@link IOException} when unblocked by the close. This models a live + * log stream that produces some bytes and then stays open with no further data until the client + * stops redirection - the case a finite {@link java.io.ByteArrayInputStream} cannot reproduce. + */ + private static final class BlockingStream extends InputStream { + private final byte[] data; + private int pos; + private final CountDownLatch closed = new CountDownLatch(1); + + BlockingStream(String data) { + this.data = data.getBytes(StandardCharsets.UTF_8); + } + + @Override + public synchronized int read(byte[] b, int off, int len) throws IOException { + if (pos < data.length) { + int n = Math.min(len, data.length - pos); + System.arraycopy(data, pos, b, off, n); + pos += n; + return n; + } + // Payload exhausted: block like a live stream awaiting more bytes, until close() unblocks us. + try { + closed.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + throw new IOException("stream closed"); + } + + @Override + public int read() throws IOException { + byte[] one = new byte[1]; + int n = read(one, 0, 1); + return n == -1 ? -1 : one[0] & 0xff; + } + + @Override + public void close() { + closed.countDown(); + } + } } From 6c7a2e1d1129af3b611d6e9377ec0b6c640a4978 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 14:40:15 +0000 Subject: [PATCH 29/37] fix: address StreamedLog review items (format, javadoc, comment, fields) Apply spotless formatting (test line exceeded 100 cols), document that start() now opens the stream synchronously and can throw ApifyApiException, reword the stream-close comment to reflect stop()'s idempotent close, and move MockBackend streaming fields to the field block. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../java/com/apify/client/StreamedLog.java | 9 +++++++- .../java/com/apify/client/MockBackend.java | 22 +++++++++---------- .../com/apify/client/StreamedLogTest.java | 4 +++- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 9c931ce..b3519e1 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -74,7 +74,13 @@ public final class StreamedLog implements AutoCloseable { /** * Starts redirecting the log in a background daemon thread. * + *

The live log stream is opened synchronously before the thread is launched, so this call + * blocks briefly on the HTTP round-trip and surfaces a failure to open the stream to the caller + * rather than on the background thread. + * * @throws IllegalStateException if redirection is already running + * @throws ApifyApiException if the log stream cannot be opened (the API returns a non-2xx + * status); other transport failures propagate as their own runtime exception */ public synchronized void start() { if (streamingThread != null) { @@ -133,7 +139,8 @@ private void streamLog() { // outside the try so the final flush in finally can still emit it (see below). byte[] lineRemainder = new byte[0]; // The stream was opened in start() and stored in activeStream; own its lifecycle here via - // try-with-resources so it is always closed exactly once when the reader exits. + // try-with-resources so the reader closes it when it exits (idempotent with stop()'s close, + // which may close it first to unblock a pending read). try (InputStream stream = activeStream) { byte[] readBuffer = new byte[READ_BUFFER_BYTES]; int read; diff --git a/src/test/java/com/apify/client/MockBackend.java b/src/test/java/com/apify/client/MockBackend.java index e485675..1df7af6 100644 --- a/src/test/java/com/apify/client/MockBackend.java +++ b/src/test/java/com/apify/client/MockBackend.java @@ -46,6 +46,17 @@ static final class Scripted { byte[] lastBodyBytes; final List bodies = new ArrayList<>(); + /** + * Optional scripted response for {@link #sendStreaming}; defaults to the first {@code send} + * entry. + */ + private Scripted streamResponse; + + /** Optional custom stream body (e.g. a blocking stream that simulates a live, open log). */ + private InputStream streamBody; + + private int streamBodyStatus; + MockBackend(List responses) { this.responses = responses; } @@ -87,23 +98,12 @@ public synchronized HttpResponse send(HttpRequest request) throws IOExce return new FakeResponse(request.uri(), r.status, r.body); } - /** - * Optional scripted response for {@link #sendStreaming}; defaults to the first {@code send} - * entry. - */ - private Scripted streamResponse; - /** Scripts the next {@link #sendStreaming} call to return the given status and body. */ void scriptStream(int status, String body) { this.streamResponse = new Scripted(status, body, null); this.streamBody = null; } - /** Optional custom stream body (e.g. a blocking stream that simulates a live, open log). */ - private InputStream streamBody; - - private int streamBodyStatus; - /** * Scripts the next {@link #sendStreaming} call to return the given status and an arbitrary, * possibly blocking, {@link InputStream}. Unlike {@link #scriptStream(int, String)}, the body is diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index b9a4682..d4d66d8 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -137,7 +137,9 @@ void deliversLastMessageWhenStoppingLiveStream() throws InterruptedException { backend.scriptStream(200, stream); List collected = new CopyOnWriteArrayList<>(); StreamedLog streamedLog = - client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(collected::add)); + client(backend) + .run("run123") + .getStreamedLog(new StreamedLogOptions().toLog(collected::add)); streamedLog.start(); // Wait until a and b have been redirected, which proves the reader has consumed the body and is // now blocked with c held back as the pending last message. From 5f4069f7f605e25bb76bd02688f81146468a154d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 15:13:20 +0000 Subject: [PATCH 30/37] fix: make StreamedLog.close idempotent, rename regression test, align doc placeholder - StreamedLog.close() is now synchronized and idempotent: running-check + stop happen atomically via a shared private stopStreaming(); close() never throws on a double/concurrent close while stop() keeps its explicit-misuse contract. - Rename ReviewFixesTest -> ClientBehaviourRegressionTest to name the behaviours it verifies rather than the review process; update stale reference comment. - docs/runs.md: use RUN_ID placeholder to match the UPPER_SNAKE convention. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 4 +++ docs/runs.md | 2 +- .../java/com/apify/client/StreamedLog.java | 31 ++++++++++++------- ...ava => ClientBehaviourRegressionTest.java} | 8 +++-- .../com/apify/client/StreamedLogTest.java | 15 +++++++++ .../integration/UserIntegrationTest.java | 2 +- 6 files changed, 47 insertions(+), 15 deletions(-) rename src/test/java/com/apify/client/{ReviewFixesTest.java => ClientBehaviourRegressionTest.java} (98%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a49969..c325d6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,10 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). still-null stream and then waited forever on a read that never returned. - Documentation: added `java.util.ArrayList` and `java.util.function.Consumer` to the stated snippet import list in `docs/README.md` so the streamed-log example compiles as written. +- `StreamedLog.close()` is now fully idempotent: the running-check and stop happen atomically under + the monitor, so a double or concurrent `close()` can no longer throw `IllegalStateException`. +- Documentation: `docs/runs.md` streamed-log snippet now uses the `RUN_ID` placeholder, matching the + convention used across the other snippets. ## [0.2.0] - 2026-07-10 diff --git a/docs/runs.md b/docs/runs.md index 9f25d9c..e1c03a3 100644 --- a/docs/runs.md +++ b/docs/runs.md @@ -59,7 +59,7 @@ Actor name and run id (looked up automatically). `StreamedLogOptions` customizes (default `true`). ```java -RunClient runClient = client.run("run-id"); +RunClient runClient = client.run("RUN_ID"); List collected = new ArrayList<>(); try (StreamedLog streamedLog = runClient.getStreamedLog(new StreamedLogOptions().toLog(collected::add).fromStart(true))) { diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index b3519e1..4c2885f 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -109,6 +109,26 @@ public synchronized void stop() { if (streamingThread == null) { throw new IllegalStateException("Streaming task is not active"); } + stopStreaming(); + } + + /** + * Stops redirection if it is still running; a no-op otherwise. Fully idempotent and safe under a + * concurrent {@link #stop()} or a double {@code close()}: the running-check and the stop happen + * atomically under the monitor, so unlike {@link #stop()} this never throws. + */ + @Override + public synchronized void close() { + if (streamingThread != null) { + stopStreaming(); + } + } + + /** + * Signals the reader to stop, unblocks it by closing the live stream, and joins it. Callers must + * hold the monitor and must have already checked that a thread is running. + */ + private void stopStreaming() { stopLogging = true; // Close the live stream so a blocked read returns promptly instead of waiting for more bytes. closeQuietly(activeStream); @@ -121,17 +141,6 @@ public synchronized void stop() { } } - /** Stops redirection if it is still running; a no-op otherwise. */ - @Override - public void close() { - synchronized (this) { - if (streamingThread == null) { - return; - } - } - stop(); - } - /** Reads the live raw log stream and forwards complete messages to the destination. */ private void streamLog() { // Bytes after the last newline: an incomplete trailing line kept for the next read so a message diff --git a/src/test/java/com/apify/client/ReviewFixesTest.java b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java similarity index 98% rename from src/test/java/com/apify/client/ReviewFixesTest.java rename to src/test/java/com/apify/client/ClientBehaviourRegressionTest.java index b3a1637..a8e5441 100644 --- a/src/test/java/com/apify/client/ReviewFixesTest.java +++ b/src/test/java/com/apify/client/ClientBehaviourRegressionTest.java @@ -11,8 +11,12 @@ import java.util.Optional; import org.junit.jupiter.api.Test; -/** Offline tests pinning correctness behaviours surfaced in review (idempotency, chunking, ...). */ -class ReviewFixesTest { +/** + * Offline regression tests pinning client behaviours: idempotency-key derivation, filter + * propagation through nested clients, retry/timeout policy, request chunking, pagination/iteration + * termination, and wait/poll semantics. + */ +class ClientBehaviourRegressionTest { private static ApifyClient client(MockBackend backend) { return client(backend, 0); diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index d4d66d8..cdd9e71 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -158,6 +158,21 @@ void deliversLastMessageWhenStoppingLiveStream() throws InterruptedException { "the retained last message must be delivered after stop()"); } + @Test + void closeAfterStopIsNoOp() throws InterruptedException { + // close() must be idempotent: after an explicit stop() (which nulls the running thread), a + // trailing close() - e.g. from try-with-resources - must not throw. Guards the documented + // "no-op otherwise" contract against the former check-then-act race with stop(). + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + streamedLog.start(); + streamedLog.stop(); + streamedLog.close(); // must be a no-op, not an IllegalStateException + streamedLog.close(); // idempotent on repeat + } + @Test void closeStopsActiveRedirection() throws InterruptedException { MockBackend backend = MockBackend.ofConstant(200, ""); diff --git a/src/test/java/com/apify/client/integration/UserIntegrationTest.java b/src/test/java/com/apify/client/integration/UserIntegrationTest.java index 94c25b1..adb1a88 100644 --- a/src/test/java/com/apify/client/integration/UserIntegrationTest.java +++ b/src/test/java/com/apify/client/integration/UserIntegrationTest.java @@ -42,5 +42,5 @@ void getLimits() { // runs the test-requirements mandate (unlike the per-run resources every other test creates), so // a // mutating test here would race those runs. Its request behaviour is covered offline instead by - // ReviewFixesTest#updateLimitsSendsPutToMeLimits. + // ClientBehaviourRegressionTest#updateLimitsSendsPutToMeLimits. } From de5ba23851866d84cd9b7a59e271f0ed9465fb36 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 15:19:02 +0000 Subject: [PATCH 31/37] test: add concurrent stop/close race test and fix comment accuracy Reword StreamedLogTest.closeAfterStopIsNoOp to describe sequential idempotency and add concurrentCloseNeverThrowsWhileStopRaces, which fires stop() and close() from two threads to pin that close() never throws under any interleaving. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../com/apify/client/StreamedLogTest.java | 63 ++++++++++++++++++- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index cdd9e71..67d00a4 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -1,6 +1,7 @@ package com.apify.client; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -160,9 +161,10 @@ void deliversLastMessageWhenStoppingLiveStream() throws InterruptedException { @Test void closeAfterStopIsNoOp() throws InterruptedException { - // close() must be idempotent: after an explicit stop() (which nulls the running thread), a - // trailing close() - e.g. from try-with-resources - must not throw. Guards the documented - // "no-op otherwise" contract against the former check-then-act race with stop(). + // Sequential idempotency: after an explicit stop() (which nulls the running thread), a trailing + // close() - e.g. from try-with-resources - and any repeat close() must be no-ops, not throws, + // honouring the documented "no-op otherwise" contract. (The concurrent stop()/close() race is + // covered separately by concurrentCloseNeverThrowsWhileStopRaces.) MockBackend backend = MockBackend.ofConstant(200, ""); backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); StreamedLog streamedLog = @@ -173,6 +175,61 @@ void closeAfterStopIsNoOp() throws InterruptedException { streamedLog.close(); // idempotent on repeat } + @Test + void concurrentCloseNeverThrowsWhileStopRaces() throws InterruptedException { + // Reproduces the concurrent case the fix targets: a stop() and a close() fire simultaneously. + // Because close() does its running-check and stop atomically under the monitor, it must never + // throw regardless of interleaving. (stop() may legitimately throw when it loses the race - its + // explicit-misuse contract - so only close()'s outcome is asserted.) The pre-fix check-then-act + // close() could observe a non-null thread, release the lock, then call a stop() that had + // already nulled the field and threw. Run several rounds to make the interleaving likely. + for (int round = 0; round < 50; round++) { + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z x\n"); + StreamedLog streamedLog = + client(backend).run("run123").getStreamedLog(new StreamedLogOptions().toLog(m -> {})); + streamedLog.start(); + + CountDownLatch go = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference closeError = + new java.util.concurrent.atomic.AtomicReference<>(); + Thread stopper = + new Thread( + () -> { + awaitQuietly(go); + try { + streamedLog.stop(); + } catch (IllegalStateException ignored) { + // Allowed: stop() throws when it loses the race to close(). + } + }); + Thread closer = + new Thread( + () -> { + awaitQuietly(go); + try { + streamedLog.close(); + } catch (Throwable t) { + closeError.set(t); + } + }); + stopper.start(); + closer.start(); + go.countDown(); // release both together to maximise the overlap + stopper.join(); + closer.join(); + assertNull(closeError.get(), "close() must never throw, even when racing stop()"); + } + } + + private static void awaitQuietly(CountDownLatch latch) { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + @Test void closeStopsActiveRedirection() throws InterruptedException { MockBackend backend = MockBackend.ofConstant(200, ""); From e78eeea7316bc6a794a20a48e48249bf31a173ba Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 15:22:23 +0000 Subject: [PATCH 32/37] style: import AtomicReference instead of inline fully-qualified name Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/test/java/com/apify/client/StreamedLogTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index 67d00a4..46aa03d 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -12,6 +12,7 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; /** @@ -191,8 +192,7 @@ void concurrentCloseNeverThrowsWhileStopRaces() throws InterruptedException { streamedLog.start(); CountDownLatch go = new CountDownLatch(1); - java.util.concurrent.atomic.AtomicReference closeError = - new java.util.concurrent.atomic.AtomicReference<>(); + AtomicReference closeError = new AtomicReference<>(); Thread stopper = new Thread( () -> { From 717044aac19937bd6a32fa140f183ea57b7cd025 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 15:44:17 +0000 Subject: [PATCH 33/37] docs: describe option-object fields in storages/misc/actors Add a one-line description for each previously-undescribed field on DatasetListItemsOptions, StoreListOptions, and ActorStartOptions, matching the repo's inline field-description style. Semantics verified against the option classes' setter Javadoc. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- docs/actors.md | 11 ++++++++--- docs/misc.md | 9 ++++++--- docs/storages.md | 18 ++++++++++++------ 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/docs/actors.md b/docs/actors.md index 7d5cd16..0c640c4 100644 --- a/docs/actors.md +++ b/docs/actors.md @@ -60,9 +60,14 @@ Actor created = client.actors().create(Map.of( | `webhooks()` | Read-only nested webhook collection (`NestedWebhookCollectionClient`, `list` + `iterate`, no `create`). | | `version(String)` | An `ActorVersionClient`. | -`ActorStartOptions` fields (all optional): `build`, `memoryMbytes`, `timeoutSecs`, `waitForFinish`, -`maxItems`, `maxTotalChargeUsd`, `contentType`, `restartOnError`, `forcePermissionLevel` -(`LIMITED_PERMISSIONS`/`FULL_PERMISSIONS`), and `webhooks(List)` — ad-hoc webhook definitions +`ActorStartOptions` fields (all optional): `build` (the tag or number of the build to run, e.g. +`latest`, `0.1.2`), `memoryMbytes` (memory in megabytes allocated for the run), `timeoutSecs` (run +timeout in seconds; `0` means no timeout), `waitForFinish` (maximum seconds to wait server-side for +the run to finish, max 60), `maxItems` (maximum dataset items to charge, pay-per-result Actors), +`maxTotalChargeUsd` (maximum total charge in USD, pay-per-event Actors), `contentType` (content type +of the input body, defaults to `application/json`), `restartOnError` (restart the run if it fails), +`forcePermissionLevel` (override the Actor's permission level for this run: +`LIMITED_PERMISSIONS`/`FULL_PERMISSIONS`), and `webhooks(List)` — ad-hoc webhook definitions (each a JSON-serializable `Map`, as in [Webhooks](webhooks.md)) that the client base64-encodes on the wire. diff --git a/docs/misc.md b/docs/misc.md index be90ba6..bce4eca 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -13,9 +13,12 @@ Browse public Actors in the Apify Store. | `list(StoreListOptions)` | A page of Store Actors. Returns `PaginationList`. | | `iterate(StoreListOptions, Long chunkSize)` | A lazy `Iterator` over all matches, fetching pages on demand. The options' `limit` caps the total number of Actors yielded (`null`/unset or non-positive = all); `chunkSize` is the per-request page size (`null` = server default). | -`StoreListOptions` fields: `offset`, `limit`, `search`, `sortBy`, `category`, `username`, -`pricingModel` (`FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`), -`includeUnrunnableActors`, `allowsAgenticUsers`, `responseFormat` (`full`, `agent`). +`StoreListOptions` fields: `offset` (number of Actors to skip), `limit` (maximum number of Actors to +return), `search` (full-text search query), `sortBy` (the sort field, e.g. `popularity`, `newest`), +`category` (filter Actors by category), `username` (filter Actors by owner username), `pricingModel` +(filter by pricing model: `FREE`, `FLAT_PRICE_PER_MONTH`, `PRICE_PER_DATASET_ITEM`, `PAY_PER_EVENT`), +`includeUnrunnableActors` (include Actors the current user cannot run), `allowsAgenticUsers` (only +Actors that allow agentic users), `responseFormat` (the response shape: `full` or `agent`). ```java Iterator it = diff --git a/docs/storages.md b/docs/storages.md index 54c981d..d118ba3 100644 --- a/docs/storages.md +++ b/docs/storages.md @@ -65,12 +65,18 @@ PaginationList page = client.dataset(ds.getId()).listItems(new Dataset byte[] csv = client.dataset(ds.getId()).downloadItems(DownloadItemsFormat.CSV, new DatasetDownloadOptions().bom(true)); ``` -`DatasetListItemsOptions` fields: `offset`, `limit`, `desc`, `fields`, `outputFields`, `omit`, -`skipEmpty`, `skipHidden`, `clean`, `unwind`, `flatten`, `view`, `simplified`, `skipFailedPages`, -`signature`. `fields` selects which source fields to include; `outputFields` positionally *renames* -the fields chosen by `fields` in the output (the i-th name renames the i-th `fields` entry), so it -only makes sense together with `fields`. `downloadItems(...)` returns `byte[]` (the serialized -export). `DatasetDownloadOptions` wraps a `DatasetListItemsOptions` (`items(...)`) and adds +`DatasetListItemsOptions` fields: `offset` (number of items to skip), `limit` (maximum number of +items to return), `desc` (return items newest-first), `fields` (restrict the output to these source +fields), `outputFields` (positionally *renames* the fields chosen by `fields` in the output — the +i-th name renames the i-th `fields` entry, so it only makes sense together with `fields`), `omit` +(exclude these fields from the output), `skipEmpty` (skip empty items), `skipHidden` (skip hidden +fields, i.e. those starting with `#`), `clean` (return only clean — non-empty, non-hidden — items), +`unwind` (expand these fields so each array element becomes a separate item), `flatten` (flatten +these nested fields into dot-notation keys), `view` (select a predefined dataset view for field +selection), `simplified` (return simplified — flattened and cleaned — items), `skipFailedPages` +(skip items that come from failed pages), `signature` (a pre-shared URL signature granting access +without an API token). `downloadItems(...)` returns `byte[]` (the serialized export). +`DatasetDownloadOptions` wraps a `DatasetListItemsOptions` (`items(...)`) and adds `attachment`, `bom`, `delimiter`, `skipHeaderRow`, `xmlRoot`, `xmlRow`, `feedTitle`, `feedDescription`. From 970fda85fc6b9151a497919d948d8fe2e37fe8a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 16:03:24 +0000 Subject: [PATCH 34/37] fix: prevent StreamedLog self-join deadlock and eager prefix-lookup failure Guard stopStreaming() against a thread joining itself: the toLog consumer runs on the reader thread, so stop()/close() from inside it would join the thread on itself and hang forever while holding the synchronized monitor. Wrap the default log-prefix lookup (two synchronous GETs at helper construction) in try/catch, falling back to the runId-only prefix so a 401/403/5xx on the cosmetic lookup no longer aborts getStreamedLog(). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/main/java/com/apify/client/RunClient.java | 19 ++-- .../java/com/apify/client/StreamedLog.java | 25 ++++-- .../com/apify/client/StreamedLogTest.java | 90 +++++++++++++++++++ 3 files changed, 120 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/apify/client/RunClient.java b/src/main/java/com/apify/client/RunClient.java index 37709dd..68b3bd2 100644 --- a/src/main/java/com/apify/client/RunClient.java +++ b/src/main/java/com/apify/client/RunClient.java @@ -227,15 +227,22 @@ public StreamedLog getStreamedLog(StreamedLogOptions options) { * followed by {@code " -> "}. Falls back to just the run id when the Actor name is unavailable. */ private String buildDefaultLogPrefix() { + String runPart = "runId:" + id; String actorName = ""; - Optional run = get(); - if (run.isPresent() && run.get().getActId() != null && !run.get().getActId().isEmpty()) { - Optional actor = root.actor(run.get().getActId()).get(); - if (actor.isPresent() && actor.get().getName() != null) { - actorName = actor.get().getName(); + try { + Optional run = get(); + if (run.isPresent() && run.get().getActId() != null && !run.get().getActId().isEmpty()) { + Optional actor = root.actor(run.get().getActId()).get(); + if (actor.isPresent() && actor.get().getName() != null) { + actorName = actor.get().getName(); + } } + } catch (RuntimeException e) { + // The Actor-name lookup is cosmetic. The getters swallow 404, but an auth (401/403), + // transport, or 5xx-after-retries failure would otherwise throw out of getStreamedLog() and + // abort helper creation even though streaming itself might have worked. Fall back to the + // runId-only prefix (actorName stays "") so the helper is always created. } - String runPart = "runId:" + id; String name = actorName.isEmpty() ? runPart : actorName + " " + runPart; return name + " -> "; } diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 4c2885f..7a6d2b9 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -125,20 +125,29 @@ public synchronized void close() { } /** - * Signals the reader to stop, unblocks it by closing the live stream, and joins it. Callers must - * hold the monitor and must have already checked that a thread is running. + * Signals the reader to stop, unblocks it by closing the live stream, and joins it (unless called + * from the reader thread itself, which cannot join itself). Callers must hold the monitor and + * must have already checked that a thread is running. */ private void stopStreaming() { stopLogging = true; // Close the live stream so a blocked read returns promptly instead of waiting for more bytes. closeQuietly(activeStream); - try { - streamingThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - streamingThread = null; + // A thread must never join itself. The destination consumer runs on the streaming thread, so a + // user calling stop()/close() from inside that consumer (e.g. "stop redirecting once I see line + // X") would otherwise join() the current thread on itself and block forever - and because + // stop()/close() are synchronized, that hung thread keeps the monitor, deadlocking every later + // start/stop/close with no exception. Setting stopLogging + closing the stream already unwinds + // the reader loop, so in that self-stop case we skip only the join; the field is still cleared + // below so lifecycle state stays consistent. + if (Thread.currentThread() != streamingThread) { + try { + streamingThread.join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } } + streamingThread = null; } /** Reads the live raw log stream and forwards complete messages to the destination. */ diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index 46aa03d..8afaf70 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -1,6 +1,8 @@ package com.apify.client; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -12,6 +14,7 @@ import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -246,6 +249,93 @@ void closeStopsActiveRedirection() throws InterruptedException { assertTrue(collected.contains("2999-01-01T00:00:00.000Z x")); } + @Test + void stopFromInsideConsumerDoesNotDeadlock() throws InterruptedException { + // The destination consumer runs on the background reader thread. A user may stop redirection + // from inside it ("stop once I see line X"). Because that calls stopStreaming() on the reader + // thread itself, an unguarded streamingThread.join() would join the thread on itself and hang + // forever - and, since stop() is synchronized, hold the monitor so every later start/stop/close + // deadlocks too. The self-join guard must let the call return so the reader breaks its loop and + // flushes the retained message. If it deadlocked, "b" would never be flushed and the poll below + // would time out with only "a" collected. A blocking stream is required: a finite one would end + // on its own and mask the hang. + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream( + 200, new BlockingStream("2999-01-01T00:00:00.000Z a\n2999-01-01T00:00:01.000Z b\n")); + List collected = new CopyOnWriteArrayList<>(); + AtomicReference ref = new AtomicReference<>(); + AtomicBoolean stopRequested = new AtomicBoolean(false); + StreamedLog streamedLog = + client(backend) + .run("run123") + .getStreamedLog( + new StreamedLogOptions() + .toLog( + message -> { + collected.add(message); + // Self-stop on the reader thread; fire once (stop() throws after the + // thread reference is cleared). + if (stopRequested.compareAndSet(false, true)) { + ref.get().stop(); + } + })); + ref.set(streamedLog); + streamedLog.start(); + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (collected.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(5); + } + assertEquals( + List.of("2999-01-01T00:00:00.000Z a", "2999-01-01T00:00:01.000Z b"), + collected, + "self-stop from inside the consumer must not deadlock and must flush the retained message"); + } + + @Test + void closeFromInsideConsumerDoesNotDeadlock() throws InterruptedException { + // Same self-join hazard as stopFromInsideConsumerDoesNotDeadlock, via close() (which routes + // through the same stopStreaming()). close() is idempotent, so the consumer can call it on + // every + // message without guarding. + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream( + 200, new BlockingStream("2999-01-01T00:00:00.000Z a\n2999-01-01T00:00:01.000Z b\n")); + List collected = new CopyOnWriteArrayList<>(); + AtomicReference ref = new AtomicReference<>(); + StreamedLog streamedLog = + client(backend) + .run("run123") + .getStreamedLog( + new StreamedLogOptions() + .toLog( + message -> { + collected.add(message); + ref.get().close(); // idempotent self-close on the reader thread + })); + ref.set(streamedLog); + streamedLog.start(); + long deadline = System.nanoTime() + Duration.ofSeconds(5).toNanos(); + while (collected.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(5); + } + assertEquals( + List.of("2999-01-01T00:00:00.000Z a", "2999-01-01T00:00:01.000Z b"), + collected, + "self-close from inside the consumer must not deadlock and must flush the retained message"); + } + + @Test + void defaultPrefixLookupFailureStillCreatesHelper() { + // The no-arg getStreamedLog() builds a cosmetic per-run prefix by GETting the run (and its + // Actor). Those getters swallow only 404; a 401/403/5xx-after-retries would otherwise throw out + // of getStreamedLog() and abort helper creation even though streaming might work. The lookup is + // wrapped so it falls back to the runId-only prefix instead of failing. + MockBackend backend = new MockBackend(List.of(MockBackend.ok(401, "{\"error\":{}}"))); + StreamedLog streamedLog = + assertDoesNotThrow(() -> client(backend).run("run123").getStreamedLog()); + assertNotNull(streamedLog); + } + /** * An {@link InputStream} that serves a fixed payload once and then blocks on {@code read()} until * it is closed, throwing an {@link IOException} when unblocked by the close. This models a live From 390bd3a1311160b35e0abf46fde823f480a3d56c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 16:11:13 +0000 Subject: [PATCH 35/37] docs: correct StreamedLog.stop() Javadoc for consumer self-call Document that stop() joins the reader only when called from another thread; a self-call from inside the consumer skips the join and returns immediately, and a repeat self-stop throws because the reader's final flush may re-invoke the consumer. Steer consumer-driven self-termination to the idempotent close(). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- src/main/java/com/apify/client/StreamedLog.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 7a6d2b9..2fe46dc 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -101,9 +101,18 @@ public synchronized void start() { } /** - * Stops log redirection and waits for the background thread to finish. + * Stops log redirection, waiting for the background reader thread to finish. * - * @throws IllegalStateException if redirection is not running + *

Called from any other thread (the normal case) this joins the reader before returning. + * Called from inside the destination consumer - which runs on the reader thread - it + * cannot join itself, so it only signals the stop and returns immediately; redirection then + * unwinds as the reader returns from the consumer. For stopping from inside the consumer prefer + * {@link #close()}: the reader's final flush can re-invoke the consumer for the last buffered + * message, and a second {@code stop()} on the by-then-stopped helper throws, whereas {@code + * close()} is idempotent. + * + * @throws IllegalStateException if redirection is not running - including a repeat call from + * within the consumer after an initial self-stop has already cleared the running thread */ public synchronized void stop() { if (streamingThread == null) { From 00cab63f3544f184f0cf2cee5a2d670aeddddc8a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 16:41:55 +0000 Subject: [PATCH 36/37] fix: harden StreamedLog against throwing consumers and racing readers Guard destination.accept() so a throwing user consumer is caught and logged at WARNING (matching the JS reference) instead of escaping as an uncaught daemon-thread exception. Make the pending-message buffer local to each reader run so a start() racing a still-draining self-stopped reader cannot corrupt shared non-thread-safe parsing state. Also align docs: getStatistics() shown as Optional in the index, store() cross-references StoreCollectionClient, and the account example documents getExtra() for non-modelled fields. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- CHANGELOG.md | 7 +++ docs/README.md | 6 +-- docs/misc.md | 13 ++++-- .../java/com/apify/client/StreamedLog.java | 31 +++++++++---- .../com/apify/client/StreamedLogTest.java | 44 +++++++++++++++++++ .../com/apify/client/examples/GetAccount.java | 3 ++ 6 files changed, 90 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c325d6d..624363d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). the monitor, so a double or concurrent `close()` can no longer throw `IllegalStateException`. - Documentation: `docs/runs.md` streamed-log snippet now uses the `RUN_ID` placeholder, matching the convention used across the other snippets. +- `StreamedLog`: a destination consumer that throws no longer escapes as an uncaught exception on the + background daemon thread. Matching the reference client, the failure is caught, redirection stops, + and a warning is logged. +- `StreamedLog`: the pending-message buffer is now local to each reader run instead of a shared + field, so a `start()` racing a still-draining reader can no longer corrupt shared parsing state. +- Documentation: `docs/README.md` now lists `dataset(id).getStatistics()` as returning + `Optional`, matching the method table in `docs/storages.md`. ## [0.2.0] - 2026-07-10 diff --git a/docs/README.md b/docs/README.md index ff80cfe..f2459be 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,9 +45,9 @@ A few methods return data whose shape is not modelled by this client and is inst Jackson `JsonNode` (or accept an arbitrary `Object` serialized to JSON): - Read: `me().monthlyUsage(...)`, `me().limits()`, `task(id).getInput()`, - `build(id).getOpenApiDefinition()`, `dataset(id).getStatistics()`, and the raw request-queue - operations (`listRequests`, `listAndLockHead`, `prolongRequestLock`, `unlockRequests`, - `batchDeleteRequests`). + `build(id).getOpenApiDefinition()`, `dataset(id).getStatistics()` (returned as + `Optional`), and the raw request-queue operations (`listRequests`, `listAndLockHead`, + `prolongRequestLock`, `unlockRequests`, `batchDeleteRequests`). - Write: `task(id).updateInput(...)` and `me().updateLimits(...)` accept an arbitrary JSON-serializable value, as do definition/`update`/`create` arguments generally — a `Map`, a `JsonNode`, or your own POJO. diff --git a/docs/misc.md b/docs/misc.md index bce4eca..dc5bf13 100644 --- a/docs/misc.md +++ b/docs/misc.md @@ -6,7 +6,7 @@ ## Apify Store — `client.store()` -Browse public Actors in the Apify Store. +Browse public Actors in the Apify Store. `client.store()` returns a `StoreCollectionClient`. | Method | Description | |---|---| @@ -47,11 +47,18 @@ The usage/limits methods are only available for `me()`; calling them on `user(id ```java Optional me = client.me().get(); -me.ifPresent(u -> System.out.println("Account: " + u.getId())); +me.ifPresent( + u -> { + System.out.println("Account: " + u.getId()); + // Fields not modelled on User (email, plan, proxy, ...) are exposed through the untyped + // extras map, which getExtra() returns as a Map. + System.out.println("Email: " + u.getExtra().get("email")); + }); JsonNode usage = client.me().monthlyUsage(); ``` -`User` fields: `getId()`, `getUsername()`, plus `getExtra()`. +`User` fields: `getId()`, `getUsername()`, plus `getExtra()` — a `Map` carrying any +account fields not modelled directly (for `me()`, private details such as `email` and `plan`). ## Logs — `client.log(id)` diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 2fe46dc..7c997f8 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -58,9 +58,6 @@ public final class StreamedLog implements AutoCloseable { */ private final Instant relevancyTimeLimit; - /** Decoded log text seen so far but not yet split into complete messages. */ - private final StringBuilder pending = new StringBuilder(); - private volatile boolean stopLogging; private volatile InputStream activeStream; private Thread streamingThread; @@ -87,7 +84,6 @@ public synchronized void start() { throw new IllegalStateException("Streaming task already active"); } stopLogging = false; - pending.setLength(0); // Open the live stream here, before launching the reader thread, so activeStream is guaranteed // to be set once the thread exists. If it were opened inside the thread, a stop() that ran // during the HTTP round-trip would close a still-null stream, leaving the reader blocked on a @@ -165,6 +161,10 @@ private void streamLog() { // split across chunk boundaries (including a partial UTF-8 sequence) is not corrupted. Declared // outside the try so the final flush in finally can still emit it (see below). byte[] lineRemainder = new byte[0]; + // Decoded log text seen so far but not yet split into complete messages. Local to this reader + // (not a shared field) so that if a start() ever races a still-draining reader, the two readers + // never mutate the same non-thread-safe buffer. + StringBuilder pending = new StringBuilder(); // The stream was opened in start() and stored in activeStream; own its lifecycle here via // try-with-resources so the reader closes it when it exits (idempotent with stop()'s close, // which may close it first to unblock a pending read). @@ -183,7 +183,7 @@ private void streamLog() { String completeText = new String(combined, 0, lastNewline + 1, StandardCharsets.UTF_8); lineRemainder = new byte[combined.length - (lastNewline + 1)]; System.arraycopy(combined, lastNewline + 1, lineRemainder, 0, lineRemainder.length); - emitMessages(completeText, false); + emitMessages(pending, completeText, false); } else { lineRemainder = combined; } @@ -204,7 +204,7 @@ private void streamLog() { // unblocks the read via an IOException that skips the loop body. Running the final flush in // finally guarantees that retained message (and any unterminated trailing line) is still // delivered on stop, matching the reference client. - emitMessages(new String(lineRemainder, StandardCharsets.UTF_8), true); + emitMessages(pending, new String(lineRemainder, StandardCharsets.UTF_8), true); } } @@ -213,7 +213,7 @@ private void streamLog() { * When {@code flush} is {@code false} the last message is held back (it may still be growing); * when {@code true} everything remaining is emitted. */ - private void emitMessages(String text, boolean flush) { + private void emitMessages(StringBuilder pending, String text, boolean flush) { pending.append(text); String buffered = pending.toString(); @@ -268,7 +268,22 @@ private void emitIfRelevant(String message, String timestamp) { // Unparseable timestamp: keep the message rather than silently dropping log output. } } - destination.accept(message); + try { + destination.accept(message); + } catch (RuntimeException e) { + // The destination is a user-supplied consumer and runs on this background daemon thread. If + // it + // throws, letting the exception unwind would kill the daemon thread with an uncaught + // exception + // (and, via the final flush in finally, throw a second time). Match the reference client: + // stop + // redirecting and log a warning rather than propagating. Setting stopLogging breaks the + // reader + // loop so a repeatedly-throwing consumer is not invoked again for every remaining message. + stopLogging = true; + Logger.getLogger(StreamedLog.class.getName()) + .log(Level.WARNING, "Log redirection stopped due to error", e); + } } private static int lastIndexOf(byte[] bytes, byte target) { diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index 8afaf70..2b85f9b 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -15,6 +15,7 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -97,6 +98,49 @@ void defaultDestinationDoesNotThrow() throws InterruptedException { // No assertion on output (goes to java.util.logging); the point is it runs without throwing. } + @Test + void throwingConsumerDoesNotKillDaemonThreadUncaught() throws InterruptedException { + // The destination consumer runs on the background daemon reader thread. A user consumer that + // throws must not escape as an uncaught exception that silently kills that thread (and, via the + // final flush, throws again). Matching the reference client, redirection must degrade to a + // logged warning. We capture any exception that reaches the reader thread's uncaught handler + // and + // assert none does, while confirming the consumer was actually exercised. + AtomicReference uncaught = new AtomicReference<>(); + Thread.UncaughtExceptionHandler previous = Thread.getDefaultUncaughtExceptionHandler(); + Thread.setDefaultUncaughtExceptionHandler( + (t, e) -> { + if ("apify-streamed-log".equals(t.getName())) { + uncaught.set(e); + } else if (previous != null) { + previous.uncaughtException(t, e); + } + }); + try { + MockBackend backend = MockBackend.ofConstant(200, ""); + backend.scriptStream(200, "2999-01-01T00:00:00.000Z boom\n2999-01-01T00:00:01.000Z after\n"); + AtomicInteger calls = new AtomicInteger(); + StreamedLog streamedLog = + client(backend) + .run("run123") + .getStreamedLog( + new StreamedLogOptions() + .toLog( + message -> { + calls.incrementAndGet(); + throw new RuntimeException("consumer failed"); + })); + streamedLog.start(); + streamedLog.stop(); // joins the reader; if it died uncaught, the handler above has fired + assertNull( + uncaught.get(), + "a throwing destination consumer must not escape as an uncaught daemon-thread exception"); + assertTrue(calls.get() >= 1, "the destination consumer should have been invoked"); + } finally { + Thread.setDefaultUncaughtExceptionHandler(previous); + } + } + @Test void startTwiceThrows() throws InterruptedException { MockBackend backend = MockBackend.ofConstant(200, ""); diff --git a/src/test/java/com/apify/client/examples/GetAccount.java b/src/test/java/com/apify/client/examples/GetAccount.java index b5c7c33..c8091b5 100644 --- a/src/test/java/com/apify/client/examples/GetAccount.java +++ b/src/test/java/com/apify/client/examples/GetAccount.java @@ -20,5 +20,8 @@ public static void main(String[] args) { } System.out.println("Account ID: " + user.get().getId()); System.out.println("Username: " + user.get().getUsername()); + // Fields not modelled on User (email, plan, ...) live in the untyped extras map that + // getExtra() returns as a Map. + System.out.println("Email: " + user.get().getExtra().get("email")); } } From c4eb1ce769cf68b8501a7cb2ff5d40b3cc86d699 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 16:53:24 +0000 Subject: [PATCH 37/37] fix: unwind StreamedLog redirection on first consumer failure Introduce a forwardingFailed flag so a throwing destination consumer is invoked exactly once: the emit loop returns on the first failure and the final flush is skipped, logging a single warning and delivering nothing further (matching the JS reference). Condense and correct the catch-block comment. Tighten the hermetic test to assert exactly-once invocation. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019rkZYoebybdnv7MFH9Ua6E --- .../java/com/apify/client/StreamedLog.java | 33 ++++++++++++------- .../com/apify/client/StreamedLogTest.java | 6 +++- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/apify/client/StreamedLog.java b/src/main/java/com/apify/client/StreamedLog.java index 7c997f8..4e23fba 100644 --- a/src/main/java/com/apify/client/StreamedLog.java +++ b/src/main/java/com/apify/client/StreamedLog.java @@ -59,6 +59,13 @@ public final class StreamedLog implements AutoCloseable { private final Instant relevancyTimeLimit; private volatile boolean stopLogging; + + /** + * Set once a destination consumer throws, so redirection unwinds on the first failure and + * forwards nothing further (matching the reference client). Reset per {@link #start()}. + */ + private volatile boolean forwardingFailed; + private volatile InputStream activeStream; private Thread streamingThread; @@ -84,6 +91,7 @@ public synchronized void start() { throw new IllegalStateException("Streaming task already active"); } stopLogging = false; + forwardingFailed = false; // Open the live stream here, before launching the reader thread, so activeStream is guaranteed // to be set once the thread exists. If it were opened inside the thread, a stop() that ran // during the HTTP round-trip would close a still-null stream, leaving the reader blocked on a @@ -203,8 +211,11 @@ private void streamLog() { // last complete message is held back in `pending` by emitMessages(..., false), and a stop // unblocks the read via an IOException that skips the loop body. Running the final flush in // finally guarantees that retained message (and any unterminated trailing line) is still - // delivered on stop, matching the reference client. - emitMessages(pending, new String(lineRemainder, StandardCharsets.UTF_8), true); + // delivered on stop, matching the reference client. Skipped once a consumer has thrown: there + // is nothing more to deliver and the flush must not re-invoke the failed consumer. + if (!forwardingFailed) { + emitMessages(pending, new String(lineRemainder, StandardCharsets.UTF_8), true); + } } } @@ -239,6 +250,10 @@ private void emitMessages(StringBuilder pending, String text, boolean flush) { int from = messageStarts.get(i); int to = (i + 1 < messageStarts.size()) ? messageStarts.get(i + 1) : buffered.length(); emitIfRelevant(buffered.substring(from, to).trim(), timestamps.get(i)); + if (forwardingFailed) { + // A consumer threw: unwind on the first failure without forwarding the rest of this batch. + return; + } } if (flush) { @@ -271,15 +286,11 @@ private void emitIfRelevant(String message, String timestamp) { try { destination.accept(message); } catch (RuntimeException e) { - // The destination is a user-supplied consumer and runs on this background daemon thread. If - // it - // throws, letting the exception unwind would kill the daemon thread with an uncaught - // exception - // (and, via the final flush in finally, throw a second time). Match the reference client: - // stop - // redirecting and log a warning rather than propagating. Setting stopLogging breaks the - // reader - // loop so a repeatedly-throwing consumer is not invoked again for every remaining message. + // The destination is a user-supplied consumer running on this background daemon thread; an + // uncaught throw would kill the thread. Match the reference client: stop redirecting on the + // first failure (forwardingFailed unwinds the emit loop and skips the final flush, so the + // consumer is not invoked again) and log a single warning instead of propagating. + forwardingFailed = true; stopLogging = true; Logger.getLogger(StreamedLog.class.getName()) .log(Level.WARNING, "Log redirection stopped due to error", e); diff --git a/src/test/java/com/apify/client/StreamedLogTest.java b/src/test/java/com/apify/client/StreamedLogTest.java index 2b85f9b..efaca90 100644 --- a/src/test/java/com/apify/client/StreamedLogTest.java +++ b/src/test/java/com/apify/client/StreamedLogTest.java @@ -135,7 +135,11 @@ void throwingConsumerDoesNotKillDaemonThreadUncaught() throws InterruptedExcepti assertNull( uncaught.get(), "a throwing destination consumer must not escape as an uncaught daemon-thread exception"); - assertTrue(calls.get() >= 1, "the destination consumer should have been invoked"); + // Two complete lines are streamed, but redirection must unwind on the first throw and forward + // nothing further (no re-invocation for the remaining line or the final flush), matching the + // reference client's single warning. + assertEquals( + 1, calls.get(), "the destination consumer must not be invoked again after it throws"); } finally { Thread.setDefaultUncaughtExceptionHandler(previous); }