From 9351189e5fc0f82e90d40962acfc12630a0cda44 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 16 Jun 2026 12:53:47 +0200 Subject: [PATCH 1/2] feat: Add apify.errors domain-level error taxonomy --- src/apify/_utils.py | 1 + src/apify/errors.py | 158 +++++++++++++++++++++ tests/unit/actor/test_configuration.py | 18 --- tests/unit/test_errors.py | 184 +++++++++++++++++++++++++ website/docusaurus.config.js | 1 + 5 files changed, 344 insertions(+), 18 deletions(-) create mode 100644 src/apify/errors.py create mode 100644 tests/unit/test_errors.py diff --git a/src/apify/_utils.py b/src/apify/_utils.py index 8469ae97b..097795e83 100644 --- a/src/apify/_utils.py +++ b/src/apify/_utils.py @@ -74,6 +74,7 @@ def is_running_in_ipython() -> bool: 'Actor', 'Charging', 'Configuration', + 'Errors', 'Event data', 'Event managers', 'Events', diff --git a/src/apify/errors.py b/src/apify/errors.py new file mode 100644 index 000000000..786d985b1 --- /dev/null +++ b/src/apify/errors.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from apify_client.errors import ForbiddenError as _ForbiddenError +from apify_client.errors import InvalidRequestError as _InvalidRequestError +from apify_client.errors import RateLimitError as _RateLimitError +from apify_client.errors import ServerError as _ServerError +from apify_client.errors import UnauthorizedError as _UnauthorizedError + +from apify._utils import docs_group + +if TYPE_CHECKING: + from apify_client._models import Run + + +@docs_group('Errors') +class ActorError(Exception): + """Base class for all domain-level Apify SDK errors. + + Carries a machine-readable `code` and a `retryable` flag so callers can branch on a failure without reading + the human-readable error message. + """ + + code: str = 'actor-error' + """Stable, machine-readable identifier of the error category.""" + + retryable: bool = False + """Whether retrying the same operation might succeed (e.g. a transient rate limit or server error).""" + + def __init__( + self, + message: str | None = None, + *, + code: str | None = None, + retryable: bool | None = None, + ) -> None: + super().__init__(message) + if code is not None: + self.code = code + if retryable is not None: + self.retryable = retryable + + @classmethod + def from_client_error(cls, error: Exception) -> ActorError: + """Map an `apify_client` exception to the matching domain-level error. + + The mapping is driven by the client's typed, HTTP-status-based exceptions. Unmapped client errors (and any + other exception) fall back to a plain `ActorError`. The original exception is not chained automatically; + callers should use `raise ActorError.from_client_error(err) from err`. + + Args: + error: The exception raised by `apify_client`. + + Returns: + The corresponding domain-level error. + """ + if isinstance(error, (_UnauthorizedError, _ForbiddenError)): + return ActorAuthenticationError(str(error)) + + if isinstance(error, _RateLimitError): + return ActorRateLimitError(str(error)) + + if isinstance(error, _ServerError): + return ActorError(str(error), retryable=True) + + if isinstance(error, _InvalidRequestError): + return ActorInputValidationError(str(error)) + + return ActorError(str(error)) + + +@docs_group('Errors') +class ActorRunError(ActorError): + """Raised when an Actor run reaches a terminal failure state (e.g. `FAILED` or `ABORTED`). + + Unlike the HTTP-derived errors, this one is derived from the run itself, so it exposes the run metadata needed + to decide what to do next. + """ + + code = 'actor-run-failed' + + def __init__(self, run: Run) -> None: + self.run_id = run.id + self.status = run.status + self.exit_code = run.exit_code + self.status_message = run.status_message + + message = f'Actor run {run.id!r} ended with status {run.status!r}' + if run.status_message: + message = f'{message}: {run.status_message}' + + super().__init__(message) + + @classmethod + def from_run(cls, run: Run) -> ActorRunError: + """Build the most specific run error for a terminal Actor run. + + Args: + run: The terminal Actor run. + + Returns: + An `ActorTimeoutError` for a timed-out run, otherwise an `ActorRunError`. + """ + if run.status == 'TIMED-OUT': + return ActorTimeoutError(run) + return ActorRunError(run) + + +@docs_group('Errors') +class ActorTimeoutError(ActorRunError): + """Raised when an Actor run exceeds its timeout (`TIMED-OUT`). Retrying with a longer timeout may help.""" + + code = 'actor-timed-out' + retryable = True + + +@docs_group('Errors') +class ActorInputValidationError(ActorError, ValueError): + """Raised when input fails validation. + + Subclasses `ValueError` so existing `except ValueError` handlers keep catching it. + """ + + code = 'input-validation-error' + + +@docs_group('Errors') +class ActorChargeLimitExceededError(ActorError): + """Raised when an Actor run hits its configured maximum total charge (`max_total_charge_usd`).""" + + code = 'charge-limit-exceeded' + + +@docs_group('Errors') +class ActorAuthenticationError(ActorError): + """Raised when an API request is unauthorized or forbidden (HTTP 401 / 403).""" + + code = 'authentication-error' + + +@docs_group('Errors') +class ActorRateLimitError(ActorError): + """Raised when the Apify API rate limit is exceeded (HTTP 429). Retryable after a backoff.""" + + code = 'rate-limit-exceeded' + retryable = True + + +__all__ = [ + 'ActorAuthenticationError', + 'ActorChargeLimitExceededError', + 'ActorError', + 'ActorInputValidationError', + 'ActorRateLimitError', + 'ActorRunError', + 'ActorTimeoutError', +] diff --git a/tests/unit/actor/test_configuration.py b/tests/unit/actor/test_configuration.py index 486cbe07c..0fd686dda 100644 --- a/tests/unit/actor/test_configuration.py +++ b/tests/unit/actor/test_configuration.py @@ -392,21 +392,3 @@ def test_actor_storage_json_env_var(monkeypatch: pytest.MonkeyPatch) -> None: assert config.actor_storages['datasets'] == datasets assert config.actor_storages['request_queues'] == request_queues assert config.actor_storages['key_value_stores'] == key_value_stores - - -@pytest.mark.parametrize( - ('env_var', 'attr', 'expected'), - [ - ('APIFY_TIMEOUT_AT', 'timeout_at', None), - ('ACTOR_MAX_PAID_DATASET_ITEMS', 'max_paid_dataset_items', None), - ('ACTOR_MAX_TOTAL_CHARGE_USD', 'max_total_charge_usd', None), - ('APIFY_USER_IS_PAYING', 'user_is_paying', False), - ], -) -def test_typed_env_var_empty_string_falls_back_to_default( - monkeypatch: pytest.MonkeyPatch, env_var: str, attr: str, expected: object -) -> None: - """Platform may set a typed env var to '' instead of leaving it unset; that must not crash `Actor.init()`.""" - monkeypatch.setenv(env_var, '') - config = ApifyConfiguration() - assert getattr(config, attr) == expected diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py new file mode 100644 index 000000000..05261ec34 --- /dev/null +++ b/tests/unit/test_errors.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Any, cast + +import pytest + +from apify_client._models import Run +from apify_client.errors import ( + ApifyApiError, + ConflictError, + ForbiddenError, + InvalidRequestError, + NotFoundError, + ServerError, + UnauthorizedError, +) +from apify_client.errors import RateLimitError as ClientRateLimitError + +import apify +from apify.errors import ( + ActorAuthenticationError, + ActorChargeLimitExceededError, + ActorError, + ActorInputValidationError, + ActorRateLimitError, + ActorRunError, + ActorTimeoutError, +) + + +class _FakeResponse: + """Minimal stand-in for `apify_client`'s HTTP response, enough to build its API errors.""" + + def __init__(self, status_code: int) -> None: + self.status_code = status_code + self.text = 'error text' + + def json(self) -> dict[str, Any]: + return {'error': {'message': 'boom', 'type': 'some-error-type'}} + + +def _client_error(error_cls: type[ApifyApiError], status_code: int) -> ApifyApiError: + return error_cls(cast('Any', _FakeResponse(status_code)), 1) + + +def _make_run(*, status: str, exit_code: int | None = None, status_message: str | None = None) -> Run: + return Run.model_validate( + { + 'id': 'run123', + 'actId': 'act123', + 'userId': 'user123', + 'startedAt': datetime.now(UTC).isoformat(), + 'status': status, + 'statusMessage': status_message, + 'exitCode': exit_code, + 'meta': {'origin': 'DEVELOPMENT'}, + 'buildId': 'build123', + 'defaultDatasetId': 'ds123', + 'defaultKeyValueStoreId': 'kvs123', + 'defaultRequestQueueId': 'rq123', + 'containerUrl': 'https://container', + 'buildNumber': '0.0.1', + 'generalAccess': 'RESTRICTED', + 'stats': {'restartCount': 0, 'resurrectCount': 0, 'computeUnits': 1}, + 'options': {'build': 'latest', 'timeoutSecs': 4, 'memoryMbytes': 1024, 'diskMbytes': 1024}, + } + ) + + +def test_actor_error_defaults() -> None: + error = ActorError('something went wrong') + assert error.code == 'apify-error' + assert error.retryable is False + assert str(error) == 'something went wrong' + + +def test_actor_error_overrides_are_instance_scoped() -> None: + error = ActorError('boom', code='custom', retryable=True) + assert error.code == 'custom' + assert error.retryable is True + # Overriding on an instance must not leak to the class default. + assert ActorError.code == 'apify-error' + assert ActorError.retryable is False + + +@pytest.mark.parametrize( + ('error_cls', 'expected_code', 'expected_retryable'), + [ + (ActorRateLimitError, 'rate-limit-exceeded', True), + (ActorTimeoutError, 'actor-timed-out', True), + (ActorAuthenticationError, 'authentication-error', False), + (ActorChargeLimitExceededError, 'charge-limit-exceeded', False), + (ActorInputValidationError, 'input-validation-error', False), + (ActorRunError, 'actor-run-failed', False), + ], +) +def test_subclass_codes_and_retryable( + error_cls: type[ActorError], expected_code: str, *, expected_retryable: bool +) -> None: + assert error_cls.code == expected_code + assert error_cls.retryable is expected_retryable + assert issubclass(error_cls, ActorError) + + +def test_input_validation_error_is_value_error() -> None: + """`except ValueError` must still catch `ActorInputValidationError`.""" + with pytest.raises(ValueError, match='bad input'): + raise ActorInputValidationError('bad input') + + +def test_actor_timeout_error_is_actor_run_error() -> None: + assert issubclass(ActorTimeoutError, ActorRunError) + + +def test_actor_run_error_carries_run_metadata() -> None: + run = _make_run(status='FAILED', exit_code=1, status_message='Actor crashed') + error = ActorRunError(run) + assert error.run_id == 'run123' + assert error.status == 'FAILED' + assert error.exit_code == 1 + assert error.status_message == 'Actor crashed' + assert error.retryable is False + assert 'run123' in str(error) + assert 'Actor crashed' in str(error) + + +def test_actor_run_error_from_run_failed() -> None: + error = ActorRunError.from_run(_make_run(status='FAILED')) + assert type(error) is ActorRunError + assert not error.retryable + + +def test_actor_run_error_from_run_timed_out() -> None: + error = ActorRunError.from_run(_make_run(status='TIMED-OUT')) + assert isinstance(error, ActorTimeoutError) + assert error.retryable is True + assert error.run_id == 'run123' + assert error.code == 'actor-timed-out' + + +@pytest.mark.parametrize( + ('client_error', 'expected_cls', 'expected_retryable'), + [ + (_client_error(UnauthorizedError, 401), ActorAuthenticationError, False), + (_client_error(ForbiddenError, 403), ActorAuthenticationError, False), + (_client_error(ClientRateLimitError, 429), ActorRateLimitError, True), + (_client_error(ServerError, 500), ActorError, True), + (_client_error(InvalidRequestError, 400), ActorInputValidationError, False), + (_client_error(NotFoundError, 404), ActorError, False), + (_client_error(ConflictError, 409), ActorError, False), + ], +) +def test_from_client_error_mapping( + client_error: ApifyApiError, + expected_cls: type[ActorError], + *, + expected_retryable: bool, +) -> None: + mapped = ActorError.from_client_error(client_error) + assert type(mapped) is expected_cls + assert mapped.retryable is expected_retryable + + +def test_from_client_error_unknown_exception_falls_back() -> None: + mapped = ActorError.from_client_error(RuntimeError('not a client error')) + assert type(mapped) is ActorError + assert mapped.retryable is False + assert 'not a client error' in str(mapped) + + +def test_errors_exported_from_top_level() -> None: + for name in ( + 'ActorError', + 'ActorRunError', + 'ActorTimeoutError', + 'ActorAuthenticationError', + 'ActorChargeLimitExceededError', + 'ActorInputValidationError', + 'ActorRateLimitError', + ): + assert hasattr(apify, name) + assert name in apify.__all__ + assert getattr(apify, name) is getattr(apify.errors, name) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index d6ef5fd68..d593892f8 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -9,6 +9,7 @@ const GROUP_ORDER = [ 'Actor', 'Charging', 'Configuration', + 'Errors', 'Event data', 'Event managers', 'Events', From 4d3616276dadecd107ddb6c7693cd1af74912ffa Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Tue, 16 Jun 2026 13:53:53 +0200 Subject: [PATCH 2/2] feat: map apify-client errors to domain-level errors at API call sites --- src/apify/_actor.py | 125 ++++++++++-------- src/apify/_charging.py | 7 +- src/apify/errors.py | 40 +++++- .../storage_clients/_apify/_dataset_client.py | 33 +++-- .../_apify/_key_value_store_client.py | 50 ++++--- .../_apify/_request_queue_client.py | 14 +- tests/unit/test_errors.py | 20 +-- 7 files changed, 175 insertions(+), 114 deletions(-) diff --git a/src/apify/_actor.py b/src/apify/_actor.py index e00e46638..4abfdd08c 100644 --- a/src/apify/_actor.py +++ b/src/apify/_actor.py @@ -32,6 +32,7 @@ from apify._proxy_configuration import ProxyConfiguration from apify._utils import docs_group, docs_name, ensure_context, get_system_info, is_running_in_ipython from apify._webhook import to_client_representations +from apify.errors import map_client_errors from apify.events import ApifyEventManager, EventManager, LocalEventManager from apify.log import _configure_logging, logger from apify.storage_clients import ApifyStorageClient, SmartApifyStorageClient @@ -936,17 +937,18 @@ async def start( raise ValueError(f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, or a `timedelta`.') actor_client = client.actor(actor_id) - return await actor_client.start( - run_input=run_input, - content_type=content_type, - build=build, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=actor_start_timeout, - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - ) + with map_client_errors(): + return await actor_client.start( + run_input=run_input, + content_type=content_type, + build=build, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=actor_start_timeout, + force_permission_level=force_permission_level, + webhooks=to_client_representations(webhooks), + ) @_ensure_context async def abort( @@ -975,10 +977,11 @@ async def abort( client = self.new_client(token=token) if token else self.apify_client run_client = client.run(run_id) - if status_message: - await run_client.update(status_message=status_message) + with map_client_errors(): + if status_message: + await run_client.update(status_message=status_message) - run = await run_client.abort(gracefully=gracefully) + run = await run_client.abort(gracefully=gracefully) if run is None: raise RuntimeError(f'Failed to abort Actor run with ID "{run_id}".') @@ -1047,19 +1050,20 @@ async def call( raise ValueError(f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, or a `timedelta`.') actor_client = client.actor(actor_id) - run = await actor_client.call( - run_input=run_input, - content_type=content_type, - build=build, - max_total_charge_usd=max_total_charge_usd, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=actor_call_timeout, - force_permission_level=force_permission_level, - webhooks=to_client_representations(webhooks), - wait_duration=wait, - logger=logger, - ) + with map_client_errors(): + run = await actor_client.call( + run_input=run_input, + content_type=content_type, + build=build, + max_total_charge_usd=max_total_charge_usd, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=actor_call_timeout, + force_permission_level=force_permission_level, + webhooks=to_client_representations(webhooks), + wait_duration=wait, + logger=logger, + ) if run is None: raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".') @@ -1120,15 +1124,16 @@ async def call_task( raise ValueError(f'Invalid timeout {timeout!r}: expected `None`, `"inherit"`, or a `timedelta`.') task_client = client.task(task_id) - run = await task_client.call( - task_input=task_input, - build=build, - restart_on_error=restart_on_error, - memory_mbytes=memory_mbytes, - run_timeout=task_call_timeout, - webhooks=to_client_representations(webhooks), - wait_duration=wait, - ) + with map_client_errors(): + run = await task_client.call( + task_input=task_input, + build=build, + restart_on_error=restart_on_error, + memory_mbytes=memory_mbytes, + run_timeout=task_call_timeout, + webhooks=to_client_representations(webhooks), + wait_duration=wait, + ) if run is None: raise RuntimeError(f'Failed to call Task with ID "{task_id}".') @@ -1171,12 +1176,13 @@ async def metamorph( if not self.configuration.actor_run_id: raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.') - await self.apify_client.run(self.configuration.actor_run_id).metamorph( - target_actor_id=target_actor_id, - run_input=run_input, - target_actor_build=target_actor_build, - content_type=content_type, - ) + with map_client_errors(): + await self.apify_client.run(self.configuration.actor_run_id).metamorph( + target_actor_id=target_actor_id, + run_input=run_input, + target_actor_build=target_actor_build, + content_type=content_type, + ) if custom_after_sleep: await asyncio.sleep(custom_after_sleep.total_seconds()) @@ -1242,7 +1248,8 @@ async def safe_dispatch(listener: Any, data: Any) -> None: except TimeoutError: self.log.warning('Pre-reboot event listeners did not finish within timeout; proceeding with reboot') - await self.apify_client.run(self.configuration.actor_run_id).reboot() + with map_client_errors(): + await self.apify_client.run(self.configuration.actor_run_id).reboot() except BaseException: # Reset the flag so that a failed or cancelled reboot can be retried. self._is_rebooting = False @@ -1283,17 +1290,18 @@ async def add_webhook(self, webhook: Webhook, *, idempotency_key: str | None = N if not self.configuration.actor_run_id: raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.') - await self.apify_client.webhooks().create( - actor_run_id=self.configuration.actor_run_id, - event_types=webhook.event_types, - request_url=webhook.request_url, - payload_template=webhook.payload_template, - headers_template=webhook.headers_template, - ignore_ssl_errors=webhook.ignore_ssl_errors, - do_not_retry=webhook.do_not_retry, - idempotency_key=idempotency_key if idempotency_key is not None else webhook.idempotency_key, - is_ad_hoc=True, - ) + with map_client_errors(): + await self.apify_client.webhooks().create( + actor_run_id=self.configuration.actor_run_id, + event_types=webhook.event_types, + request_url=webhook.request_url, + payload_template=webhook.payload_template, + headers_template=webhook.headers_template, + ignore_ssl_errors=webhook.ignore_ssl_errors, + do_not_retry=webhook.do_not_retry, + idempotency_key=idempotency_key if idempotency_key is not None else webhook.idempotency_key, + is_ad_hoc=True, + ) @_ensure_context async def set_status_message( @@ -1321,10 +1329,11 @@ async def set_status_message( raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.') run_client = self.apify_client.run(self.configuration.actor_run_id) - run = await run_client.update( - status_message=status_message, - is_status_message_terminal=is_terminal, - ) + with map_client_errors(): + run = await run_client.update( + status_message=status_message, + is_status_message_terminal=is_terminal, + ) if run is None: raise RuntimeError( diff --git a/src/apify/_charging.py b/src/apify/_charging.py index 0a23ad4ef..2291a04fe 100644 --- a/src/apify/_charging.py +++ b/src/apify/_charging.py @@ -19,6 +19,7 @@ from apify_client._models import PricingPerEvent as ClientPricingPerEvent from apify._utils import ReentrantLock, docs_group, ensure_context +from apify.errors import map_client_errors from apify.log import logger from apify.storages import Dataset @@ -449,7 +450,8 @@ async def charge(self, event_name: str, *, count: int = 1) -> ChargeResult: # the platform handles them automatically based on dataset writes. pass elif event_name in self._pricing_info: - await self._client.run(self._actor_run_id).charge(event_name, count=charged_count) + with map_client_errors(): + await self._client.run(self._actor_run_id).charge(event_name, count=charged_count) elif event_name in self._tier_priced_events: logger.warning( f"Event '{event_name}' is tier-priced and is not chargeable via the pay-per-event API." @@ -572,7 +574,8 @@ async def _fetch_pricing_info(self) -> _FetchedPricingInfoDict: if self._actor_run_id is None: raise RuntimeError('Actor run ID not found even though the Actor is running on Apify') - run = await self._client.run(self._actor_run_id).get() + with map_client_errors(): + run = await self._client.run(self._actor_run_id).get() if run is None: raise RuntimeError('Actor run not found') diff --git a/src/apify/errors.py b/src/apify/errors.py index 786d985b1..85b706128 100644 --- a/src/apify/errors.py +++ b/src/apify/errors.py @@ -1,7 +1,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING +import contextlib +import functools +from typing import TYPE_CHECKING, ParamSpec, TypeVar +from apify_client.errors import ApifyApiError from apify_client.errors import ForbiddenError as _ForbiddenError from apify_client.errors import InvalidRequestError as _InvalidRequestError from apify_client.errors import RateLimitError as _RateLimitError @@ -11,8 +14,14 @@ from apify._utils import docs_group if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Coroutine, Iterator + from typing import Any + from apify_client._models import Run +_P = ParamSpec('_P') +_R = TypeVar('_R') + @docs_group('Errors') class ActorError(Exception): @@ -147,6 +156,35 @@ class ActorRateLimitError(ActorError): retryable = True +@contextlib.contextmanager +def map_client_errors() -> Iterator[None]: + """Translate `apify_client` API errors into domain-level `ActorError`s. + + Wrap any `apify_client` call with this context manager so that an `ApifyApiError` (e.g. an HTTP 401/403/429/5xx + response) surfaces as the matching `ActorError` subclass instead of a raw client exception. The original error + is preserved as the `__cause__` of the raised `ActorError`. + """ + try: + yield + except ApifyApiError as error: + raise ActorError.from_client_error(error) from error + + +def catch_client_errors(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Coroutine[Any, Any, _R]]: + """Decorate an async function so the `apify_client` errors it raises become domain-level `ActorError`s. + + This is the method-level counterpart of `map_client_errors`, intended for thin wrappers around `apify_client` + calls such as the storage client operations. + """ + + @functools.wraps(func) + async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + with map_client_errors(): + return await func(*args, **kwargs) + + return wrapper + + __all__ = [ 'ActorAuthenticationError', 'ActorChargeLimitExceededError', diff --git a/src/apify/storage_clients/_apify/_dataset_client.py b/src/apify/storage_clients/_apify/_dataset_client.py index de9634fdc..82f006f7a 100644 --- a/src/apify/storage_clients/_apify/_dataset_client.py +++ b/src/apify/storage_clients/_apify/_dataset_client.py @@ -12,6 +12,7 @@ from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata from ._api_client_creation import create_storage_api_client +from apify.errors import ActorError, catch_client_errors, map_client_errors from apify.storage_clients._ppe_dataset_mixin import DatasetClientPpeMixin if TYPE_CHECKING: @@ -57,11 +58,12 @@ def __init__( """A lock to ensure that only one operation is performed at a time.""" @override + @catch_client_errors async def get_metadata(self) -> DatasetMetadata: metadata = await self._api_client.get() if metadata is None: - raise ValueError('Failed to retrieve dataset metadata.') + raise ActorError('Failed to retrieve dataset metadata.') return DatasetMetadata( id=metadata.id, @@ -73,6 +75,7 @@ async def get_metadata(self) -> DatasetMetadata: ) @classmethod + @catch_client_errors async def open( cls, *, @@ -132,11 +135,13 @@ async def purge(self) -> None: ) @override + @catch_client_errors async def drop(self) -> None: async with self._lock: await self._api_client.delete() @override + @catch_client_errors async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable]) -> None: async def payloads_generator(items: Sequence[Mapping[str, JsonSerializable]]) -> AsyncIterator[str]: for index, item in enumerate(items): @@ -155,6 +160,7 @@ async def payloads_generator(items: Sequence[Mapping[str, JsonSerializable]]) -> await self._charge_for_items(count_items=limit) @override + @catch_client_errors async def get_data( self, *, @@ -199,18 +205,19 @@ async def iterate_items( skip_empty: bool = False, skip_hidden: bool = False, ) -> AsyncIterator[dict]: - async for item in self._api_client.iterate_items( - offset=offset, - limit=limit, - clean=clean, - desc=desc, - fields=fields, - omit=omit, - unwind=unwind, - skip_empty=skip_empty, - skip_hidden=skip_hidden, - ): - yield item + with map_client_errors(): + async for item in self._api_client.iterate_items( + offset=offset, + limit=limit, + clean=clean, + desc=desc, + fields=fields, + omit=omit, + unwind=unwind, + skip_empty=skip_empty, + skip_hidden=skip_hidden, + ): + yield item @classmethod async def _check_and_serialize(cls, item: Mapping[str, JsonSerializable], index: int | None = None) -> str: diff --git a/src/apify/storage_clients/_apify/_key_value_store_client.py b/src/apify/storage_clients/_apify/_key_value_store_client.py index 9fdce93c4..24b39c901 100644 --- a/src/apify/storage_clients/_apify/_key_value_store_client.py +++ b/src/apify/storage_clients/_apify/_key_value_store_client.py @@ -11,6 +11,7 @@ from ._api_client_creation import create_storage_api_client from ._models import ApifyKeyValueStoreMetadata +from apify.errors import ActorError, catch_client_errors, map_client_errors if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -42,11 +43,12 @@ def __init__( """A lock to ensure that only one operation is performed at a time.""" @override + @catch_client_errors async def get_metadata(self) -> ApifyKeyValueStoreMetadata: metadata = await self._api_client.get() if metadata is None: - raise ValueError('Failed to retrieve key-value store metadata.') + raise ActorError('Failed to retrieve key-value store metadata.') return ApifyKeyValueStoreMetadata( id=metadata.id, @@ -58,6 +60,7 @@ async def get_metadata(self) -> ApifyKeyValueStoreMetadata: ) @classmethod + @catch_client_errors async def open( cls, *, @@ -110,16 +113,19 @@ async def purge(self) -> None: ) @override + @catch_client_errors async def drop(self) -> None: async with self._lock: await self._api_client.delete() @override + @catch_client_errors async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: response = await self._api_client.get_record(key) return KeyValueStoreRecord.model_validate(response) if response else None @override + @catch_client_errors async def set_value(self, *, key: str, value: Any, content_type: str | None = None) -> None: async with self._lock: await self._api_client.set_record( @@ -129,6 +135,7 @@ async def set_value(self, *, key: str, value: Any, content_type: str | None = No ) @override + @catch_client_errors async def delete_value(self, *, key: str) -> None: async with self._lock: await self._api_client.delete_record(key=key) @@ -142,33 +149,36 @@ async def iterate_keys( ) -> AsyncIterator[KeyValueStoreRecordMetadata]: count = 0 - while True: - list_key_page = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) - - for item in list_key_page.items: - record_metadata = KeyValueStoreRecordMetadata( - key=item.key, - size=item.size, - content_type='application/octet-stream', # Content type not available from list_keys - ) - yield record_metadata - count += 1 - - # If we've reached the limit, stop yielding - if limit and count >= limit: + with map_client_errors(): + while True: + list_key_page = await self._api_client.list_keys(exclusive_start_key=exclusive_start_key) + + for item in list_key_page.items: + record_metadata = KeyValueStoreRecordMetadata( + key=item.key, + size=item.size, + content_type='application/octet-stream', # Content type not available from list_keys + ) + yield record_metadata + count += 1 + + # If we've reached the limit, stop yielding + if limit and count >= limit: + break + + # If we've reached the limit or there are no more pages, exit the loop + if (limit and count >= limit) or not list_key_page.is_truncated: break - # If we've reached the limit or there are no more pages, exit the loop - if (limit and count >= limit) or not list_key_page.is_truncated: - break - - exclusive_start_key = list_key_page.next_exclusive_start_key + exclusive_start_key = list_key_page.next_exclusive_start_key @override + @catch_client_errors async def record_exists(self, *, key: str) -> bool: return await self._api_client.record_exists(key=key) @override + @catch_client_errors async def get_public_url(self, *, key: str) -> str: """Get a URL for the given key that may be used to publicly access the value in the remote key-value store. diff --git a/src/apify/storage_clients/_apify/_request_queue_client.py b/src/apify/storage_clients/_apify/_request_queue_client.py index de6737806..83c0c03da 100644 --- a/src/apify/storage_clients/_apify/_request_queue_client.py +++ b/src/apify/storage_clients/_apify/_request_queue_client.py @@ -11,6 +11,7 @@ from ._models import ApifyRequestQueueMetadata, RequestQueueStats from ._request_queue_shared_client import ApifyRequestQueueSharedClient from ._request_queue_single_client import ApifyRequestQueueSingleClient +from apify.errors import ActorError, catch_client_errors if TYPE_CHECKING: from collections.abc import Sequence @@ -66,6 +67,7 @@ def __init__( raise RuntimeError(f"Unsupported access type: {access}. Allowed values are 'single' or 'shared'.") @override + @catch_client_errors async def get_metadata(self) -> ApifyRequestQueueMetadata: """Retrieve current metadata about the request queue. @@ -79,7 +81,7 @@ async def get_metadata(self) -> ApifyRequestQueueMetadata: metadata = await self._api_client.get() if metadata is None: - raise ValueError('Failed to fetch request queue metadata from the API.') + raise ActorError('Failed to fetch request queue metadata from the API.') total_request_count = metadata.total_request_count handled_request_count = metadata.handled_request_count @@ -101,6 +103,7 @@ async def get_metadata(self) -> ApifyRequestQueueMetadata: ) @classmethod + @catch_client_errors async def open( cls, *, @@ -145,7 +148,7 @@ async def open( # Fetch initial metadata from the API. raw_metadata = await api_client.get() if raw_metadata is None: - raise ValueError('Failed to retrieve request queue metadata from the API.') + raise ActorError('Failed to retrieve request queue metadata from the API.') metadata = ApifyRequestQueueMetadata.model_validate(raw_metadata.model_dump(by_alias=True)) return cls( @@ -162,10 +165,12 @@ async def purge(self) -> None: ) @override + @catch_client_errors async def drop(self) -> None: await self._api_client.delete() @override + @catch_client_errors async def add_batch_of_requests( self, requests: Sequence[Request], @@ -175,18 +180,22 @@ async def add_batch_of_requests( return await self._implementation.add_batch_of_requests(requests, forefront=forefront) @override + @catch_client_errors async def fetch_next_request(self) -> Request | None: return await self._implementation.fetch_next_request() @override + @catch_client_errors async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None: return await self._implementation.mark_request_as_handled(request) @override + @catch_client_errors async def get_request(self, unique_key: str) -> Request | None: return await self._implementation.get_request(unique_key) @override + @catch_client_errors async def reclaim_request( self, request: Request, @@ -196,5 +205,6 @@ async def reclaim_request( return await self._implementation.reclaim_request(request, forefront=forefront) @override + @catch_client_errors async def is_empty(self) -> bool: return await self._implementation.is_empty() diff --git a/tests/unit/test_errors.py b/tests/unit/test_errors.py index 05261ec34..c3c5bd6e8 100644 --- a/tests/unit/test_errors.py +++ b/tests/unit/test_errors.py @@ -17,7 +17,6 @@ ) from apify_client.errors import RateLimitError as ClientRateLimitError -import apify from apify.errors import ( ActorAuthenticationError, ActorChargeLimitExceededError, @@ -70,7 +69,7 @@ def _make_run(*, status: str, exit_code: int | None = None, status_message: str def test_actor_error_defaults() -> None: error = ActorError('something went wrong') - assert error.code == 'apify-error' + assert error.code == 'actor-error' assert error.retryable is False assert str(error) == 'something went wrong' @@ -80,7 +79,7 @@ def test_actor_error_overrides_are_instance_scoped() -> None: assert error.code == 'custom' assert error.retryable is True # Overriding on an instance must not leak to the class default. - assert ActorError.code == 'apify-error' + assert ActorError.code == 'actor-error' assert ActorError.retryable is False @@ -167,18 +166,3 @@ def test_from_client_error_unknown_exception_falls_back() -> None: assert type(mapped) is ActorError assert mapped.retryable is False assert 'not a client error' in str(mapped) - - -def test_errors_exported_from_top_level() -> None: - for name in ( - 'ActorError', - 'ActorRunError', - 'ActorTimeoutError', - 'ActorAuthenticationError', - 'ActorChargeLimitExceededError', - 'ActorInputValidationError', - 'ActorRateLimitError', - ): - assert hasattr(apify, name) - assert name in apify.__all__ - assert getattr(apify, name) is getattr(apify.errors, name)