From 4d1d3f5f7aa72b5e97140a1ab275850b7e97a49e Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 16:00:20 +0200 Subject: [PATCH 01/72] ref(boto3): split integration into internal modules --- sentry_sdk/integrations/boto3/__init__.py | 13 ++++++ sentry_sdk/integrations/boto3/_client.py | 44 +++++++++++++++++++ .../{boto3.py => boto3/_instrumentation.py} | 36 +-------------- tests/integrations/boto3/test_client.py | 6 +++ tests/integrations/boto3/test_s3.py | 4 +- 5 files changed, 67 insertions(+), 36 deletions(-) create mode 100644 sentry_sdk/integrations/boto3/__init__.py create mode 100644 sentry_sdk/integrations/boto3/_client.py rename sentry_sdk/integrations/{boto3.py => boto3/_instrumentation.py} (85%) create mode 100644 tests/integrations/boto3/test_client.py diff --git a/sentry_sdk/integrations/boto3/__init__.py b/sentry_sdk/integrations/boto3/__init__.py new file mode 100644 index 0000000000..3815763fdf --- /dev/null +++ b/sentry_sdk/integrations/boto3/__init__.py @@ -0,0 +1,13 @@ +from sentry_sdk.integrations import Integration + + +class Boto3Integration(Integration): + identifier = "boto3" + origin = f"auto.http.{identifier}" + + @staticmethod + def setup_once() -> None: + # local import to avoid import cycle + from sentry_sdk.integrations.boto3._client import _patch_botocore_client + + _patch_botocore_client() diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py new file mode 100644 index 0000000000..b5803b6e80 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_client.py @@ -0,0 +1,44 @@ +from functools import partial +from typing import TYPE_CHECKING + +from sentry_sdk.integrations import DidNotEnable, _check_minimum_version +from sentry_sdk.integrations.boto3._instrumentation import ( + _sentry_after_call, + _sentry_after_call_error, + _sentry_before_sign, + _sentry_request_created, +) +from sentry_sdk.utils import parse_version + +if TYPE_CHECKING: + from typing import Any + +try: + from botocore import __version__ as BOTOCORE_VERSION + from botocore.client import BaseClient +except ImportError: + raise DidNotEnable("botocore is not installed") + + +def _patch_botocore_client() -> None: + from sentry_sdk.integrations.boto3 import Boto3Integration + + version = parse_version(BOTOCORE_VERSION) + _check_minimum_version(Boto3Integration, version, "botocore") + + orig_init = BaseClient.__init__ + + def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: + orig_init(self, *args, **kwargs) + meta = self.meta + service_id = meta.service_model.service_id + meta.events.register( + "request-created", + partial(_sentry_request_created, service_id=service_id), + ) + # run after other `before-sign` handlers, allowing it to see and preserve existing baggage. + meta.events.register_last("before-sign", _sentry_before_sign) + meta.events.register("after-call", _sentry_after_call) + meta.events.register("after-call-error", _sentry_after_call_error) + + BaseClient.__init__ = sentry_patched_init # type: ignore diff --git a/sentry_sdk/integrations/boto3.py b/sentry_sdk/integrations/boto3/_instrumentation.py similarity index 85% rename from sentry_sdk/integrations/boto3.py rename to sentry_sdk/integrations/boto3/_instrumentation.py index 18d7accf6d..752da96053 100644 --- a/sentry_sdk/integrations/boto3.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -1,9 +1,9 @@ -from functools import partial from typing import TYPE_CHECKING import sentry_sdk from sentry_sdk.consts import OP, SPANDATA -from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version +from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( @@ -16,7 +16,6 @@ from sentry_sdk.utils import ( capture_internal_exceptions, parse_url, - parse_version, ) if TYPE_CHECKING: @@ -26,43 +25,12 @@ try: - from botocore import __version__ as BOTOCORE_VERSION from botocore.awsrequest import AWSRequest - from botocore.client import BaseClient from botocore.response import StreamingBody except ImportError: raise DidNotEnable("botocore is not installed") -class Boto3Integration(Integration): - identifier = "boto3" - origin = f"auto.http.{identifier}" - - @staticmethod - def setup_once() -> None: - version = parse_version(BOTOCORE_VERSION) - _check_minimum_version(Boto3Integration, version, "botocore") - - orig_init = BaseClient.__init__ - - def sentry_patched_init( - self: "BaseClient", *args: "Any", **kwargs: "Any" - ) -> None: - orig_init(self, *args, **kwargs) - meta = self.meta - service_id = meta.service_model.service_id - meta.events.register( - "request-created", - partial(_sentry_request_created, service_id=service_id), - ) - # run after other `before-sign` handlers, allowing it to see and preserve existing baggage. - meta.events.register_last("before-sign", _sentry_before_sign) - meta.events.register("after-call", _sentry_after_call) - meta.events.register("after-call-error", _sentry_after_call_error) - - BaseClient.__init__ = sentry_patched_init # type: ignore - - def _sentry_request_created( service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" ) -> None: diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py new file mode 100644 index 0000000000..db8f1b9263 --- /dev/null +++ b/tests/integrations/boto3/test_client.py @@ -0,0 +1,6 @@ +from sentry_sdk.integrations.boto3 import Boto3Integration + + +def test_public_api(): + assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" + assert Boto3Integration.identifier == "boto3" diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index 888e44ee4c..910c520534 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -253,7 +253,7 @@ def test_omit_url_data_if_parsing_fails( items = capture_items("span") with mock.patch( - "sentry_sdk.integrations.boto3.parse_url", + "sentry_sdk.integrations.boto3._instrumentation.parse_url", side_effect=ValueError, ): with sentry_sdk.traces.start_span( @@ -294,7 +294,7 @@ def test_omit_url_data_if_parsing_fails( events = capture_events() with mock.patch( - "sentry_sdk.integrations.boto3.parse_url", + "sentry_sdk.integrations.boto3._instrumentation.parse_url", side_effect=ValueError, ): with sentry_sdk.start_transaction() as transaction, MockResponse( From 0c238306c52fc0f603351684f2bdd41d1f8e25fa Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 10:31:37 +0200 Subject: [PATCH 02/72] ref(boto3): Move consts to --- sentry_sdk/integrations/boto3/__init__.py | 18 +++++++++++++----- .../integrations/boto3/_instrumentation.py | 9 +++++---- sentry_sdk/integrations/boto3/consts.py | 2 ++ 3 files changed, 20 insertions(+), 9 deletions(-) create mode 100644 sentry_sdk/integrations/boto3/consts.py diff --git a/sentry_sdk/integrations/boto3/__init__.py b/sentry_sdk/integrations/boto3/__init__.py index 3815763fdf..d36b44d271 100644 --- a/sentry_sdk/integrations/boto3/__init__.py +++ b/sentry_sdk/integrations/boto3/__init__.py @@ -1,13 +1,21 @@ -from sentry_sdk.integrations import Integration +from sentry_sdk.integrations import DidNotEnable, Integration, _check_minimum_version +from sentry_sdk.integrations.boto3._client import _patch_botocore_client +from sentry_sdk.integrations.boto3.consts import IDENTIFIER, ORIGIN +from sentry_sdk.utils import parse_version + +try: + from botocore import __version__ as BOTOCORE_VERSION +except ImportError: + raise DidNotEnable("botocore is not installed") class Boto3Integration(Integration): - identifier = "boto3" - origin = f"auto.http.{identifier}" + identifier = IDENTIFIER + origin = ORIGIN @staticmethod def setup_once() -> None: - # local import to avoid import cycle - from sentry_sdk.integrations.boto3._client import _patch_botocore_client + version = parse_version(BOTOCORE_VERSION) + _check_minimum_version(Boto3Integration, version, "botocore") _patch_botocore_client() diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 752da96053..c233e93760 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -4,6 +4,7 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( @@ -61,7 +62,7 @@ def _sentry_request_created( name=description, attributes={ "sentry.op": OP.HTTP_CLIENT, - "sentry.origin": Boto3Integration.origin, + "sentry.origin": ORIGIN, SPANDATA.RPC_METHOD: f"{service_id}/{operation_name}", }, ) @@ -73,7 +74,7 @@ def _sentry_request_created( span = sentry_sdk.start_span( op=OP.HTTP_CLIENT, name=description, - origin=Boto3Integration.origin, + origin=ORIGIN, ) if parsed_url: @@ -182,14 +183,14 @@ def _sentry_after_call( parent_span=span, attributes={ "sentry.op": OP.HTTP_CLIENT_STREAM, - "sentry.origin": Boto3Integration.origin, + "sentry.origin": ORIGIN, }, ) else: streaming_span = span.start_child( op=OP.HTTP_CLIENT_STREAM, name=span.description, - origin=Boto3Integration.origin, + origin=ORIGIN, ) orig_read = body.read diff --git a/sentry_sdk/integrations/boto3/consts.py b/sentry_sdk/integrations/boto3/consts.py new file mode 100644 index 0000000000..67d0d18239 --- /dev/null +++ b/sentry_sdk/integrations/boto3/consts.py @@ -0,0 +1,2 @@ +IDENTIFIER = "boto3" +ORIGIN = f"auto.http.{IDENTIFIER}" From b45f76c96b512fc27fddfc79483cfe27b909e06a Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 10:37:54 +0200 Subject: [PATCH 03/72] fix(boto3): Move imports --- sentry_sdk/integrations/boto3/_instrumentation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index c233e93760..803b9233e8 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -3,7 +3,6 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span @@ -35,6 +34,8 @@ def _sentry_request_created( service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" ) -> None: + from sentry_sdk.integrations.boto3 import Boto3Integration + description = "aws.%s.%s" % (service_id.hyphenize(), operation_name) client = sentry_sdk.get_client() @@ -110,6 +111,8 @@ def _sentry_request_created( def _sentry_before_sign( request: "AWSRequest", signature_version: "Any", **kwargs: "Any" ) -> None: + from sentry_sdk.integrations.boto3 import Boto3Integration + client = sentry_sdk.get_client() if client.get_integration(Boto3Integration) is None: return From ab843bc5cc5345f5566cec69732280a7a5b20481 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 13:50:22 +0200 Subject: [PATCH 04/72] ref(boto3): Remove dupliocate `_check_minimum_version()` --- sentry_sdk/integrations/boto3/_client.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index b5803b6e80..5fa901499b 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -1,31 +1,24 @@ from functools import partial from typing import TYPE_CHECKING -from sentry_sdk.integrations import DidNotEnable, _check_minimum_version +from sentry_sdk.integrations import DidNotEnable from sentry_sdk.integrations.boto3._instrumentation import ( _sentry_after_call, _sentry_after_call_error, _sentry_before_sign, _sentry_request_created, ) -from sentry_sdk.utils import parse_version if TYPE_CHECKING: from typing import Any try: - from botocore import __version__ as BOTOCORE_VERSION from botocore.client import BaseClient except ImportError: raise DidNotEnable("botocore is not installed") def _patch_botocore_client() -> None: - from sentry_sdk.integrations.boto3 import Boto3Integration - - version = parse_version(BOTOCORE_VERSION) - _check_minimum_version(Boto3Integration, version, "botocore") - orig_init = BaseClient.__init__ def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: From b21b888842ad14152ffcca8cc89b8f0f141a1050 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 14:08:41 +0200 Subject: [PATCH 05/72] ref(boto3): Update integration checks to use IDENTIFIER constant --- sentry_sdk/integrations/boto3/_instrumentation.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 803b9233e8..1e6168d435 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -3,7 +3,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.integrations.boto3.consts import ORIGIN +from sentry_sdk.integrations.boto3.consts import IDENTIFIER, ORIGIN from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( @@ -34,12 +34,11 @@ def _sentry_request_created( service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" ) -> None: - from sentry_sdk.integrations.boto3 import Boto3Integration description = "aws.%s.%s" % (service_id.hyphenize(), operation_name) client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: + if client.get_integration(IDENTIFIER) is None: return parsed_url = None @@ -111,10 +110,9 @@ def _sentry_request_created( def _sentry_before_sign( request: "AWSRequest", signature_version: "Any", **kwargs: "Any" ) -> None: - from sentry_sdk.integrations.boto3 import Boto3Integration client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: + if client.get_integration(IDENTIFIER) is None: return with capture_internal_exceptions(): From f91d31118c98fe0d577d8ea9fef79171a9ea50a9 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 16:44:48 +0200 Subject: [PATCH 06/72] fix(boto3): harden StreamingBody span finalization --- .../integrations/boto3/_instrumentation.py | 124 +++++++++++++---- tests/integrations/boto3/test_client.py | 128 ++++++++++++++++++ 2 files changed, 222 insertions(+), 30 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 1e6168d435..3b96feae92 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -1,10 +1,10 @@ from typing import TYPE_CHECKING import sentry_sdk -from sentry_sdk.consts import OP, SPANDATA +from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS from sentry_sdk.integrations import DidNotEnable from sentry_sdk.integrations.boto3.consts import IDENTIFIER, ORIGIN -from sentry_sdk.traces import StreamedSpan +from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( add_http_breadcrumb, @@ -162,26 +162,39 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: ) -def _sentry_after_call( - context: "Dict[str, Any]", parsed: "Dict[str, Any]", **kwargs: "Any" +def _finish_span( + span: "Union[Span, StreamedSpan]", + error: "Optional[BaseException]" = None, ) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) + with capture_internal_exceptions(): + if not isinstance(span, StreamedSpan): + if error is not None: + span.set_status(SPANSTATUS.INTERNAL_ERROR) + span.finish() + return - # Span could be absent if the integration is disabled. - if span is None: - return + if error is None: + span.end() + else: + span.__exit__(type(error), error, error.__traceback__) - span.__exit__(None, None, None) + +def _instrument_streaming_body( + span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]" +) -> bool: + if isinstance(span, NoOpStreamedSpan): + return False body = parsed.get("Body") if not isinstance(body, StreamingBody): - return + return False streaming_span: "Union[Span, StreamedSpan]" if isinstance(span, StreamedSpan): streaming_span = sentry_sdk.traces.start_span( name=span.name, parent_span=span, + active=False, attributes={ "sentry.op": OP.HTTP_CLIENT_STREAM, "sentry.origin": ORIGIN, @@ -196,35 +209,86 @@ def _sentry_after_call( orig_read = body.read orig_close = body.close + raw_stream = body._raw_stream # type: ignore[attr-defined] + orig_raw_close = raw_stream.close + finished = False + + def finish(error: "Optional[BaseException]" = None) -> None: + nonlocal finished + if finished: + return + + finished = True + _finish_span(streaming_span, error) + + def content_length_reached() -> bool: + content_length = getattr(body, "_content_length", None) + amount_read = getattr(body, "_amount_read", None) + return ( + content_length is not None + and amount_read is not None + and amount_read >= int(content_length) + ) def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: try: ret = orig_read(*args, **kwargs) - if ret: - return ret - - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() + with capture_internal_exceptions(): + amount = args[0] if args else kwargs.get("amt") + if ( + amount is None + or amount < 0 + or (amount > 0 and not ret) + or content_length_reached() + ): + finish() return ret - except Exception: - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() + except BaseException as error: + finish(error) raise - body.read = sentry_streaming_body_read # type: ignore - def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: - if isinstance(streaming_span, StreamedSpan): - streaming_span.end() - else: - streaming_span.finish() - orig_close(*args, **kwargs) + try: + orig_close(*args, **kwargs) + finish() + except BaseException as error: + finish(error) + raise + + def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: + try: + orig_raw_close(*args, **kwargs) + finish() + except BaseException as error: + finish(error) + raise + + try: + # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing + # StreamingBody.close(), so both levels need to be instrumented. + raw_stream.close = sentry_raw_stream_close + body.read = sentry_streaming_body_read # type: ignore + body.close = sentry_streaming_body_close # type: ignore + except Exception: + finish() + raise + + return True + + +def _sentry_after_call( + context: "Dict[str, Any]", parsed: "Dict[str, Any]", **kwargs: "Any" +) -> None: + span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) + + # Span could be absent if the integration is disabled. + if span is None: + return - body.close = sentry_streaming_body_close # type: ignore + span.__exit__(None, None, None) + + with capture_internal_exceptions(): + _instrument_streaming_body(span, parsed) def _sentry_after_call_error( diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index db8f1b9263..de1d44de49 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -1,6 +1,134 @@ +import boto3 +import pytest +from botocore.awsrequest import AWSResponse +from botocore.config import Config + +import sentry_sdk +from sentry_sdk.consts import OP from sentry_sdk.integrations.boto3 import Boto3Integration +from tests.integrations.boto3.aws_mock import Body + +session = boto3.Session( # type: ignore[attr-defined] + aws_access_key_id="-", + aws_secret_access_key="-", + region_name="eu-north-1", +) def test_public_api(): assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" assert Boto3Integration.identifier == "boto3" + + +@pytest.fixture +def client_factory(sentry_init, monkeypatch, span_streaming): + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream" if span_streaming else "static", + # avoid SDK's machine hostname being used as server name. + server_name="", + ) + # remove retry delay to speed up tests + monkeypatch.setattr("botocore.endpoint.time.sleep", lambda delay: None) + + def make_client(service_name="s3", attempt_count=1, **client_kwargs): + return session.client( + service_name, + config=Config( + # `total_max_attempts` includes the initial request. + retries={"total_max_attempts": attempt_count, "mode": "standard"} + ), + **client_kwargs, + ) + + return make_client + + +def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streaming): + items = capture_items() + + if span_streaming: + with sentry_sdk.traces.start_span(name="parent"): # type: ignore[attr-defined] + invoke_client_method() + + sentry_sdk.flush() + spans = [ + item.payload + for item in items + if item.type == "span" + and item.payload["attributes"].get("sentry.origin") + == Boto3Integration.origin + ] + else: + with sentry_sdk.start_transaction(): + invoke_client_method() + + transaction = next(item.payload for item in items if item.type == "transaction") + spans = [ + span + for span in transaction["spans"] + if span["origin"] == Boto3Integration.origin + ] + + spans_by_op = {} + for span in spans: + op = ( + span["attributes"].get("sentry.op") if span_streaming else span["op"] + ) + spans_by_op.setdefault(op, []).append(span) + return spans_by_op + + +def _assert_span_finished(span, span_streaming): + finished_timestamp = "end_timestamp" if span_streaming else "timestamp" + assert span[finished_timestamp] is not None + + +def _assert_one_failed_span(spans, span_streaming): + assert len(spans) == 1 + assert spans[0]["status"] in ("error", "internal_error") + _assert_span_finished(spans[0], span_streaming) + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_body_read_failure_finishes_stream_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + original_exception = OSError("stream read failed") + + class _FailingBody(Body): + def __init__(self, exception): + super().__init__(b"") + self._exception = exception + + def read(self, *args, **kwargs): + raise self._exception + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + {"content-length": "1"}, + _FailingBody(original_exception), + ) + + client.meta.events.register("before-send", respond) + + def invoke_client_method_and_read_body(): + body = client.get_object(Bucket="bucket", Key="foo")["Body"] + with pytest.raises(OSError) as exc_info: + body.read() + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method_and_read_body, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(client_spans) == 1 + _assert_one_failed_span(stream_spans, span_streaming) From ec2a236e3ef854ca083c51e903d1dcfa730a5d77 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 17:13:42 +0200 Subject: [PATCH 07/72] lint --- tests/integrations/boto3/test_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index de1d44de49..df08b19dd5 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -73,9 +73,7 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami spans_by_op = {} for span in spans: - op = ( - span["attributes"].get("sentry.op") if span_streaming else span["op"] - ) + op = span["attributes"].get("sentry.op") if span_streaming else span["op"] spans_by_op.setdefault(op, []).append(span) return spans_by_op From 3b53fa49a43764bdae9ee5efec84e96ef86b075f Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 10:46:59 +0200 Subject: [PATCH 08/72] ref(boto3): renaming vars --- .../integrations/boto3/_instrumentation.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 3b96feae92..307abf9472 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -213,7 +213,7 @@ def _instrument_streaming_body( orig_raw_close = raw_stream.close finished = False - def finish(error: "Optional[BaseException]" = None) -> None: + def finish_span(error: "Optional[BaseException]" = None) -> None: nonlocal finished if finished: return @@ -232,35 +232,35 @@ def content_length_reached() -> bool: def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: try: - ret = orig_read(*args, **kwargs) + read_return_value = orig_read(*args, **kwargs) with capture_internal_exceptions(): - amount = args[0] if args else kwargs.get("amt") + amount_of_bytes_requested = args[0] if args else kwargs.get("amt") if ( - amount is None - or amount < 0 - or (amount > 0 and not ret) + amount_of_bytes_requested is None + or amount_of_bytes_requested < 0 + or (amount_of_bytes_requested > 0 and not read_return_value) or content_length_reached() ): - finish() - return ret + finish_span() + return read_return_value except BaseException as error: - finish(error) + finish_span(error) raise def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: try: orig_close(*args, **kwargs) - finish() + finish_span() except BaseException as error: - finish(error) + finish_span(error) raise def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: try: orig_raw_close(*args, **kwargs) - finish() + finish_span() except BaseException as error: - finish(error) + finish_span(error) raise try: @@ -270,7 +270,7 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: body.read = sentry_streaming_body_read # type: ignore body.close = sentry_streaming_body_close # type: ignore except Exception: - finish() + finish_span() raise return True From eea1fc10564678cf239e55289e6e6b61a02e3ccf Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 11:06:44 +0200 Subject: [PATCH 09/72] ref(boto3): Add specific comment on why we initialize with --- sentry_sdk/integrations/boto3/_instrumentation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 307abf9472..8c58f8a779 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -193,7 +193,12 @@ def _instrument_streaming_body( if isinstance(span, StreamedSpan): streaming_span = sentry_sdk.traces.start_span( name=span.name, + # `parent_span` is set explicitly to the boto span. parent_span=span, + # avoid making the streaming span the current span on the scope since the application might + # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it + # could restore the parent span on the scope, breaking the parent-child relation of newly + # created spans; 2. newly created spans would be attached to the streaming span. active=False, attributes={ "sentry.op": OP.HTTP_CLIENT_STREAM, From 364f54776d3fa4fe7c1da526794ca7c67407820b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 18:07:05 +0200 Subject: [PATCH 10/72] fix(boto3): ensure span finishes correctly when reading streaming body --- sentry_sdk/integrations/boto3/_instrumentation.py | 8 +++++++- tests/integrations/boto3/test_client.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 8c58f8a779..6d589f7d2b 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -217,6 +217,7 @@ def _instrument_streaming_body( raw_stream = body._raw_stream # type: ignore[attr-defined] orig_raw_close = raw_stream.close finished = False + read_in_progress = False def finish_span(error: "Optional[BaseException]" = None) -> None: nonlocal finished @@ -236,6 +237,8 @@ def content_length_reached() -> bool: ) def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: + nonlocal read_in_progress + read_in_progress = True try: read_return_value = orig_read(*args, **kwargs) with capture_internal_exceptions(): @@ -251,6 +254,8 @@ def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: except BaseException as error: finish_span(error) raise + finally: + read_in_progress = False def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: try: @@ -263,7 +268,8 @@ def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: try: orig_raw_close(*args, **kwargs) - finish_span() + if not read_in_progress: + finish_span() except BaseException as error: finish_span(error) raise diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index df08b19dd5..6c81ba9313 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -104,6 +104,8 @@ def __init__(self, exception): self._exception = exception def read(self, *args, **kwargs): + # urllib3 closes the response before propagating some read failures. + self.close() raise self._exception def respond(request, **kwargs): From 4f3cfcefa6ae578f38819308aa67254e3805a0f9 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 12:39:08 +0200 Subject: [PATCH 11/72] fix(boto3): trace the complete client-call lifecycle --- sentry_sdk/consts.py | 12 + sentry_sdk/integrations/boto3/_client.py | 98 ++++++- sentry_sdk/integrations/boto3/_context.py | 44 +++ .../integrations/boto3/_instrumentation.py | 203 +++++++------ sentry_sdk/integrations/stdlib.py | 12 +- tests/integrations/boto3/test_client.py | 268 +++++++++++++++++- tests/integrations/boto3/test_s3.py | 47 +-- 7 files changed, 568 insertions(+), 116 deletions(-) create mode 100644 sentry_sdk/integrations/boto3/_context.py diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 9bad8aa7db..b1470dabf6 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -1172,6 +1172,18 @@ class SPANDATA: Used in inbound filters. """ + SENTRY_OP = "sentry.op" + """ + The operation of a span. + Example: "http.client" + """ + + SENTRY_ORIGIN = "sentry.origin" + """ + The origin of the instrumentation (e.g. span, log, etc.) + Example: "auto.http.otel.fastify" + """ + SENTRY_RELEASE = "sentry.release" """ The Sentry release. diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 5fa901499b..33d9899055 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -1,37 +1,109 @@ -from functools import partial +from contextlib import contextmanager from typing import TYPE_CHECKING +import sentry_sdk from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.integrations.boto3._context import AwsCallContext from sentry_sdk.integrations.boto3._instrumentation import ( - _sentry_after_call, - _sentry_after_call_error, + _finish_span, + _instrument_streaming_body, _sentry_before_sign, _sentry_request_created, + _start_client_span, ) +from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan +from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any + from typing import Any, Iterator, Optional, Union + + from sentry_sdk.tracing import Span try: from botocore.client import BaseClient except ImportError: - raise DidNotEnable("botocore is not installed") + raise DidNotEnable("botocore not installed") + + +@contextmanager +def _activate_client_span(span: "StreamedSpan") -> "Iterator[StreamedSpan]": + """Temporarily activate an inactive boto span without ending it.""" + if isinstance(span, NoOpStreamedSpan): + yield span + return + + scope = sentry_sdk.get_current_scope() + previous_span = scope.streamed_span + scope.streamed_span = span + try: + yield span + finally: + scope.streamed_span = previous_span def _patch_botocore_client() -> None: + from sentry_sdk.integrations.boto3 import Boto3Integration + orig_init = BaseClient.__init__ + orig_make_api_call = BaseClient._make_api_call # type: ignore def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: orig_init(self, *args, **kwargs) meta = self.meta - service_id = meta.service_model.service_id - meta.events.register( - "request-created", - partial(_sentry_request_created, service_id=service_id), - ) - # run after other `before-sign` handlers, allowing it to see and preserve existing baggage. + meta.events.register("request-created", _sentry_request_created) + # run after other `before-sign` handlers so existing baggage is preserved. meta.events.register_last("before-sign", _sentry_before_sign) - meta.events.register("after-call", _sentry_after_call) - meta.events.register("after-call-error", _sentry_after_call_error) + + def sentry_patched_make_api_call( + self: "BaseClient", operation_name: str, api_params: "Any" + ) -> "Any": + """ + Track a single API call, including retries, serialization, and endpoint + resolution. For streaming responses, keep the span open until the + response body is consumed or closed. + https://github.com/boto/botocore/blob/develop/botocore/client.py + https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span + """ + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return orig_make_api_call(self, operation_name, api_params) + + ctx = AwsCallContext(operation_name) + + # add optional metadata to context. + with capture_internal_exceptions(): + ctx.add_metadata(self) + + span: "Optional[Union[Span, StreamedSpan]]" = None + with capture_internal_exceptions(): + span = _start_client_span(ctx) + + if span is None: + return orig_make_api_call(self, operation_name, api_params) + + # activate without finishing; a streaming response may outlive the call. + span_ctx = ( + _activate_client_span(span) if isinstance(span, StreamedSpan) else span + ) + + try: + with span_ctx: + parsed = orig_make_api_call(self, operation_name, api_params) + except BaseException as error: + # finish `StreamedSpan` explicitly; static spans are finished by + # their context manager. + if isinstance(span, StreamedSpan): + _finish_span(span, error) + raise + + streaming_body_instrumented = False + with capture_internal_exceptions(): + streaming_body_instrumented = _instrument_streaming_body(span, parsed) + + # `StreamingBody`s finish their span when consumed or closed. + if isinstance(span, StreamedSpan) and not streaming_body_instrumented: + _finish_span(span) + return parsed BaseClient.__init__ = sentry_patched_init # type: ignore + BaseClient._make_api_call = sentry_patched_make_api_call # type: ignore diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py new file mode 100644 index 0000000000..38f7e0d76a --- /dev/null +++ b/sentry_sdk/integrations/boto3/_context.py @@ -0,0 +1,44 @@ +from typing import TYPE_CHECKING + +from sentry_sdk.integrations import DidNotEnable +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Any, Optional + +try: + from botocore.client import BaseClient +except ImportError: + raise DidNotEnable("botocore not installed") + + +class AwsCallContext: + __slots__ = ( + "service_id", + "service_id_hyphenized", + "operation_name", + ) + + def __init__(self, operation_name: str) -> None: + self.operation_name: str = operation_name + self.service_id: "Optional[str]" = None + self.service_id_hyphenized: "Optional[str]" = None + + def add_metadata(self, client: "BaseClient") -> None: + def _get_attr(obj: "Any", name: str) -> "Any": + if obj is None: + return None + + with capture_internal_exceptions(): + return getattr(obj, name) + + client_meta = _get_attr(client, "meta") + service_model = _get_attr(client_meta, "service_model") + + # modeled AWS service identity used in span names, e.g. `API Gateway`. + service_id = _get_attr(service_model, "service_id") + if service_id is not None: + with capture_internal_exceptions(): + self.service_id = str(service_id) + with capture_internal_exceptions(): + self.service_id_hyphenized = service_id.hyphenize() diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 6d589f7d2b..5a01d0eecd 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -19,28 +19,93 @@ ) if TYPE_CHECKING: - from typing import Any, Dict, Optional, Type, Union - - from botocore.model import ServiceId + from typing import Any, Dict, Optional, Union + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext try: from botocore.awsrequest import AWSRequest from botocore.response import StreamingBody except ImportError: - raise DidNotEnable("botocore is not installed") + raise DidNotEnable("botocore not installed") -def _sentry_request_created( - service_id: "ServiceId", request: "AWSRequest", operation_name: str, **kwargs: "Any" -) -> None: - - description = "aws.%s.%s" % (service_id.hyphenize(), operation_name) +def _start_client_span( + ctx: "AwsCallContext", +) -> "Optional[Union[Span, StreamedSpan]]": client = sentry_sdk.get_client() if client.get_integration(IDENTIFIER) is None: + return None + + # Use unknown if `service_id_hyphenized` is unavailable so a span name can + # still be created, e.g. "aws.unknown.GetObject". + service_name = ctx.service_id_hyphenized or "unknown" + span_name = "aws.%s.%s" % (service_name, ctx.operation_name) + + if has_span_streaming_enabled(client.options): + if sentry_sdk.traces.get_current_span() is None: + return None + + attributes: "Attributes" = { + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + } + if ctx.service_id: + attributes[SPANDATA.RPC_METHOD] = "%s/%s" % ( + ctx.service_id, + ctx.operation_name, + ) + return sentry_sdk.traces.start_span( + name=span_name, + attributes=attributes, + # `StreamingBody` responses outlive `_make_api_call()`. `_activate_client_span()` + # activates this span only while the call itself runs. + active=False, + ) + + span = sentry_sdk.start_span( + name=span_name, + op=OP.HTTP_CLIENT, + origin=ORIGIN, + ) + with capture_internal_exceptions(): + if ctx.service_id_hyphenized: + span.set_tag("aws.service_id", ctx.service_id_hyphenized) + span.set_tag("aws.operation_name", ctx.operation_name) + return span + + +def _set_request_attributes( + span: "Union[Span, StreamedSpan]", + request: "AWSRequest", +) -> None: + client = sentry_sdk.get_client() + + parsed_url = None + if request.url is not None: + with capture_internal_exceptions(): + parsed_url = parse_url(request.url, sanitize=False) + + if isinstance(span, StreamedSpan): + span.set_attributes(get_url_attributes(client, parsed_url)) + if request.method is not None: + span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) return + if parsed_url is not None: + span.set_data("aws.request.url", parsed_url.url) + span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) + span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) + + if request.method is not None: + span.set_data(SPANDATA.HTTP_METHOD, request.method) + + +def _add_request_breadcrumb(request: "AWSRequest") -> None: + client = sentry_sdk.get_client() + parsed_url = None if request.url is not None: with capture_internal_exceptions(): @@ -48,39 +113,12 @@ def _sentry_request_created( breadcrumb: "dict[str, Any]" = {} - is_span_streaming_enabled = has_span_streaming_enabled(client.options) - span: "Union[Span, StreamedSpan, None]" = None - if is_span_streaming_enabled: - url_attributes = get_url_attributes(client, parsed_url) - breadcrumb.update(url_attributes) - + if has_span_streaming_enabled(client.options): + breadcrumb.update(get_url_attributes(client, parsed_url)) if request.method is not None: breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method - - if sentry_sdk.traces.get_current_span() is not None: - span = sentry_sdk.traces.start_span( - name=description, - attributes={ - "sentry.op": OP.HTTP_CLIENT, - "sentry.origin": ORIGIN, - SPANDATA.RPC_METHOD: f"{service_id}/{operation_name}", - }, - ) - span.set_attributes(url_attributes) - - if request.method is not None: - span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) else: - span = sentry_sdk.start_span( - op=OP.HTTP_CLIENT, - name=description, - origin=ORIGIN, - ) - - if parsed_url: - span.set_data("aws.request.url", parsed_url.url) - span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) - span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) + if parsed_url is not None: breadcrumb.update( { "aws.request.url": parsed_url.url, @@ -89,21 +127,45 @@ def _sentry_request_created( } ) - span.set_tag("aws.service_id", service_id.hyphenize()) - span.set_tag("aws.operation_name", operation_name) if request.method is not None: - span.set_data(SPANDATA.HTTP_METHOD, request.method) breadcrumb[SPANDATA.HTTP_METHOD] = request.method - # We do it in order for subsequent http calls/retries be - # attached to this span. - span.__enter__() - add_http_breadcrumb(None, breadcrumb) - if span is not None: - # request.context is an open-ended data-structure - # where we can add anything useful in request life cycle. + +def _sentry_request_created( + request: "AWSRequest", operation_name: str, **kwargs: "Any" +) -> None: + """ + Enrich a single `AWSRequest` attempt. Botocore creates a fresh + `AWSRequest` on every retry. + https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 + """ + from sentry_sdk.integrations.boto3 import Boto3Integration + + client = sentry_sdk.get_client() + if client.get_integration(Boto3Integration) is None: + return + + with capture_internal_exceptions(): + _add_request_breadcrumb(request) + + span = ( + sentry_sdk.traces.get_current_span() + if has_span_streaming_enabled(client.options) + else sentry_sdk.get_current_span() + ) + if span is None: + return + + # An ignored streamed span is not activated; avoid enriching its parent. + if isinstance(span, StreamedSpan) and ( + span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN + ): + return + + _set_request_attributes(span, request) + # Each attempt has a fresh `request.context`; carry the active client span. request.context["_sentrysdk_span"] = span @@ -116,8 +178,9 @@ def _sentry_before_sign( return with capture_internal_exceptions(): - # presigned requests are executed later by another caller. Adding propagation - # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. + # Presigned requests are executed later by another caller. Adding propagation + # headers here would make those headers part of the signature, requiring the + # caller to reproduce the same values. if isinstance(signature_version, str) and signature_version.endswith( ("-query", "-presign-post") ): @@ -138,24 +201,23 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: del request.headers[key] request.headers[key] = value - # use span associated with this botocore request + # Use the span associated with this botocore request. span = request.context.get("_sentrysdk_span") - headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( span=span ) for header_name, header_value in headers: if header_name != BAGGAGE_HEADER_NAME: - # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values + # Normal headers (e.g. `sentry-trace`) are non-shared, so replace + # stale values. _replace_header(request, header_name, header_value) continue - # merge existing `baggage` values under single header + # Preserve third-party baggage and replace stale `sentry-*` values. existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) combined_baggage = { BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) } - # preserve third-party baggage, replace stale `sentry-*` values add_sentry_baggage_to_headers(combined_baggage, header_value) _replace_header( request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] @@ -226,6 +288,8 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: finished = True _finish_span(streaming_span, error) + if isinstance(span, StreamedSpan): + _finish_span(span, error) def content_length_reached() -> bool: content_length = getattr(body, "_content_length", None) @@ -285,30 +349,3 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raise return True - - -def _sentry_after_call( - context: "Dict[str, Any]", parsed: "Dict[str, Any]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(None, None, None) - - with capture_internal_exceptions(): - _instrument_streaming_body(span, parsed) - - -def _sentry_after_call_error( - context: "Dict[str, Any]", exception: "Type[BaseException]", **kwargs: "Any" -) -> None: - span: "Optional[Union[Span, StreamedSpan]]" = context.pop("_sentrysdk_span", None) - - # Span could be absent if the integration is disabled. - if span is None: - return - - span.__exit__(type(exception), exception, None) diff --git a/sentry_sdk/integrations/stdlib.py b/sentry_sdk/integrations/stdlib.py index 02f8b245f7..b9d0450ac2 100644 --- a/sentry_sdk/integrations/stdlib.py +++ b/sentry_sdk/integrations/stdlib.py @@ -288,7 +288,14 @@ def putrequest( breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = method breadcrumb.update(url_attributes) - if sentry_sdk.traces.get_current_span() is not None: + parent_span = sentry_sdk.traces.get_current_span() + if parent_span is not None: + is_inactive_boto3_span = ( + client.get_integration("boto3") is not None + and parent_span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) + == getattr(client.get_integration("boto3"), "origin", None) + and not getattr(parent_span, "active", True) + ) span = sentry_sdk.traces.start_span( name="%s %s" % ( @@ -300,6 +307,9 @@ def putrequest( "sentry.op": OP.HTTP_CLIENT, SPANDATA.HTTP_REQUEST_METHOD: method, }, + # boto3 integration owns span's lifecycle; keep child inactive so it + # can't restore boto3 span later on. + active=not is_inactive_boto3_span, ) for key, value in url_attributes.items(): diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 6c81ba9313..0ad81d38bd 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -1,11 +1,17 @@ +from http.server import BaseHTTPRequestHandler, HTTPServer +from threading import Thread + import boto3 import pytest from botocore.awsrequest import AWSResponse from botocore.config import Config +from botocore.exceptions import ClientError, EndpointConnectionError +from botocore.response import StreamingBody import sentry_sdk -from sentry_sdk.consts import OP +from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.stdlib import StdlibIntegration from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -15,11 +21,159 @@ ) +@pytest.fixture +def streaming_s3_server(): + class StreamingS3Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-Length", "1") + self.send_header("Content-Type", "application/octet-stream") + self.end_headers() + self.wfile.write(b"x") + self.wfile.flush() + + def log_message(self, *args): + pass + + server = HTTPServer(("127.0.0.1", 0), StreamingS3Handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_public_api(): assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" assert Boto3Integration.identifier == "boto3" +@pytest.mark.parametrize( + "consume", + ["read_exact", "context"], +) +def test_streaming_span_order_and_scope( + sentry_init, + capture_items, + streaming_s3_server, + consume, +): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream", + default_integrations=False, + integrations=[Boto3Integration(), StdlibIntegration()], + server_name="", + ) + server = streaming_s3_server + client = session.client( + "s3", + endpoint_url="http://127.0.0.1:%s" % server.server_port, + config=Config( + retries={"total_max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + body = client.get_object(Bucket="bucket", Key="key")["Body"] + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + if consume == "read_exact": + assert body.read(1) == b"x" + elif consume == "context": + if not hasattr(body, "__enter__"): + body.close() + pytest.skip("`StreamingBody` context manager is unavailable.") + with body as raw_stream: + assert raw_stream.read() == b"x" + + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] + assert probe._parent_span_id == parent.span_id + probe.end() + + body.close() + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + sentry_sdk.flush() + spans = [item.payload for item in items] + client_spans = [ + span + for span in spans + if span["name"] == "aws.s3.GetObject" + and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + ] + http_spans = [ + span + for span in spans + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + ] + stream_spans = [ + span + for span in spans + if span["name"] == "aws.s3.GetObject" + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + ] + assert len(client_spans) == 1 + assert len(http_spans) == 1 + assert len(stream_spans) == 1 + client_span = client_spans[0] + http_span = http_spans[0] + stream_span = stream_spans[0] + + assert http_span["parent_span_id"] == client_span["span_id"] + assert stream_span["parent_span_id"] == client_span["span_id"] + assert client_span["start_timestamp"] <= http_span["start_timestamp"] + assert http_span["start_timestamp"] <= stream_span["start_timestamp"] + assert http_span["end_timestamp"] <= stream_span["end_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] + + +def test_non_body_stream_does_not_delay_client_span(sentry_init, capture_items): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream", + integrations=[Boto3Integration()], + server_name="", + ) + client = session.client("lambda") + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + {"content-length": "1"}, + Body(b"x"), + ) + + client.meta.events.register("before-send", respond) + items = capture_items("span") + + with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + response = client.invoke(FunctionName="function") + assert isinstance(response["Payload"], StreamingBody) + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + + sentry_sdk.flush() + spans = [item.payload for item in items] + boto_spans = [ + span + for span in spans + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + ] + assert len(boto_spans) == 1 + assert boto_spans[0]["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + response["Payload"].close() + + @pytest.fixture def client_factory(sentry_init, monkeypatch, span_streaming): sentry_init( @@ -45,6 +199,25 @@ def make_client(service_name="s3", attempt_count=1, **client_kwargs): return make_client +def _mock_responses(client, status_codes): + request_span_ids = [] + + def record_request(request, **kwargs): + span = request.context.get("_sentrysdk_span") + assert span is not None + request_span_ids.append(span.span_id) + + def respond(request, **kwargs): + # `request_created` runs before `before_send`, so use zero-based index for current + # attempt; `min(..., len(status_codes) - 1)` clamps to last status to avoid `IndexError`. + response_index = min(len(request_span_ids) - 1, len(status_codes) - 1) + return AWSResponse(request.url, status_codes[response_index], {}, Body(b"")) + + client.meta.events.register("request-created", record_request) + client.meta.events.register("before-send", respond) + return request_span_ids + + def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streaming): items = capture_items() @@ -57,7 +230,7 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami item.payload for item in items if item.type == "span" - and item.payload["attributes"].get("sentry.origin") + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin ] else: @@ -73,7 +246,9 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami spans_by_op = {} for span in spans: - op = span["attributes"].get("sentry.op") if span_streaming else span["op"] + op = ( + span["attributes"].get(SPANDATA.SENTRY_OP) if span_streaming else span["op"] + ) spans_by_op.setdefault(op, []).append(span) return spans_by_op @@ -89,6 +264,91 @@ def _assert_one_failed_span(spans, span_streaming): _assert_span_finished(spans[0], span_streaming) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retry_attempts_share_one_client_span( + capture_items, + client_factory, + span_streaming, +): + attempt_count = 3 + client = client_factory(attempt_count=attempt_count) + request_span_ids = _mock_responses(client, [500] * (attempt_count - 1) + [200]) + + spans_by_op = _capture_boto3_spans_by_op( + lambda: client.head_object(Bucket="bucket", Key="foo"), + capture_items, + span_streaming, + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == attempt_count + # all `AWSRequest` instances created during retries reference the same client span. + assert len(set(request_span_ids)) == 1 + assert len(client_spans) == 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_retries_exhausted_has_one_failed_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory(attempt_count=2) + request_span_ids = _mock_responses(client, [500]) + + def attempt_failed_head_object_call(): + with pytest.raises(ClientError): + client.head_object(Bucket="bucket", Key="foo.pdf") + + spans_by_op = _capture_boto3_spans_by_op( + attempt_failed_head_object_call, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(request_span_ids) == 2 + assert len(set(request_span_ids)) == 1 + _assert_one_failed_span(client_spans, span_streaming) + + +@pytest.mark.parametrize( + "event_name", + [ + pytest.param("before-parameter-build"), + pytest.param("before-send"), + ], +) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_exception_is_unchanged_and_finishes_span( + capture_items, + client_factory, + span_streaming, + event_name, +): + client = client_factory() + if event_name == "before-send": + original_exception = EndpointConnectionError( + endpoint_url="https://s3.eu-north-1.amazonaws.com" + ) + else: + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + client.meta.events.register(event_name, raise_original_exception) + + def invoke_failing_client_method(): + with pytest.raises(type(original_exception)) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_body_read_failure_finishes_stream_span( capture_items, @@ -131,4 +391,6 @@ def invoke_client_method_and_read_body(): stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) assert len(client_spans) == 1 + if span_streaming: + _assert_one_failed_span(client_spans, span_streaming=True) _assert_one_failed_span(stream_spans, span_streaming) diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index 910c520534..dcd38dab9b 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -110,9 +110,19 @@ def test_streaming( spans = [item.payload for item in items] assert len(spans) == 3 - span1 = spans[0] - assert span1["attributes"]["sentry.op"] == "http.client" - assert span1["name"] == "aws.s3.GetObject" + stream_span, client_span, parent_span = spans + assert stream_span["attributes"]["sentry.op"] == "http.client.stream" + assert stream_span["name"] == "aws.s3.GetObject" + assert stream_span["parent_span_id"] == client_span["span_id"] + + assert client_span["attributes"]["sentry.op"] == "http.client" + assert client_span["name"] == "aws.s3.GetObject" + assert client_span["parent_span_id"] == parent_span["span_id"] + + assert parent_span["name"] == "custom parent" + assert parent_span["start_timestamp"] <= client_span["start_timestamp"] + assert client_span["start_timestamp"] <= stream_span["start_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] expected_attrs = { "http.request.method": "GET", @@ -131,17 +141,12 @@ def test_streaming( } if send_default_pii: expected_attrs["url.full"] = "https://bucket.s3.amazonaws.com/foo.pdf" - assert span1["attributes"] == ApproxDict(expected_attrs) + assert client_span["attributes"] == ApproxDict(expected_attrs) - assert "url.fragment" not in span1["attributes"] - assert "url.query" not in span1["attributes"] + assert "url.fragment" not in client_span["attributes"] + assert "url.query" not in client_span["attributes"] if not send_default_pii: - assert "url.full" not in span1["attributes"] - - span2 = spans[1] - assert span2["attributes"]["sentry.op"] == "http.client.stream" - assert span2["name"] == "aws.s3.GetObject" - assert span2["parent_span_id"] == span1["span_id"] + assert "url.full" not in client_span["attributes"] else: events = capture_events() @@ -207,10 +212,20 @@ def test_streaming_close( sentry_sdk.flush() spans = [item.payload for item in items] assert len(spans) == 3 - span1 = spans[0] - assert span1["attributes"]["sentry.op"] == "http.client" - span2 = spans[1] - assert span2["attributes"]["sentry.op"] == "http.client.stream" + + stream_span, client_span, parent_span = spans + assert stream_span["attributes"]["sentry.op"] == "http.client.stream" + assert stream_span["name"] == "aws.s3.GetObject" + assert stream_span["parent_span_id"] == client_span["span_id"] + + assert client_span["attributes"]["sentry.op"] == "http.client" + assert client_span["name"] == "aws.s3.GetObject" + assert client_span["parent_span_id"] == parent_span["span_id"] + + assert parent_span["name"] == "custom parent" + assert parent_span["start_timestamp"] <= client_span["start_timestamp"] + assert client_span["start_timestamp"] <= stream_span["start_timestamp"] + assert stream_span["end_timestamp"] <= client_span["end_timestamp"] else: events = capture_events() From 644aff8f7f6876c6c83ec584dced2ce4d519297d Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:04:41 +0200 Subject: [PATCH 12/72] fix merging issues --- .../integrations/boto3/_instrumentation.py | 281 +++++++++--------- tests/integrations/boto3/test_client.py | 8 +- 2 files changed, 146 insertions(+), 143 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 5a01d0eecd..9ba4bbfc91 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -39,8 +39,8 @@ def _start_client_span( if client.get_integration(IDENTIFIER) is None: return None - # Use unknown if `service_id_hyphenized` is unavailable so a span name can - # still be created, e.g. "aws.unknown.GetObject". + # use unknown if `service_id_hyphenized` so span name can still be created. + # e.g. "aws.unkown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" span_name = "aws.%s.%s" % (service_name, ctx.operation_name) @@ -77,6 +77,133 @@ def _start_client_span( return span +def _finish_span( + span: "Union[Span, StreamedSpan]", + error: "Optional[BaseException]" = None, +) -> None: + with capture_internal_exceptions(): + if not isinstance(span, StreamedSpan): + if error is not None: + span.set_status(SPANSTATUS.INTERNAL_ERROR) + span.finish() + return + + if error is None: + span.end() + else: + span.__exit__(type(error), error, error.__traceback__) + + +def _instrument_streaming_body( + span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]" +) -> bool: + if isinstance(span, NoOpStreamedSpan): + return False + + body = parsed.get("Body") + if not isinstance(body, StreamingBody): + return False + + streaming_span: "Union[Span, StreamedSpan]" + if isinstance(span, StreamedSpan): + streaming_span = sentry_sdk.traces.start_span( + name=span.name, + # `parent_span` is set explicitly to the boto span. + parent_span=span, + # avoid making the streaming span the current span on the scope since the application might + # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it + # could restore the parent span on the scope, breaking the parent-child relation of newly + # created spans; 2. newly created spans would be attached to the streaming span. + active=False, + attributes={ + "sentry.op": OP.HTTP_CLIENT_STREAM, + "sentry.origin": ORIGIN, + }, + ) + else: + streaming_span = span.start_child( + op=OP.HTTP_CLIENT_STREAM, + name=span.description, + origin=ORIGIN, + ) + + orig_read = body.read + orig_close = body.close + raw_stream = body._raw_stream # type: ignore[attr-defined] + orig_raw_close = raw_stream.close + finished = False + read_in_progress = False + + def finish_span(error: "Optional[BaseException]" = None) -> None: + nonlocal finished + if finished: + return + + finished = True + _finish_span(streaming_span, error) + if isinstance(span, StreamedSpan): + _finish_span(span, error) + + def content_length_reached() -> bool: + content_length = getattr(body, "_content_length", None) + amount_read = getattr(body, "_amount_read", None) + return ( + content_length is not None + and amount_read is not None + and amount_read >= int(content_length) + ) + + def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: + nonlocal read_in_progress + read_in_progress = True + try: + read_return_value = orig_read(*args, **kwargs) + with capture_internal_exceptions(): + amount_of_bytes_requested = args[0] if args else kwargs.get("amt") + if ( + amount_of_bytes_requested is None + or amount_of_bytes_requested < 0 + or (amount_of_bytes_requested > 0 and not read_return_value) + or content_length_reached() + ): + finish_span() + return read_return_value + except BaseException as error: + finish_span(error) + raise + finally: + read_in_progress = False + + def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: + try: + orig_close(*args, **kwargs) + finish_span() + except BaseException as error: + finish_span(error) + raise + + def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: + try: + orig_raw_close(*args, **kwargs) + if not read_in_progress: + finish_span() + except BaseException as error: + finish_span(error) + raise + + try: + # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing + # StreamingBody.close(), so both levels need to be instrumented. + raw_stream.close = sentry_raw_stream_close + body.read = sentry_streaming_body_read # type: ignore + body.close = sentry_streaming_body_close # type: ignore + except Exception: + finish_span() + raise + + return True + + def _set_request_attributes( span: "Union[Span, StreamedSpan]", request: "AWSRequest", @@ -137,8 +264,8 @@ def _sentry_request_created( request: "AWSRequest", operation_name: str, **kwargs: "Any" ) -> None: """ - Enrich a single `AWSRequest` attempt. Botocore creates a fresh - `AWSRequest` on every retry. + Enrich a single `AWSRequest` attempt. Botocore creates a + fresh `AWSRequest` on every retry. https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 """ from sentry_sdk.integrations.boto3 import Boto3Integration @@ -158,29 +285,29 @@ def _sentry_request_created( if span is None: return - # An ignored streamed span is not activated; avoid enriching its parent. + # an ignored streamed span is not activated; avoid enriching its parent. if isinstance(span, StreamedSpan) and ( span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN ): return _set_request_attributes(span, request) - # Each attempt has a fresh `request.context`; carry the active client span. + # each attempt has a fresh `request.context`; carry the active client span. request.context["_sentrysdk_span"] = span def _sentry_before_sign( request: "AWSRequest", signature_version: "Any", **kwargs: "Any" ) -> None: + from sentry_sdk.integrations.boto3 import Boto3Integration client = sentry_sdk.get_client() - if client.get_integration(IDENTIFIER) is None: + if client.get_integration(Boto3Integration) is None: return with capture_internal_exceptions(): - # Presigned requests are executed later by another caller. Adding propagation - # headers here would make those headers part of the signature, requiring the - # caller to reproduce the same values. + # presigned requests are executed later by another caller. Adding propagation + # headers here would make those headers part of the signature, requiring the caller to reproduce the same values. if isinstance(signature_version, str) and signature_version.endswith( ("-query", "-presign-post") ): @@ -201,19 +328,18 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: del request.headers[key] request.headers[key] = value - # Use the span associated with this botocore request. + # use span associated with this botocore request span = request.context.get("_sentrysdk_span") headers = sentry_sdk.get_current_scope().iter_trace_propagation_headers( span=span ) for header_name, header_value in headers: if header_name != BAGGAGE_HEADER_NAME: - # Normal headers (e.g. `sentry-trace`) are non-shared, so replace - # stale values. + # normal headers (e.g. `sentry-trace`) are non-shared, so replace stale values _replace_header(request, header_name, header_value) continue - # Preserve third-party baggage and replace stale `sentry-*` values. + # merge existing `baggage` values under single header existing_values = request.headers.get_all(BAGGAGE_HEADER_NAME, []) combined_baggage = { BAGGAGE_HEADER_NAME: ",".join(str(value) for value in existing_values) @@ -222,130 +348,3 @@ def _replace_header(request: "AWSRequest", key: str, value: str) -> None: _replace_header( request, BAGGAGE_HEADER_NAME, combined_baggage[BAGGAGE_HEADER_NAME] ) - - -def _finish_span( - span: "Union[Span, StreamedSpan]", - error: "Optional[BaseException]" = None, -) -> None: - with capture_internal_exceptions(): - if not isinstance(span, StreamedSpan): - if error is not None: - span.set_status(SPANSTATUS.INTERNAL_ERROR) - span.finish() - return - - if error is None: - span.end() - else: - span.__exit__(type(error), error, error.__traceback__) - - -def _instrument_streaming_body( - span: "Union[Span, StreamedSpan]", parsed: "Dict[str, Any]" -) -> bool: - if isinstance(span, NoOpStreamedSpan): - return False - - body = parsed.get("Body") - if not isinstance(body, StreamingBody): - return False - - streaming_span: "Union[Span, StreamedSpan]" - if isinstance(span, StreamedSpan): - streaming_span = sentry_sdk.traces.start_span( - name=span.name, - # `parent_span` is set explicitly to the boto span. - parent_span=span, - # avoid making the streaming span the current span on the scope since the application might - # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it - # could restore the parent span on the scope, breaking the parent-child relation of newly - # created spans; 2. newly created spans would be attached to the streaming span. - active=False, - attributes={ - "sentry.op": OP.HTTP_CLIENT_STREAM, - "sentry.origin": ORIGIN, - }, - ) - else: - streaming_span = span.start_child( - op=OP.HTTP_CLIENT_STREAM, - name=span.description, - origin=ORIGIN, - ) - - orig_read = body.read - orig_close = body.close - raw_stream = body._raw_stream # type: ignore[attr-defined] - orig_raw_close = raw_stream.close - finished = False - read_in_progress = False - - def finish_span(error: "Optional[BaseException]" = None) -> None: - nonlocal finished - if finished: - return - - finished = True - _finish_span(streaming_span, error) - if isinstance(span, StreamedSpan): - _finish_span(span, error) - - def content_length_reached() -> bool: - content_length = getattr(body, "_content_length", None) - amount_read = getattr(body, "_amount_read", None) - return ( - content_length is not None - and amount_read is not None - and amount_read >= int(content_length) - ) - - def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: - nonlocal read_in_progress - read_in_progress = True - try: - read_return_value = orig_read(*args, **kwargs) - with capture_internal_exceptions(): - amount_of_bytes_requested = args[0] if args else kwargs.get("amt") - if ( - amount_of_bytes_requested is None - or amount_of_bytes_requested < 0 - or (amount_of_bytes_requested > 0 and not read_return_value) - or content_length_reached() - ): - finish_span() - return read_return_value - except BaseException as error: - finish_span(error) - raise - finally: - read_in_progress = False - - def sentry_streaming_body_close(*args: "Any", **kwargs: "Any") -> None: - try: - orig_close(*args, **kwargs) - finish_span() - except BaseException as error: - finish_span(error) - raise - - def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: - try: - orig_raw_close(*args, **kwargs) - if not read_in_progress: - finish_span() - except BaseException as error: - finish_span(error) - raise - - try: - # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing - # StreamingBody.close(), so both levels need to be instrumented. - raw_stream.close = sentry_raw_stream_close - body.read = sentry_streaming_body_read # type: ignore - body.close = sentry_streaming_body_close # type: ignore - except Exception: - finish_span() - raise - - return True diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 0ad81d38bd..7e5b02d6b9 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -54,7 +54,7 @@ def test_public_api(): @pytest.mark.parametrize( "consume", - ["read_exact", "context"], + ["read", "read_exact", "context", "close"], ) def test_streaming_span_order_and_scope( sentry_init, @@ -84,7 +84,9 @@ def test_streaming_span_order_and_scope( body = client.get_object(Bucket="bucket", Key="key")["Body"] assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] - if consume == "read_exact": + if consume == "read": + assert body.read() == b"x" + elif consume == "read_exact": assert body.read(1) == b"x" elif consume == "context": if not hasattr(body, "__enter__"): @@ -92,6 +94,8 @@ def test_streaming_span_order_and_scope( pytest.skip("`StreamingBody` context manager is unavailable.") with body as raw_stream: assert raw_stream.read() == b"x" + else: + body.close() assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] From 608ee4ac8a9d31efd17ec21297cfed7131445dea Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:20:24 +0200 Subject: [PATCH 13/72] review changes --- sentry_sdk/integrations/boto3/_instrumentation.py | 14 +++++--------- sentry_sdk/integrations/stdlib.py | 4 +++- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 9ba4bbfc91..b5031502f7 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -42,7 +42,7 @@ def _start_client_span( # use unknown if `service_id_hyphenized` so span name can still be created. # e.g. "aws.unkown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" - span_name = "aws.%s.%s" % (service_name, ctx.operation_name) + span_name = f"aws.{service_name}.{ctx.operation_name}" if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: @@ -53,10 +53,7 @@ def _start_client_span( SPANDATA.SENTRY_ORIGIN: ORIGIN, } if ctx.service_id: - attributes[SPANDATA.RPC_METHOD] = "%s/%s" % ( - ctx.service_id, - ctx.operation_name, - ) + attributes[SPANDATA.RPC_METHOD] = f"{ctx.service_id}/{ctx.operation_name}" return sentry_sdk.traces.start_span( name=span_name, attributes=attributes, @@ -286,10 +283,9 @@ def _sentry_request_created( return # an ignored streamed span is not activated; avoid enriching its parent. - if isinstance(span, StreamedSpan) and ( - span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) != ORIGIN - ): - return + if isinstance(span, StreamedSpan): + if not (span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) == ORIGIN): + return _set_request_attributes(span, request) # each attempt has a fresh `request.context`; carry the active client span. diff --git a/sentry_sdk/integrations/stdlib.py b/sentry_sdk/integrations/stdlib.py index b9d0450ac2..35157fdd97 100644 --- a/sentry_sdk/integrations/stdlib.py +++ b/sentry_sdk/integrations/stdlib.py @@ -296,6 +296,7 @@ def putrequest( == getattr(client.get_integration("boto3"), "origin", None) and not getattr(parent_span, "active", True) ) + # fmt: off span = sentry_sdk.traces.start_span( name="%s %s" % ( @@ -309,8 +310,9 @@ def putrequest( }, # boto3 integration owns span's lifecycle; keep child inactive so it # can't restore boto3 span later on. - active=not is_inactive_boto3_span, + active = not is_inactive_boto3_span, ) + # fmt: on for key, value in url_attributes.items(): span.set_attribute(key, value) From ec8cef793ffb1fc6b1585fb9593cd1e816154cf0 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 13:54:19 +0200 Subject: [PATCH 14/72] add test for when `StreamingBody` instrumentation setup fails --- .../integrations/boto3/_instrumentation.py | 9 +-- tests/integrations/boto3/test_client.py | 71 +++++++++++++++---- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index b5031502f7..2b4bd1a423 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -124,10 +124,6 @@ def _instrument_streaming_body( origin=ORIGIN, ) - orig_read = body.read - orig_close = body.close - raw_stream = body._raw_stream # type: ignore[attr-defined] - orig_raw_close = raw_stream.close finished = False read_in_progress = False @@ -189,6 +185,11 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raise try: + orig_read = body.read + orig_close = body.close + raw_stream = body._raw_stream # type: ignore[attr-defined] + orig_raw_close = raw_stream.close + # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing # StreamingBody.close(), so both levels need to be instrumented. raw_stream.close = sentry_raw_stream_close diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 7e5b02d6b9..d17960306d 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -11,6 +11,8 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body +from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from tests.integrations.boto3.aws_mock import Body @@ -47,11 +49,6 @@ def log_message(self, *args): thread.join() -def test_public_api(): - assert Boto3Integration.__module__ == "sentry_sdk.integrations.boto3" - assert Boto3Integration.identifier == "boto3" - - @pytest.mark.parametrize( "consume", ["read", "read_exact", "context", "close"], @@ -112,7 +109,7 @@ def test_streaming_span_order_and_scope( span for span in spans if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT ] http_spans = [ @@ -141,6 +138,57 @@ def test_streaming_span_order_and_scope( assert stream_span["end_timestamp"] <= client_span["end_timestamp"] +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_body_instrumentation_setup_failure_finishes_stream_span( + sentry_init, + capture_items, + span_streaming, +): + sentry_init( + traces_sample_rate=1.0, + trace_lifecycle="stream" if span_streaming else "static", + integrations=[Boto3Integration()], + server_name="", + ) + + class _RawStreamLookupFailingBody(StreamingBody): + @property + def _raw_stream(self): + raise RuntimeError("raw stream lookup failed") + + @_raw_stream.setter + def _raw_stream(self, raw_stream): + self._raw_stream_value = raw_stream + + body = _RawStreamLookupFailingBody(Body(b"x"), "1") + + def invoke(): + if not span_streaming: + with sentry_sdk.start_span( + name="client", op=OP.HTTP_CLIENT, origin=ORIGIN + ) as span: + with pytest.raises(RuntimeError, match="raw stream lookup failed"): + _instrument_streaming_body(span, {"Body": body}) + return + + span = sentry_sdk.traces.start_span( # type: ignore[attr-defined] + name="client", + attributes={ + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, + SPANDATA.SENTRY_ORIGIN: ORIGIN, + }, + active=False, + ) + with pytest.raises(RuntimeError, match="raw stream lookup failed"): + _instrument_streaming_body(span, {"Body": body}) + + spans_by_op = _capture_boto3_spans_by_op(invoke, capture_items, span_streaming) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(stream_spans) == 1 + _assert_span_finished(stream_spans[0], span_streaming) + + def test_non_body_stream_does_not_delay_client_span(sentry_init, capture_items): sentry_init( traces_sample_rate=1.0, @@ -171,7 +219,7 @@ def respond(request, **kwargs): boto_spans = [ span for span in spans - if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == Boto3Integration.origin + if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN ] assert len(boto_spans) == 1 assert boto_spans[0]["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT @@ -234,19 +282,14 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami item.payload for item in items if item.type == "span" - and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) - == Boto3Integration.origin + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN ] else: with sentry_sdk.start_transaction(): invoke_client_method() transaction = next(item.payload for item in items if item.type == "transaction") - spans = [ - span - for span in transaction["spans"] - if span["origin"] == Boto3Integration.origin - ] + spans = [span for span in transaction["spans"] if span["origin"] == ORIGIN] spans_by_op = {} for span in spans: From 3a60fb9f2163e9c51242bfae4a5291e715907d69 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:20:31 +0200 Subject: [PATCH 15/72] fix(boto3): finish streaming spans before legacy boto spans --- sentry_sdk/integrations/boto3/_client.py | 28 +++--- .../integrations/boto3/_instrumentation.py | 3 +- tests/integrations/boto3/test_client.py | 88 ++++++++++++++----- 3 files changed, 84 insertions(+), 35 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 33d9899055..04f7db217d 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -26,19 +26,30 @@ @contextmanager -def _activate_client_span(span: "StreamedSpan") -> "Iterator[StreamedSpan]": +def _activate_client_span( + span: "Union[Span, StreamedSpan]", +) -> "Iterator[Union[Span, StreamedSpan]]": """Temporarily activate an inactive boto span without ending it.""" if isinstance(span, NoOpStreamedSpan): yield span return scope = sentry_sdk.get_current_scope() - previous_span = scope.streamed_span + if not isinstance(span, StreamedSpan): + previous_span = scope.span + scope.span = span + try: + yield span + finally: + scope.span = previous_span + return + + previous_streamed_span = scope.streamed_span scope.streamed_span = span try: yield span finally: - scope.streamed_span = previous_span + scope.streamed_span = previous_streamed_span def _patch_botocore_client() -> None: @@ -82,18 +93,13 @@ def sentry_patched_make_api_call( return orig_make_api_call(self, operation_name, api_params) # activate without finishing; a streaming response may outlive the call. - span_ctx = ( - _activate_client_span(span) if isinstance(span, StreamedSpan) else span - ) + span_ctx = _activate_client_span(span) try: with span_ctx: parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as error: - # finish `StreamedSpan` explicitly; static spans are finished by - # their context manager. - if isinstance(span, StreamedSpan): - _finish_span(span, error) + _finish_span(span, error) raise streaming_body_instrumented = False @@ -101,7 +107,7 @@ def sentry_patched_make_api_call( streaming_body_instrumented = _instrument_streaming_body(span, parsed) # `StreamingBody`s finish their span when consumed or closed. - if isinstance(span, StreamedSpan) and not streaming_body_instrumented: + if not streaming_body_instrumented: _finish_span(span) return parsed diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 2b4bd1a423..313b67096e 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -134,8 +134,7 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: finished = True _finish_span(streaming_span, error) - if isinstance(span, StreamedSpan): - _finish_span(span, error) + _finish_span(span, error) def content_length_reached() -> bool: content_length = getattr(body, "_content_length", None) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index d17960306d..5d687e16b1 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -14,6 +14,8 @@ from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body from sentry_sdk.integrations.boto3.consts import ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration +from sentry_sdk.traces import StreamedSpan +from sentry_sdk.tracing import Span from tests.integrations.boto3.aws_mock import Body session = boto3.Session( # type: ignore[attr-defined] @@ -53,15 +55,17 @@ def log_message(self, *args): "consume", ["read", "read_exact", "context", "close"], ) +@pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_span_order_and_scope( sentry_init, capture_items, streaming_s3_server, consume, + span_streaming, ): sentry_init( traces_sample_rate=1.0, - trace_lifecycle="stream", + trace_lifecycle="stream" if span_streaming else "static", default_integrations=False, integrations=[Boto3Integration(), StdlibIntegration()], server_name="", @@ -75,11 +79,31 @@ def test_streaming_span_order_and_scope( s3={"addressing_style": "path"}, ), ) - items = capture_items("span") + request_client_spans = [] - with sentry_sdk.traces.start_span(name="parent") as parent: # type: ignore[attr-defined] + def record_client_span(request, **kwargs): + request_client_spans.append(request.context["_sentrysdk_span"]) + + client.meta.events.register("request-created", record_client_span) + items = capture_items() + + parent_context = ( + sentry_sdk.traces.start_span(name="parent") # type: ignore[attr-defined] + if span_streaming + else sentry_sdk.start_transaction(name="parent") + ) + with parent_context as parent: body = client.get_object(Bucket="bucket", Key="key")["Body"] - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + assert len(request_client_spans) == 1 + request_client_span = request_client_spans[0] + if span_streaming: + assert isinstance(request_client_span, StreamedSpan) + assert request_client_span.end_timestamp is None + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + else: + assert isinstance(request_client_span, Span) + assert not isinstance(request_client_span, StreamedSpan) + assert request_client_span.timestamp is None if consume == "read": assert body.read() == b"x" @@ -94,34 +118,54 @@ def test_streaming_span_order_and_scope( else: body.close() - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + if span_streaming: + assert request_client_span.end_timestamp is not None + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] - probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] - assert probe._parent_span_id == parent.span_id - probe.end() + probe = sentry_sdk.traces.start_span(name="probe") # type: ignore[attr-defined] + assert probe._parent_span_id == parent.span_id + probe.end() - body.close() - assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + body.close() + assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] + else: + assert request_client_span.timestamp is not None sentry_sdk.flush() - spans = [item.payload for item in items] + if span_streaming: + spans = [item.payload for item in items] + else: + transaction = next(item.payload for item in items if item.type == "transaction") + spans = transaction["spans"] client_spans = [ span for span in spans - if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN - and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + if span.get("name", span.get("description")) == "aws.s3.GetObject" + and ( + span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN + and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT + if span_streaming + else span["origin"] == ORIGIN and span["op"] == OP.HTTP_CLIENT + ) ] http_spans = [ span for span in spans - if span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + if ( + span["attributes"].get(SPANDATA.SENTRY_ORIGIN) == "auto.http.stdlib.httplib" + if span_streaming + else span["origin"] == "auto.http.stdlib.httplib" + ) ] stream_spans = [ span for span in spans - if span["name"] == "aws.s3.GetObject" - and span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + if span.get("name", span.get("description")) == "aws.s3.GetObject" + and ( + span["attributes"].get(SPANDATA.SENTRY_OP) == OP.HTTP_CLIENT_STREAM + if span_streaming + else span["op"] == OP.HTTP_CLIENT_STREAM + ) ] assert len(client_spans) == 1 assert len(http_spans) == 1 @@ -132,10 +176,12 @@ def test_streaming_span_order_and_scope( assert http_span["parent_span_id"] == client_span["span_id"] assert stream_span["parent_span_id"] == client_span["span_id"] + assert client_span["span_id"] == request_client_span.span_id + end_timestamp = "end_timestamp" if span_streaming else "timestamp" assert client_span["start_timestamp"] <= http_span["start_timestamp"] assert http_span["start_timestamp"] <= stream_span["start_timestamp"] - assert http_span["end_timestamp"] <= stream_span["end_timestamp"] - assert stream_span["end_timestamp"] <= client_span["end_timestamp"] + assert http_span[end_timestamp] <= stream_span[end_timestamp] + assert stream_span[end_timestamp] <= client_span[end_timestamp] @pytest.mark.parametrize("span_streaming", [True, False]) @@ -437,7 +483,5 @@ def invoke_client_method_and_read_body(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) - assert len(client_spans) == 1 - if span_streaming: - _assert_one_failed_span(client_spans, span_streaming=True) + _assert_one_failed_span(client_spans, span_streaming) _assert_one_failed_span(stream_spans, span_streaming) From 9c2f329d48815193c0a25aae25c9096d86078033 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:25:18 +0200 Subject: [PATCH 16/72] add permalink --- sentry_sdk/integrations/boto3/_instrumentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 313b67096e..9d751fb70e 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -263,7 +263,7 @@ def _sentry_request_created( """ Enrich a single `AWSRequest` attempt. Botocore creates a fresh `AWSRequest` on every retry. - https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L178-L202 + https://github.com/boto/botocore/blob/f9195c79ea2bf46350dd320d2a0bf3db7da0b460/botocore/endpoint.py#L178-L202 """ from sentry_sdk.integrations.boto3 import Boto3Integration From f9755ca8bbf664918dee22f58bcca871ed2b8d7b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 14:40:09 +0200 Subject: [PATCH 17/72] add some more comments --- sentry_sdk/integrations/boto3/_client.py | 14 +++++++++++++- sentry_sdk/integrations/boto3/_instrumentation.py | 13 ++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 04f7db217d..2163233e1d 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -29,7 +29,19 @@ def _activate_client_span( span: "Union[Span, StreamedSpan]", ) -> "Iterator[Union[Span, StreamedSpan]]": - """Temporarily activate an inactive boto span without ending it.""" + """ + Activate the boto span temporarily during `_make_api_call()` without ending it. + + Botocore returns a `StreamingBody` before its bytes are consumed. Using the + context manager would finish it as soon as `_make_api_call()` returns, so + restore the caller's span here and let the `StreamingBody` wrapper finish + the boto span when body is consumed/closed. + + faulty: desired: + boto3 [_make_api_call] boto3 [_make_api_call------] + http [request] http [request] + stream [read] stream [read] + """ if isinstance(span, NoOpStreamedSpan): yield span return diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 9d751fb70e..17d426d74d 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -105,12 +105,11 @@ def _instrument_streaming_body( if isinstance(span, StreamedSpan): streaming_span = sentry_sdk.traces.start_span( name=span.name, - # `parent_span` is set explicitly to the boto span. + # keep stream span under the boto span after `_make_api_call()` returns. parent_span=span, - # avoid making the streaming span the current span on the scope since the application might - # keep `StreamingBody` open before reading it. Otherwise: 1. when the streamingspan ends it - # could restore the parent span on the scope, breaking the parent-child relation of newly - # created spans; 2. newly created spans would be attached to the streaming span. + # the body may outlive the api call, so keep it inactive. Otherwise it + # 1. could restore the already-finished boto span when it ends; 2. make + # unrelated new spans attach to the stream span since it's the current span. active=False, attributes={ "sentry.op": OP.HTTP_CLIENT_STREAM, @@ -133,6 +132,7 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: return finished = True + # finish stream span before boto span, and only once across read/close. _finish_span(streaming_span, error) _finish_span(span, error) @@ -152,6 +152,7 @@ def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: read_return_value = orig_read(*args, **kwargs) with capture_internal_exceptions(): amount_of_bytes_requested = args[0] if args else kwargs.get("amt") + # detect read-to-end, eof, or the known content length being consumed. if ( amount_of_bytes_requested is None or amount_of_bytes_requested < 0 @@ -189,8 +190,6 @@ def sentry_raw_stream_close(*args: "Any", **kwargs: "Any") -> None: raw_stream = body._raw_stream # type: ignore[attr-defined] orig_raw_close = raw_stream.close - # StreamingBody.__exit__ closes `_raw_stream` directly, bypassing - # StreamingBody.close(), so both levels need to be instrumented. raw_stream.close = sentry_raw_stream_close body.read = sentry_streaming_body_read # type: ignore body.close = sentry_streaming_body_close # type: ignore From a9e87f5c8b9ae2f3f8201473c52ea37125e5d409 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:09:37 +0200 Subject: [PATCH 18/72] ref(boto3): use permalink --- sentry_sdk/integrations/boto3/_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 2163233e1d..8b75cbfae4 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -84,7 +84,7 @@ def sentry_patched_make_api_call( Track a single API call, including retries, serialization, and endpoint resolution. For streaming responses, keep the span open until the response body is consumed or closed. - https://github.com/boto/botocore/blob/develop/botocore/client.py + https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/client.py https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span """ client = sentry_sdk.get_client() From 2647da17145e217ae3edb46edcada1f8af945f57 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:12:00 +0200 Subject: [PATCH 19/72] ref(boto3): fix grammar in comments --- sentry_sdk/integrations/boto3/_instrumentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 17d426d74d..9aa5a52140 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -39,8 +39,8 @@ def _start_client_span( if client.get_integration(IDENTIFIER) is None: return None - # use unknown if `service_id_hyphenized` so span name can still be created. - # e.g. "aws.unkown.GetObject" + # use unknown if `service_id_hyphenized` is not set so span name can still be created. + # e.g. "aws.unknown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" span_name = f"aws.{service_name}.{ctx.operation_name}" From 8518c62f062c0cd053abe824b5cc5920f57deafe Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 17:22:38 +0200 Subject: [PATCH 20/72] feat(boto3): improve boto3 integration --- sentry_sdk/consts.py | 18 ++ sentry_sdk/integrations/boto3/_client.py | 2 +- sentry_sdk/integrations/boto3/_context.py | 22 ++- .../integrations/boto3/_instrumentation.py | 97 +++++++++-- tests/integrations/boto3/test_client.py | 164 +++++++++++++++++- tests/integrations/boto3/test_s3.py | 6 +- 6 files changed, 288 insertions(+), 21 deletions(-) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index b1470dabf6..651d709061 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -414,6 +414,12 @@ class SPANDATA: Example: "10.1.2.80" """ + CLOUD_REGION = "cloud.region" + """ + The geographical region the resource is running. + Example: "us-east-1" + """ + CODE_FILEPATH = "code.filepath" """ .. deprecated:: @@ -977,12 +983,24 @@ class SPANDATA: Example: "com.example.ExampleService/exampleMethod" """ + RPC_SERVICE = "rpc.service" + """ + The full (logical) name of the service being called, including its package name, if applicable. + Example: "myService.BestService" + """ + RPC_RESPONSE_STATUS_CODE = "rpc.response.status_code" """ Status code of the RPC returned by the RPC server or generated by the client. Example: "DEADLINE_EXCEEDED" """ + RPC_SYSTEM_NAME = "rpc.system.name" + """ + A string identifying the remoting system. + Example: "aws-api" + """ + SERVER_ADDRESS = "server.address" """ Name of the database host. diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 8b75cbfae4..8aeb4c2855 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -91,7 +91,7 @@ def sentry_patched_make_api_call( if client.get_integration(Boto3Integration) is None: return orig_make_api_call(self, operation_name, api_params) - ctx = AwsCallContext(operation_name) + ctx = AwsCallContext(operation_name, api_params) # add optional metadata to context. with capture_internal_exceptions(): diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py index 38f7e0d76a..fa250be5b6 100644 --- a/sentry_sdk/integrations/boto3/_context.py +++ b/sentry_sdk/integrations/boto3/_context.py @@ -4,7 +4,7 @@ from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any, Optional + from typing import Any, Optional, Dict try: from botocore.client import BaseClient @@ -14,15 +14,27 @@ class AwsCallContext: __slots__ = ( + "service_name", "service_id", "service_id_hyphenized", "operation_name", + "region_name", + "endpoint_url", + "params", ) - def __init__(self, operation_name: str) -> None: + def __init__(self, operation_name: str, params: "Any") -> None: self.operation_name: str = operation_name + self.params: "Dict[str, Any]" = {} + self.service_name: "Optional[str]" = None self.service_id: "Optional[str]" = None self.service_id_hyphenized: "Optional[str]" = None + self.region_name: "Optional[str]" = None + self.endpoint_url: "Optional[str]" = None + + if isinstance(params, dict): + with capture_internal_exceptions(): + self.params = dict(params) def add_metadata(self, client: "BaseClient") -> None: def _get_attr(obj: "Any", name: str) -> "Any": @@ -35,6 +47,9 @@ def _get_attr(obj: "Any", name: str) -> "Any": client_meta = _get_attr(client, "meta") service_model = _get_attr(client_meta, "service_model") + # botocore's internal identifier, e.g. `apigateway`. + self.service_name = _get_attr(service_model, "service_name") + # modeled AWS service identity used in span names, e.g. `API Gateway`. service_id = _get_attr(service_model, "service_id") if service_id is not None: @@ -42,3 +57,6 @@ def _get_attr(obj: "Any", name: str) -> "Any": self.service_id = str(service_id) with capture_internal_exceptions(): self.service_id_hyphenized = service_id.hyphenize() + + self.region_name = _get_attr(client_meta, "region_name") + self.endpoint_url = _get_attr(client_meta, "endpoint_url") diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 9aa5a52140..877f8a509b 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -1,4 +1,5 @@ from typing import TYPE_CHECKING +from urllib.parse import urlsplit import sentry_sdk from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS @@ -31,6 +32,63 @@ raise DidNotEnable("botocore not installed") +_AWS_RPC_SYSTEM_NAME = "aws-api" + + +def _set_span_attributes( + span: "Union[Span, StreamedSpan]", attributes: "Attributes" +) -> None: + if isinstance(span, StreamedSpan): + span.set_attributes(attributes) + return + + for key, value in attributes.items(): + span.set_data(key, value) + + +def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes": + if not endpoint_url: + return {} + + default_ports = { + "http": 80, + "https": 443, + } + + try: + parsed_url = urlsplit(endpoint_url) + if parsed_url.scheme not in default_ports or not parsed_url.hostname: + return {} + + # `server.port` is only defined together with `server.address`. + # Infer the effective port when the configured HTTP(S) endpoint omits it. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ + return { + SPANDATA.SERVER_ADDRESS: parsed_url.hostname, + SPANDATA.SERVER_PORT: parsed_url.port or default_ports[parsed_url.scheme], + } + + except (TypeError, UnicodeError, ValueError): + # Invalid client metadata must not prevent the AWS call from running. + return {} + + +def _get_client_attributes( + ctx: "AwsCallContext", +) -> "Attributes": + attributes: "Attributes" = {} + + # `rpc.service` is deprecated in OTel, but js still uses it. + if ctx.service_id: + attributes[SPANDATA.RPC_SERVICE] = ctx.service_id + + if ctx.region_name: + attributes[SPANDATA.CLOUD_REGION] = ctx.region_name + + attributes.update(_get_server_attributes(ctx.endpoint_url)) + return attributes + + def _start_client_span( ctx: "AwsCallContext", ) -> "Optional[Union[Span, StreamedSpan]]": @@ -43,17 +101,27 @@ def _start_client_span( # e.g. "aws.unknown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" span_name = f"aws.{service_name}.{ctx.operation_name}" + attributes: "Attributes" = { + SPANDATA.RPC_METHOD: ctx.operation_name, + SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME, + } + with capture_internal_exceptions(): + attributes.update(_get_client_attributes(ctx)) + span_op = OP.HTTP_CLIENT + span_origin = ORIGIN if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: return None - attributes: "Attributes" = { - SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, - SPANDATA.SENTRY_ORIGIN: ORIGIN, - } - if ctx.service_id: - attributes[SPANDATA.RPC_METHOD] = f"{ctx.service_id}/{ctx.operation_name}" + # `start_span()` evaluates `ignore_spans` against the initial attributes. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span + attributes.update( + { + SPANDATA.SENTRY_OP: span_op, + SPANDATA.SENTRY_ORIGIN: span_origin, + } + ) return sentry_sdk.traces.start_span( name=span_name, attributes=attributes, @@ -64,9 +132,11 @@ def _start_client_span( span = sentry_sdk.start_span( name=span_name, - op=OP.HTTP_CLIENT, - origin=ORIGIN, + op=span_op, + origin=span_origin, ) + with capture_internal_exceptions(): + _set_span_attributes(span, attributes) with capture_internal_exceptions(): if ctx.service_id_hyphenized: span.set_tag("aws.service_id", ctx.service_id_hyphenized) @@ -112,8 +182,8 @@ def _instrument_streaming_body( # unrelated new spans attach to the stream span since it's the current span. active=False, attributes={ - "sentry.op": OP.HTTP_CLIENT_STREAM, - "sentry.origin": ORIGIN, + SPANDATA.SENTRY_OP: OP.HTTP_CLIENT_STREAM, + SPANDATA.SENTRY_ORIGIN: ORIGIN, }, ) else: @@ -264,10 +334,9 @@ def _sentry_request_created( fresh `AWSRequest` on every retry. https://github.com/boto/botocore/blob/f9195c79ea2bf46350dd320d2a0bf3db7da0b460/botocore/endpoint.py#L178-L202 """ - from sentry_sdk.integrations.boto3 import Boto3Integration client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: + if client.get_integration("boto3") is None: return with capture_internal_exceptions(): @@ -294,10 +363,8 @@ def _sentry_request_created( def _sentry_before_sign( request: "AWSRequest", signature_version: "Any", **kwargs: "Any" ) -> None: - from sentry_sdk.integrations.boto3 import Boto3Integration - client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: + if client.get_integration("boto3") is None: return with capture_internal_exceptions(): diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 5d687e16b1..babd1a1df9 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -7,6 +7,7 @@ from botocore.config import Config from botocore.exceptions import ClientError, EndpointConnectionError from botocore.response import StreamingBody +from botocore.stub import Stubber import sentry_sdk from sentry_sdk.consts import OP, SPANDATA @@ -357,6 +358,165 @@ def _assert_one_failed_span(spans, span_streaming): _assert_span_finished(spans[0], span_streaming) +def _capture_stubbed_client_span( + client, + method_name, + api_params, + capture_items, + span_streaming, +): + with Stubber(client) as stubber: + stubber.add_response(method_name, {}, api_params) + spans_by_op = _capture_boto3_spans_by_op( + lambda: getattr(client, method_name)(**api_params), + capture_items, + span_streaming, + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert len(client_spans) == 1 + return client_spans[0] + + +def _span_attributes(span, span_streaming): + return span["attributes"] if span_streaming else span["data"] + + +@pytest.mark.parametrize( + ( + "service_name", + "method_name", + "api_params", + "span_name", + "rpc_service", + "rpc_method", + "endpoint_url", + "server_address", + "server_port", + ), + [ + ( + "s3", + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + "aws.s3.HeadObject", + "S3", + "HeadObject", + "http://localhost:4566", + "localhost", + 4566, + ), + ( + "events", + "list_event_buses", + {}, + "aws.eventbridge.ListEventBuses", + "EventBridge", + "ListEventBuses", + None, + "events.eu-north-1.amazonaws.com", + 443, + ), + ], +) +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_has_common_attributes( + capture_items, + client_factory, + span_streaming, + service_name, + method_name, + api_params, + span_name, + rpc_service, + rpc_method, + endpoint_url, + server_address, + server_port, +): + client = client_factory(service_name=service_name, endpoint_url=endpoint_url) + span = _capture_stubbed_client_span( + client, + method_name, + api_params, + capture_items, + span_streaming, + ) + attributes = _span_attributes(span, span_streaming) + + assert span["name" if span_streaming else "description"] == span_name + assert attributes[SPANDATA.RPC_SERVICE] == rpc_service + assert attributes[SPANDATA.RPC_METHOD] == rpc_method + assert attributes[SPANDATA.RPC_SYSTEM_NAME] == "aws-api" + assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" + assert attributes[SPANDATA.SERVER_ADDRESS] == server_address + assert attributes[SPANDATA.SERVER_PORT] == server_port + + +def test_client_call_attributes_are_available_at_span_creation( + sentry_init, capture_items +): + # attribute-based filtering happens during span creation, at the same boundary + # where creation attributes are made available for sampling decisions. + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream", + ignore_spans=[ + { + "attributes": { + SPANDATA.RPC_METHOD: "HeadObject", + SPANDATA.RPC_SERVICE: "S3", + SPANDATA.RPC_SYSTEM_NAME: "aws-api", + SPANDATA.SERVER_ADDRESS: "s3.eu-north-1.amazonaws.com", + SPANDATA.SERVER_PORT: 443, + } + } + ], + ) + client = session.client("s3") + items = capture_items("span") + + with Stubber(client) as stubber: + stubber.add_response("head_object", {}, {"Bucket": "bucket", "Key": "foo"}) + with sentry_sdk.traces.start_span(name="parent"): + client.head_object(Bucket="bucket", Key="foo") + + sentry_sdk.flush() + client_spans = [ + item.payload + for item in items + if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) + == Boto3Integration.origin + ] + assert client_spans == [] + + +def test_client_call_omits_missing_region( + sentry_init, + capture_items, + monkeypatch, +): + sentry_init( + traces_sample_rate=1.0, + integrations=[Boto3Integration()], + trace_lifecycle="stream", + server_name="", + ) + client = session.client("s3") + monkeypatch.setattr(type(client.meta), "region_name", property(lambda _: None)) + + span = _capture_stubbed_client_span( + client, + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + capture_items, + span_streaming=True, + ) + + assert SPANDATA.CLOUD_REGION not in span["attributes"] + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_retry_attempts_share_one_client_span( capture_items, @@ -483,5 +643,7 @@ def invoke_client_method_and_read_body(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) - _assert_one_failed_span(client_spans, span_streaming) + assert len(client_spans) == 1 + if span_streaming: + _assert_one_failed_span(client_spans, span_streaming=True) _assert_one_failed_span(stream_spans, span_streaming) diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index dcd38dab9b..8c8b24ba13 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -126,7 +126,8 @@ def test_streaming( expected_attrs = { "http.request.method": "GET", - "rpc.method": "S3/GetObject", + "rpc.method": "GetObject", + "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", "sentry.origin": "auto.http.boto3", @@ -287,7 +288,8 @@ def test_omit_url_data_if_parsing_fails( assert spans[0]["attributes"] == ApproxDict( { "http.request.method": "GET", - "rpc.method": "S3/ListObjects", + "rpc.method": "ListObjects", + "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", "sentry.origin": "auto.http.boto3", From 7bf92ddef25bf7b4ae253b9aaa645738bb70ac63 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 17:42:26 +0200 Subject: [PATCH 21/72] lint --- sentry_sdk/integrations/boto3/_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py index fa250be5b6..83867d20b4 100644 --- a/sentry_sdk/integrations/boto3/_context.py +++ b/sentry_sdk/integrations/boto3/_context.py @@ -4,7 +4,7 @@ from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any, Optional, Dict + from typing import Any, Dict, Optional try: from botocore.client import BaseClient From 606c6301861368837d0f09764b8b6be862bdc248 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 10:52:21 +0200 Subject: [PATCH 22/72] fix(boto3): overwrite server attributes when request URL is resolved --- sentry_sdk/integrations/boto3/_instrumentation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 877f8a509b..13398c0a64 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -69,7 +69,7 @@ def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes": } except (TypeError, UnicodeError, ValueError): - # Invalid client metadata must not prevent the AWS call from running. + # invalid client metadata must not prevent the AWS call from running. return {} @@ -281,6 +281,9 @@ def _set_request_attributes( with capture_internal_exceptions(): parsed_url = parse_url(request.url, sanitize=False) + # overwrite server attributes when actual request URL is resolved. + _set_span_attributes(span, _get_server_attributes(request.url)) + if isinstance(span, StreamedSpan): span.set_attributes(get_url_attributes(client, parsed_url)) if request.method is not None: From 74083ab16d5291e481d9d32ac1b4e91034f332f8 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:24:07 +0200 Subject: [PATCH 23/72] ref(boto3): move `aws-api` to `consts.py` --- sentry_sdk/integrations/boto3/_instrumentation.py | 7 ++----- sentry_sdk/integrations/boto3/consts.py | 4 ++++ tests/integrations/boto3/test_client.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 13398c0a64..0ad25cc2ad 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -4,7 +4,7 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.integrations.boto3.consts import IDENTIFIER, ORIGIN +from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, IDENTIFIER, ORIGIN from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( @@ -32,9 +32,6 @@ raise DidNotEnable("botocore not installed") -_AWS_RPC_SYSTEM_NAME = "aws-api" - - def _set_span_attributes( span: "Union[Span, StreamedSpan]", attributes: "Attributes" ) -> None: @@ -103,7 +100,7 @@ def _start_client_span( span_name = f"aws.{service_name}.{ctx.operation_name}" attributes: "Attributes" = { SPANDATA.RPC_METHOD: ctx.operation_name, - SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME, + SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, } with capture_internal_exceptions(): attributes.update(_get_client_attributes(ctx)) diff --git a/sentry_sdk/integrations/boto3/consts.py b/sentry_sdk/integrations/boto3/consts.py index 67d0d18239..e88f9d4c66 100644 --- a/sentry_sdk/integrations/boto3/consts.py +++ b/sentry_sdk/integrations/boto3/consts.py @@ -1,2 +1,6 @@ IDENTIFIER = "boto3" ORIGIN = f"auto.http.{IDENTIFIER}" + +# value is used by `rpc.system` (deprecated in OTel, but we still support it for now) and `rpc.system.name` +# https://opentelemetry.io/docs/specs/semconv/cloud-providers/aws-sdk/#aws-sdk-spans +AWS_RPC_SYSTEM_NAME = "aws-api" diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index babd1a1df9..9bdf1f9b17 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -13,7 +13,7 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body -from sentry_sdk.integrations.boto3.consts import ORIGIN +from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span @@ -447,7 +447,7 @@ def test_client_call_has_common_attributes( assert span["name" if span_streaming else "description"] == span_name assert attributes[SPANDATA.RPC_SERVICE] == rpc_service assert attributes[SPANDATA.RPC_METHOD] == rpc_method - assert attributes[SPANDATA.RPC_SYSTEM_NAME] == "aws-api" + assert attributes[SPANDATA.RPC_SYSTEM_NAME] == AWS_RPC_SYSTEM_NAME assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" assert attributes[SPANDATA.SERVER_ADDRESS] == server_address assert attributes[SPANDATA.SERVER_PORT] == server_port @@ -467,7 +467,7 @@ def test_client_call_attributes_are_available_at_span_creation( "attributes": { SPANDATA.RPC_METHOD: "HeadObject", SPANDATA.RPC_SERVICE: "S3", - SPANDATA.RPC_SYSTEM_NAME: "aws-api", + SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, SPANDATA.SERVER_ADDRESS: "s3.eu-north-1.amazonaws.com", SPANDATA.SERVER_PORT: 443, } From 7e5ba8690230709127300256ebc82940485316b2 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:25:50 +0200 Subject: [PATCH 24/72] tests(boto3): move `span_streaming` inside method call --- tests/integrations/boto3/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 9bdf1f9b17..344fb6de5d 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -644,6 +644,5 @@ def invoke_client_method_and_read_body(): stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) assert len(client_spans) == 1 - if span_streaming: - _assert_one_failed_span(client_spans, span_streaming=True) + _assert_one_failed_span(client_spans, span_streaming) _assert_one_failed_span(stream_spans, span_streaming) From 6868ffd0b388ea9e31a2600e6701436a453f12af Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:27:37 +0200 Subject: [PATCH 25/72] ref(boto3): move `DEFAULT_PORTS` to `consts.py` --- .../integrations/boto3/_instrumentation.py | 16 ++++++++-------- sentry_sdk/integrations/boto3/consts.py | 6 ++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 0ad25cc2ad..315fecb803 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -4,7 +4,12 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS from sentry_sdk.integrations import DidNotEnable -from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, IDENTIFIER, ORIGIN +from sentry_sdk.integrations.boto3.consts import ( + AWS_RPC_SYSTEM_NAME, + DEFAULT_PORTS, + IDENTIFIER, + ORIGIN, +) from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan from sentry_sdk.tracing import BAGGAGE_HEADER_NAME, Span from sentry_sdk.tracing_utils import ( @@ -47,14 +52,9 @@ def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes": if not endpoint_url: return {} - default_ports = { - "http": 80, - "https": 443, - } - try: parsed_url = urlsplit(endpoint_url) - if parsed_url.scheme not in default_ports or not parsed_url.hostname: + if parsed_url.scheme not in DEFAULT_PORTS or not parsed_url.hostname: return {} # `server.port` is only defined together with `server.address`. @@ -62,7 +62,7 @@ def _get_server_attributes(endpoint_url: "Optional[str]") -> "Attributes": # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ return { SPANDATA.SERVER_ADDRESS: parsed_url.hostname, - SPANDATA.SERVER_PORT: parsed_url.port or default_ports[parsed_url.scheme], + SPANDATA.SERVER_PORT: parsed_url.port or DEFAULT_PORTS[parsed_url.scheme], } except (TypeError, UnicodeError, ValueError): diff --git a/sentry_sdk/integrations/boto3/consts.py b/sentry_sdk/integrations/boto3/consts.py index e88f9d4c66..7d0d441257 100644 --- a/sentry_sdk/integrations/boto3/consts.py +++ b/sentry_sdk/integrations/boto3/consts.py @@ -4,3 +4,9 @@ # value is used by `rpc.system` (deprecated in OTel, but we still support it for now) and `rpc.system.name` # https://opentelemetry.io/docs/specs/semconv/cloud-providers/aws-sdk/#aws-sdk-spans AWS_RPC_SYSTEM_NAME = "aws-api" + +# default ports for HTTP and HTTPS +DEFAULT_PORTS = { + "http": 80, + "https": 443, +} From d32a43f2125a6d95a0024a6a9a6fe8ca4ea7e9c0 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 15:02:07 +0200 Subject: [PATCH 26/72] ref(boto3): simplify client span lifecycle and header handling --- sentry_sdk/integrations/boto3/_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 8aeb4c2855..63e6479c48 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -115,6 +115,9 @@ def sentry_patched_make_api_call( raise streaming_body_instrumented = False + with capture_internal_exceptions(): + streaming_body_instrumented = _instrument_streaming_body(span, parsed) + streaming_body_instrumented = False with capture_internal_exceptions(): streaming_body_instrumented = _instrument_streaming_body(span, parsed) From 822438b0d9ae054e4cda1e936735164c5962408e Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 14 Sep 2026 14:31:40 +0200 Subject: [PATCH 27/72] feat(boto3): add attributes to `consts.py` --- sentry_sdk/consts.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 651d709061..90e89afc95 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -390,6 +390,18 @@ class SPANDATA: Example: ["Token limit exceeded"] """ + AWS_EXTENDED_REQUEST_ID = "aws.extended_request_id" + """ + The AWS extended request ID as returned in the response headers. + Example: "wzHcyEWfmOGDIE5QOhTAqFDoDWP3y8IUvpNINCwL9N4TEHbUw0/gZJ+VZTmCNCWR7fezEN3eCiQ=" + """ + + AWS_REQUEST_ID = "aws.request_id" + """ + The AWS request ID as returned in the response headers. + Example: "79b9da39-b7ae-508a-a6bc-864b2829c622" + """ + CACHE_HIT = "cache.hit" """ A boolean indicating whether the requested data was found in the cache. @@ -547,6 +559,12 @@ class SPANDATA: Example: my_user """ + ERROR_TYPE = "error.type" + """ + Describes a class of error the operation ended with. + Example: "timeout" + """ + GEN_AI_AGENT_NAME = "gen_ai.agent.name" """ The name of the agent being used. @@ -886,6 +904,12 @@ class SPANDATA: Example: GET """ + HTTP_REQUEST_RESEND_COUNT = "http.request.resend_count" + """ + The ordinal number of request resending attempt (for any reason, including redirects). + Example: 2 + """ + HTTP_ROUTE = "http.route" """ The matched route, that is, the path template used to match the request. From 46d24b2b3dc463277d4cf87f1cca7ff7dcf9ca1f Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 15 Sep 2026 14:56:11 +0200 Subject: [PATCH 28/72] merge changes --- sentry_sdk/integrations/boto3/_client.py | 17 +++- .../integrations/boto3/_instrumentation.py | 98 +++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 63e6479c48..ba390c9450 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -6,9 +6,12 @@ from sentry_sdk.integrations.boto3._context import AwsCallContext from sentry_sdk.integrations.boto3._instrumentation import ( _finish_span, + _get_error_attributes, + _get_response_attributes, _instrument_streaming_body, _sentry_before_sign, _sentry_request_created, + _set_span_attributes, _start_client_span, ) from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan @@ -109,7 +112,19 @@ def sentry_patched_make_api_call( try: with span_ctx: - parsed = orig_make_api_call(self, operation_name, api_params) + try: + parsed = orig_make_api_call(self, operation_name, api_params) + except BaseException as error: + if span is not None: + with capture_internal_exceptions(): + _set_span_attributes(span, _get_error_attributes(error)) + raise + else: + if span is not None: + with capture_internal_exceptions(): + _set_span_attributes( + span, _get_response_attributes(parsed) + ) except BaseException as error: _finish_span(span, error) raise diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 315fecb803..b0e2056676 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -32,6 +32,7 @@ try: from botocore.awsrequest import AWSRequest + from botocore.exceptions import ClientError from botocore.response import StreamingBody except ImportError: raise DidNotEnable("botocore not installed") @@ -40,6 +41,7 @@ def _set_span_attributes( span: "Union[Span, StreamedSpan]", attributes: "Attributes" ) -> None: + """Will be removed in the major.""" if isinstance(span, StreamedSpan): span.set_attributes(attributes) return @@ -86,6 +88,95 @@ def _get_client_attributes( return attributes +def _get_response_attributes(response: "Any") -> "Attributes": + if not isinstance(response, dict): + return {} + + metadata = response.get("ResponseMetadata") + if not isinstance(metadata, dict): + return {} + attributes: "Attributes" = {} + + # botocore injects HTTP status into `ResponseMetadata` after parsing. + # https://github.com/boto/botocore/blob/develop/botocore/parsers.py#L273-L284 + status_code = metadata.get("HTTPStatusCode") + if isinstance(status_code, int) and 100 <= status_code <= 599: + attributes[SPANDATA.HTTP_STATUS_CODE] = status_code + + retry_attempts = metadata.get("RetryAttempts") + # botocore represents retries as `attempts - 1`; OTel suggests "if and only if", so skip zero. + # https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L221-L229 + # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span + if ( + isinstance(retry_attempts, int) + # avoid emitting `resend_count=True`. + and not isinstance(retry_attempts, bool) + and retry_attempts > 0 + ): + attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] = retry_attempts + + headers = metadata.get("HTTPHeaders") + if not isinstance(headers, dict): + headers = {} + + request_id = metadata.get("RequestId") + if not isinstance(request_id, str) or not request_id: + request_id = next( + ( + value + for value in ( + headers.get("x-amzn-requestid"), + headers.get("x-amzn-request-id"), + headers.get("x-amz-request-id"), + ) + if isinstance(value, str) and value + ), + None, + ) + if isinstance(request_id, str) and request_id: + attributes[SPANDATA.AWS_REQUEST_ID] = request_id + + # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. + # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html + extended_request_id = metadata.get("HostId") + if not isinstance(extended_request_id, str) or not extended_request_id: + extended_request_id = headers.get("x-amz-id-2") + if isinstance(extended_request_id, str) and extended_request_id: + attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id + + return attributes + + +def _get_error_type(exception: "BaseException") -> str: + if isinstance(exception, ClientError): + # `ClientError` wraps AWS service errors; `Error.Code` identifies the + # actual service error, e.g. `AccessDeniedException`. + # https://docs.aws.amazon.com/boto3/latest/guide/error-handling.html + error = exception.response.get("Error") + if isinstance(error, dict): + error_code = error.get("Code") + if isinstance(error_code, str) and error_code: + return error_code + + # failures before a service response have no AWS error code. + # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ + exception_type = type(exception) + exception_name = exception_type.__qualname__ + exception_module = exception_type.__module__ + if exception_module not in ("builtins", "__builtins__"): + return "%s.%s" % (exception_module, exception_name) + return exception_name + + +def _get_error_attributes(exception: "BaseException") -> "Attributes": + attributes: "Attributes" = {} + if isinstance(exception, ClientError): + attributes.update(_get_response_attributes(exception.response)) + + attributes[SPANDATA.ERROR_TYPE] = _get_error_type(exception) + return attributes + + def _start_client_span( ctx: "AwsCallContext", ) -> "Optional[Union[Span, StreamedSpan]]": @@ -200,6 +291,13 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: finished = True # finish stream span before boto span, and only once across read/close. + if error is not None: + with capture_internal_exceptions(): + attributes = _get_error_attributes(error) + _set_span_attributes(streaming_span, attributes) + if isinstance(span, StreamedSpan): + _set_span_attributes(span, attributes) + _finish_span(streaming_span, error) _finish_span(span, error) From 30d1e3f1faced9bed1bfe1466ec140807c51abb8 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Thu, 17 Sep 2026 17:54:09 +0200 Subject: [PATCH 29/72] add tests --- tests/integrations/boto3/test_client.py | 342 +++++++++++++++++++++++- 1 file changed, 340 insertions(+), 2 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 344fb6de5d..f71c34be6b 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -12,7 +12,11 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration -from sentry_sdk.integrations.boto3._instrumentation import _instrument_streaming_body +from sentry_sdk.integrations.boto3._instrumentation import ( + _get_error_attributes, + _get_response_attributes, + _instrument_streaming_body, +) from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from sentry_sdk.traces import StreamedSpan @@ -355,6 +359,7 @@ def _assert_span_finished(span, span_streaming): def _assert_one_failed_span(spans, span_streaming): assert len(spans) == 1 assert spans[0]["status"] in ("error", "internal_error") + assert _span_attributes(spans[0], span_streaming)[SPANDATA.ERROR_TYPE] _assert_span_finished(spans[0], span_streaming) @@ -364,9 +369,12 @@ def _capture_stubbed_client_span( api_params, capture_items, span_streaming, + response=None, ): with Stubber(client) as stubber: - stubber.add_response(method_name, {}, api_params) + stubber.add_response( + method_name, response if response is not None else {}, api_params + ) spans_by_op = _capture_boto3_spans_by_op( lambda: getattr(client, method_name)(**api_params), capture_items, @@ -382,6 +390,124 @@ def _span_attributes(span, span_streaming): return span["attributes"] if span_streaming else span["data"] +@pytest.mark.parametrize( + ("response", "expected"), + [ + (None, {}), + ({}, {}), + ({"ResponseMetadata": None}, {}), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HostId": "extended-request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 0, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + }, + ), + ( + { + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + }, + { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, + }, + ), + ], +) +def test_get_response_attributes(response, expected): + assert _get_response_attributes(response) == expected + + +@pytest.mark.parametrize( + "header_name", + ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"], +) +def test_get_response_attributes_reads_request_id_header(header_name): + response = { + "ResponseMetadata": { + "HTTPHeaders": {header_name: "request-id"}, + } + } + + assert _get_response_attributes(response) == {SPANDATA.AWS_REQUEST_ID: "request-id"} + + +def test_get_response_attributes_reads_extended_request_id_header(): + response = { + "ResponseMetadata": { + "HTTPHeaders": {"x-amz-id-2": "extended-request-id"}, + } + } + + assert _get_response_attributes(response) == { + SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id" + } + + +@pytest.mark.parametrize( + ("field", "value", "attribute"), + [ + ("RequestId", 123, SPANDATA.AWS_REQUEST_ID), + ("RequestId", "", SPANDATA.AWS_REQUEST_ID), + ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), + ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), + ("RetryAttempts", "2", SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", False, SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ("RetryAttempts", -1, SPANDATA.HTTP_REQUEST_RESEND_COUNT), + ], +) +def test_get_response_attributes_ignores_malformed_field(field, value, attribute): + metadata = { + "RequestId": "request-id", + "HTTPStatusCode": 200, + "RetryAttempts": 2, + } + metadata[field] = value + + attributes = _get_response_attributes({"ResponseMetadata": metadata}) + expected = { + SPANDATA.AWS_REQUEST_ID: "request-id", + SPANDATA.HTTP_STATUS_CODE: 200, + SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, + } + expected.pop(attribute) + assert attributes == expected + + +@pytest.mark.parametrize( + "error_response", + [None, {"Code": ""}, {"Code": 123}], +) +def test_get_error_attributes_ignores_malformed_client_error_code(error_response): + error = ClientError( + { + "Error": {"Code": "placeholder"}, + "ResponseMetadata": {"HTTPStatusCode": 400}, + }, + "HeadObject", + ) + error.response["Error"] = error_response + + assert _get_error_attributes(error) == { + SPANDATA.HTTP_STATUS_CODE: 400, + SPANDATA.ERROR_TYPE: "botocore.exceptions.ClientError", + } + + @pytest.mark.parametrize( ( "service_name", @@ -517,6 +643,37 @@ def test_client_call_omits_missing_region( assert SPANDATA.CLOUD_REGION not in span["attributes"] +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_call_has_response_attributes( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + span = _capture_stubbed_client_span( + client, + "head_object", + {"Bucket": "bucket", "Key": "foo"}, + capture_items, + span_streaming, + response={ + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "request-id", + "HostId": "extended-request-id", + "RetryAttempts": 0, + } + }, + ) + attributes = _span_attributes(span, span_streaming) + + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes + assert SPANDATA.ERROR_TYPE not in attributes + + @pytest.mark.parametrize("span_streaming", [True, False]) def test_retry_attempts_share_one_client_span( capture_items, @@ -538,6 +695,8 @@ def test_retry_attempts_share_one_client_span( # all `AWSRequest` instances created during retries reference the same client span. assert len(set(request_span_ids)) == 1 assert len(client_spans) == 1 + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == attempt_count - 1 @pytest.mark.parametrize("span_streaming", [True, False]) @@ -561,6 +720,57 @@ def attempt_failed_head_object_call(): assert len(request_span_ids) == 2 assert len(set(request_span_ids)) == 1 _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 500 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_client_error_has_response_attributes_and_is_unchanged( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + original_exception = ClientError( + { + "Error": { + "Code": "AccessDeniedException", + "Message": "must not become a span attribute", + }, + "ResponseMetadata": { + "RequestId": "request-id", + "HTTPStatusCode": 403, + "RetryAttempts": 1, + }, + }, + "HeadObject", + ) + + def raise_client_error(**kwargs): + raise original_exception + + client.meta.events.register("before-parameter-build", raise_client_error) + + def invoke_failing_client_method(): + with pytest.raises(ClientError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 + assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 + assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + assert "Error.Message" not in attributes + assert "exception.message" not in attributes + assert "error.message" not in attributes @pytest.mark.parametrize( @@ -601,6 +811,132 @@ def invoke_failing_client_method(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) _assert_one_failed_span(client_spans, span_streaming) + attributes = _span_attributes(client_spans[0], span_streaming) + expected_error_type = ( + "botocore.exceptions.EndpointConnectionError" + if event_name == "before-send" + else "ValueError" + ) + assert attributes[SPANDATA.ERROR_TYPE] == expected_error_type + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_response_attribute_extraction_failure_does_not_change_response( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + api_params = {"Bucket": "bucket", "Key": "foo"} + original_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} + returned_responses = [] + + def fail_attribute_extraction(response): + raise RuntimeError("attribute extraction failed") + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._instrumentation._get_response_attributes", + fail_attribute_extraction, + ) + + def invoke_client_method(): + returned_responses.append(client.head_object(**api_params)) + + with Stubber(client) as stubber: + stubber.add_response("head_object", original_response, api_params) + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method, capture_items, span_streaming + ) + + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + assert returned_responses == [original_response] + assert returned_responses[0] is original_response + assert len(client_spans) == 1 + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.tests_internal_exceptions +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_error_attribute_extraction_failure_does_not_replace_original_exception( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + client = client_factory() + original_exception = ValueError("parameter processing failed") + + def raise_original_exception(**kwargs): + raise original_exception + + def fail_attribute_extraction(exception): + raise RuntimeError("attribute extraction failed") + + client.meta.events.register("before-parameter-build", raise_original_exception) + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._instrumentation._get_error_attributes", + fail_attribute_extraction, + ) + + def invoke_failing_client_method(): + with pytest.raises(ValueError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is original_exception + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + assert len(client_spans) == 1 + assert client_spans[0]["status"] in ("error", "internal_error") + _assert_span_finished(client_spans[0], span_streaming) + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_streaming_response_attributes_belong_to_client_span( + capture_items, + client_factory, + span_streaming, +): + client = client_factory() + + def respond(request, **kwargs): + return AWSResponse( + request.url, + 200, + { + "content-length": "5", + "x-amz-request-id": "request-id", + }, + Body(b"hello"), + ) + + client.meta.events.register("before-send", respond) + + def invoke_client_method_and_read_body(): + body = client.get_object(Bucket="bucket", Key="foo")["Body"] + assert body.read() == b"hello" + assert body.read() == b"" + + spans_by_op = _capture_boto3_spans_by_op( + invoke_client_method_and_read_body, capture_items, span_streaming + ) + client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) + stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) + + assert len(client_spans) == 1 + assert len(stream_spans) == 1 + client_attributes = _span_attributes(client_spans[0], span_streaming) + stream_attributes = _span_attributes(stream_spans[0], span_streaming) + assert client_attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert client_attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in client_attributes + assert SPANDATA.AWS_REQUEST_ID not in stream_attributes + assert SPANDATA.HTTP_STATUS_CODE not in stream_attributes + @pytest.mark.parametrize("span_streaming", [True, False]) def test_streaming_body_read_failure_finishes_stream_span( @@ -646,3 +982,5 @@ def invoke_client_method_and_read_body(): assert len(client_spans) == 1 _assert_one_failed_span(client_spans, span_streaming) _assert_one_failed_span(stream_spans, span_streaming) + attributes = _span_attributes(stream_spans[0], span_streaming) + assert attributes[SPANDATA.ERROR_TYPE] == "OSError" From 250699d055c83947bc4b9831c36a318931a69ffd Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 15 Sep 2026 17:21:18 +0200 Subject: [PATCH 30/72] patch correct methods --- tests/integrations/boto3/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index f71c34be6b..9e0d678449 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -837,7 +837,7 @@ def fail_attribute_extraction(response): raise RuntimeError("attribute extraction failed") monkeypatch.setattr( - "sentry_sdk.integrations.boto3._instrumentation._get_response_attributes", + "sentry_sdk.integrations.boto3._client._get_response_attributes", fail_attribute_extraction, ) @@ -876,7 +876,7 @@ def fail_attribute_extraction(exception): client.meta.events.register("before-parameter-build", raise_original_exception) monkeypatch.setattr( - "sentry_sdk.integrations.boto3._instrumentation._get_error_attributes", + "sentry_sdk.integrations.boto3._client._get_error_attributes", fail_attribute_extraction, ) From d9edef50eb13dde787d7cd8d5ca5cb5a214d21cf Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 09:54:42 +0200 Subject: [PATCH 31/72] lint --- sentry_sdk/integrations/boto3/_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index ba390c9450..b7e0e1e526 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -122,9 +122,7 @@ def sentry_patched_make_api_call( else: if span is not None: with capture_internal_exceptions(): - _set_span_attributes( - span, _get_response_attributes(parsed) - ) + _set_span_attributes(span, _get_response_attributes(parsed)) except BaseException as error: _finish_span(span, error) raise From 90383dcbf80b601c117bad2ee946fd877420a059 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 13:55:57 +0200 Subject: [PATCH 32/72] use ordered search for request id and hist id --- .../integrations/boto3/_instrumentation.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index b0e2056676..3245c98f40 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -119,29 +119,33 @@ def _get_response_attributes(response: "Any") -> "Attributes": if not isinstance(headers, dict): headers = {} - request_id = metadata.get("RequestId") - if not isinstance(request_id, str) or not request_id: - request_id = next( - ( - value - for value in ( - headers.get("x-amzn-requestid"), - headers.get("x-amzn-request-id"), - headers.get("x-amz-request-id"), - ) - if isinstance(value, str) and value - ), - None, - ) - if isinstance(request_id, str) and request_id: + request_id = next( + ( + value + for value in ( + metadata.get("RequestId"), + headers.get("x-amzn-requestid"), + headers.get("x-amzn-request-id"), + headers.get("x-amz-request-id"), + ) + if isinstance(value, str) and value + ), + None, + ) + if request_id is not None: attributes[SPANDATA.AWS_REQUEST_ID] = request_id # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html - extended_request_id = metadata.get("HostId") - if not isinstance(extended_request_id, str) or not extended_request_id: - extended_request_id = headers.get("x-amz-id-2") - if isinstance(extended_request_id, str) and extended_request_id: + extended_request_id = next( + ( + value + for value in (metadata.get("HostId"), headers.get("x-amz-id-2")) + if isinstance(value, str) and value + ), + None, + ) + if extended_request_id is not None: attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id return attributes From d482500c9f59138d106a58d10e01be4214b279ec Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 14:14:14 +0200 Subject: [PATCH 33/72] ref(boto3): remove redundant span checks during enrichment --- sentry_sdk/integrations/boto3/_client.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index b7e0e1e526..2bbcb986c6 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -115,14 +115,12 @@ def sentry_patched_make_api_call( try: parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as error: - if span is not None: - with capture_internal_exceptions(): - _set_span_attributes(span, _get_error_attributes(error)) + with capture_internal_exceptions(): + _set_span_attributes(span, _get_error_attributes(error)) raise else: - if span is not None: - with capture_internal_exceptions(): - _set_span_attributes(span, _get_response_attributes(parsed)) + with capture_internal_exceptions(): + _set_span_attributes(span, _get_response_attributes(parsed)) except BaseException as error: _finish_span(span, error) raise From 26fdcb1dc81f5326500605db7864f7db14fc0089 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 15:07:33 +0200 Subject: [PATCH 34/72] fix duplicates when merging --- sentry_sdk/integrations/boto3/_client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 2bbcb986c6..5d4ecc5e16 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -126,9 +126,6 @@ def sentry_patched_make_api_call( raise streaming_body_instrumented = False - with capture_internal_exceptions(): - streaming_body_instrumented = _instrument_streaming_body(span, parsed) - streaming_body_instrumented = False with capture_internal_exceptions(): streaming_body_instrumented = _instrument_streaming_body(span, parsed) From eeac30a040869d03a740b6101ce8cf6e18807020 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Mon, 21 Sep 2026 17:57:55 +0200 Subject: [PATCH 35/72] use consts and string lookup for `client.getintegration` --- sentry_sdk/integrations/boto3/_client.py | 5 ++--- sentry_sdk/integrations/boto3/_instrumentation.py | 4 ++-- tests/integrations/boto3/test_client.py | 2 +- tests/integrations/boto3/test_s3.py | 9 +++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 5d4ecc5e16..3c125787bc 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -4,6 +4,7 @@ import sentry_sdk from sentry_sdk.integrations import DidNotEnable from sentry_sdk.integrations.boto3._context import AwsCallContext +from sentry_sdk.integrations.boto3.consts import IDENTIFIER from sentry_sdk.integrations.boto3._instrumentation import ( _finish_span, _get_error_attributes, @@ -68,8 +69,6 @@ def _activate_client_span( def _patch_botocore_client() -> None: - from sentry_sdk.integrations.boto3 import Boto3Integration - orig_init = BaseClient.__init__ orig_make_api_call = BaseClient._make_api_call # type: ignore @@ -91,7 +90,7 @@ def sentry_patched_make_api_call( https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/#rpc-client-span """ client = sentry_sdk.get_client() - if client.get_integration(Boto3Integration) is None: + if client.get_integration(IDENTIFIER) is None: return orig_make_api_call(self, operation_name, api_params) ctx = AwsCallContext(operation_name, api_params) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 3245c98f40..031dd91297 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -438,7 +438,7 @@ def _sentry_request_created( """ client = sentry_sdk.get_client() - if client.get_integration("boto3") is None: + if client.get_integration(IDENTIFIER) is None: return with capture_internal_exceptions(): @@ -466,7 +466,7 @@ def _sentry_before_sign( request: "AWSRequest", signature_version: "Any", **kwargs: "Any" ) -> None: client = sentry_sdk.get_client() - if client.get_integration("boto3") is None: + if client.get_integration(IDENTIFIER) is None: return with capture_internal_exceptions(): diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 9e0d678449..3e0d5265ee 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -613,7 +613,7 @@ def test_client_call_attributes_are_available_at_span_creation( item.payload for item in items if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) - == Boto3Integration.origin + == ORIGIN ] assert client_spans == [] diff --git a/tests/integrations/boto3/test_s3.py b/tests/integrations/boto3/test_s3.py index 8c8b24ba13..4af2e55ee2 100644 --- a/tests/integrations/boto3/test_s3.py +++ b/tests/integrations/boto3/test_s3.py @@ -7,6 +7,7 @@ from sentry_sdk import capture_message from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration +from sentry_sdk.integrations.boto3.consts import ORIGIN from tests.conftest import ApproxDict from tests.integrations.boto3 import read_fixture from tests.integrations.boto3.aws_mock import MockResponse @@ -130,7 +131,7 @@ def test_streaming( "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", - "sentry.origin": "auto.http.boto3", + "sentry.origin": ORIGIN, "sentry.release": mock.ANY, "sentry.sdk.name": "sentry.python", "sentry.sdk.version": mock.ANY, @@ -292,7 +293,7 @@ def test_omit_url_data_if_parsing_fails( "rpc.service": "S3", "sentry.environment": "production", "sentry.op": "http.client", - "sentry.origin": "auto.http.boto3", + "sentry.origin": ORIGIN, "sentry.release": mock.ANY, "sentry.sdk.name": "sentry.python", "sentry.sdk.version": mock.ANY, @@ -364,7 +365,7 @@ def test_span_origin( spans = [item.payload for item in items] assert spans[1]["attributes"]["sentry.origin"] == "manual" - assert spans[0]["attributes"]["sentry.origin"] == "auto.http.boto3" + assert spans[0]["attributes"]["sentry.origin"] == ORIGIN else: events = capture_events() @@ -376,7 +377,7 @@ def test_span_origin( (event,) = events assert event["contexts"]["trace"]["origin"] == "manual" - assert event["spans"][0]["origin"] == "auto.http.boto3" + assert event["spans"][0]["origin"] == ORIGIN def test_breadcrumb(sentry_init, capture_events): From 8b61aac1bda784c8e9923b513385f73ca7d54ab8 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 11:15:46 +0200 Subject: [PATCH 36/72] lint --- sentry_sdk/integrations/boto3/_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 3c125787bc..8b7cfeed0a 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -4,7 +4,6 @@ import sentry_sdk from sentry_sdk.integrations import DidNotEnable from sentry_sdk.integrations.boto3._context import AwsCallContext -from sentry_sdk.integrations.boto3.consts import IDENTIFIER from sentry_sdk.integrations.boto3._instrumentation import ( _finish_span, _get_error_attributes, @@ -15,6 +14,7 @@ _set_span_attributes, _start_client_span, ) +from sentry_sdk.integrations.boto3.consts import IDENTIFIER from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan from sentry_sdk.utils import capture_internal_exceptions From 308eb5f39ef336c1fd7481935ad823a4685beb0c Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 12:05:23 +0200 Subject: [PATCH 37/72] ruff --- tests/integrations/boto3/test_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 3e0d5265ee..d679c576a9 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -612,8 +612,7 @@ def test_client_call_attributes_are_available_at_span_creation( client_spans = [ item.payload for item in items - if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) - == ORIGIN + if item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN ] assert client_spans == [] From 8e4ce5abfc3ca489bc3101eda1a9266320ca78f3 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:47:55 +0200 Subject: [PATCH 38/72] ref(boto3): add permalinks --- sentry_sdk/integrations/boto3/_instrumentation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 031dd91297..457f28e150 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -98,14 +98,14 @@ def _get_response_attributes(response: "Any") -> "Attributes": attributes: "Attributes" = {} # botocore injects HTTP status into `ResponseMetadata` after parsing. - # https://github.com/boto/botocore/blob/develop/botocore/parsers.py#L273-L284 + # https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/parsers.py#L273-L284 status_code = metadata.get("HTTPStatusCode") if isinstance(status_code, int) and 100 <= status_code <= 599: attributes[SPANDATA.HTTP_STATUS_CODE] = status_code retry_attempts = metadata.get("RetryAttempts") # botocore represents retries as `attempts - 1`; OTel suggests "if and only if", so skip zero. - # https://github.com/boto/botocore/blob/develop/botocore/endpoint.py#L221-L229 + # https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/endpoint.py#L221-L229 # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span if ( isinstance(retry_attempts, int) From 832d90d123a621f8ad38b33c217b482ca45b3a42 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 10:54:31 +0200 Subject: [PATCH 39/72] fix(boto3): add `error.type` to both span kinds --- sentry_sdk/integrations/boto3/_instrumentation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 457f28e150..884000ff4f 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -299,8 +299,7 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: with capture_internal_exceptions(): attributes = _get_error_attributes(error) _set_span_attributes(streaming_span, attributes) - if isinstance(span, StreamedSpan): - _set_span_attributes(span, attributes) + _set_span_attributes(span, attributes) _finish_span(streaming_span, error) _finish_span(span, error) From 0ee5f141ee82bc3fd6ebf5f90a77ebc5a0f434ef Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 11:02:58 +0200 Subject: [PATCH 40/72] ref(boto3): specify docstring for `_set_span_attributes()` further --- sentry_sdk/integrations/boto3/_instrumentation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 884000ff4f..59236d1290 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -41,7 +41,11 @@ def _set_span_attributes( span: "Union[Span, StreamedSpan]", attributes: "Attributes" ) -> None: - """Will be removed in the major.""" + """ + Will be removed in the next major version (3.0). This helper makes + it easier to migrate to `StreamedSpan` without having to remove + multiple conditional blocks intertwined with other logic. + """ if isinstance(span, StreamedSpan): span.set_attributes(attributes) return From a8f521e55c4de51266186f383e951fb2420a6c7f Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 11:05:13 +0200 Subject: [PATCH 41/72] ref(boto3): use f-string instead --- sentry_sdk/integrations/boto3/_instrumentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 59236d1290..3ec10ef620 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -172,7 +172,7 @@ def _get_error_type(exception: "BaseException") -> str: exception_name = exception_type.__qualname__ exception_module = exception_type.__module__ if exception_module not in ("builtins", "__builtins__"): - return "%s.%s" % (exception_module, exception_name) + return f"{exception_module}.{exception_name}" return exception_name From 2d92b4ee590a3bf226b581964e78813fd9ac0b20 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 11:19:07 +0200 Subject: [PATCH 42/72] ref(boto3): remove comment regarding `HTTP_REQUEST_RESEND_COUNT` --- sentry_sdk/integrations/boto3/_instrumentation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 3ec10ef620..1907468835 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -108,9 +108,6 @@ def _get_response_attributes(response: "Any") -> "Attributes": attributes[SPANDATA.HTTP_STATUS_CODE] = status_code retry_attempts = metadata.get("RetryAttempts") - # botocore represents retries as `attempts - 1`; OTel suggests "if and only if", so skip zero. - # https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/endpoint.py#L221-L229 - # https://opentelemetry.io/docs/specs/semconv/http/http-spans/#http-client-span if ( isinstance(retry_attempts, int) # avoid emitting `resend_count=True`. From 12dd5e63f0082f66dfdb5aad15c1ffbf7094c474 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 11:23:04 +0200 Subject: [PATCH 43/72] ref(boto3): remove `else` block --- sentry_sdk/integrations/boto3/_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 8b7cfeed0a..d30770ebe7 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -117,9 +117,8 @@ def sentry_patched_make_api_call( with capture_internal_exceptions(): _set_span_attributes(span, _get_error_attributes(error)) raise - else: - with capture_internal_exceptions(): - _set_span_attributes(span, _get_response_attributes(parsed)) + with capture_internal_exceptions(): + _set_span_attributes(span, _get_response_attributes(parsed)) except BaseException as error: _finish_span(span, error) raise From 21794bf3287f32fc25452bcd9de641fc2e1ab198 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 12:55:32 +0200 Subject: [PATCH 44/72] tests(boto3): remove uncessary testcases --- tests/integrations/boto3/test_client.py | 26 ------------------------- 1 file changed, 26 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index d679c576a9..2706488109 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -13,7 +13,6 @@ from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration from sentry_sdk.integrations.boto3._instrumentation import ( - _get_error_attributes, _get_response_attributes, _instrument_streaming_body, ) @@ -393,9 +392,7 @@ def _span_attributes(span, span_streaming): @pytest.mark.parametrize( ("response", "expected"), [ - (None, {}), ({}, {}), - ({"ResponseMetadata": None}, {}), ( { "ResponseMetadata": { @@ -465,9 +462,6 @@ def test_get_response_attributes_reads_extended_request_id_header(): ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), - ("RetryAttempts", "2", SPANDATA.HTTP_REQUEST_RESEND_COUNT), - ("RetryAttempts", False, SPANDATA.HTTP_REQUEST_RESEND_COUNT), - ("RetryAttempts", -1, SPANDATA.HTTP_REQUEST_RESEND_COUNT), ], ) def test_get_response_attributes_ignores_malformed_field(field, value, attribute): @@ -488,26 +482,6 @@ def test_get_response_attributes_ignores_malformed_field(field, value, attribute assert attributes == expected -@pytest.mark.parametrize( - "error_response", - [None, {"Code": ""}, {"Code": 123}], -) -def test_get_error_attributes_ignores_malformed_client_error_code(error_response): - error = ClientError( - { - "Error": {"Code": "placeholder"}, - "ResponseMetadata": {"HTTPStatusCode": 400}, - }, - "HeadObject", - ) - error.response["Error"] = error_response - - assert _get_error_attributes(error) == { - SPANDATA.HTTP_STATUS_CODE: 400, - SPANDATA.ERROR_TYPE: "botocore.exceptions.ClientError", - } - - @pytest.mark.parametrize( ( "service_name", From 7859e97871d80b027c049d49f42150759807fa7a Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 12:57:20 +0200 Subject: [PATCH 45/72] ref(boto3): remove uncessary defensive conditionals --- .../integrations/boto3/_instrumentation.py | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 1907468835..74f8cce948 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -25,7 +25,7 @@ ) if TYPE_CHECKING: - from typing import Any, Dict, Optional, Union + from typing import Any, Dict, Mapping, Optional, Union from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext @@ -92,13 +92,8 @@ def _get_client_attributes( return attributes -def _get_response_attributes(response: "Any") -> "Attributes": - if not isinstance(response, dict): - return {} - - metadata = response.get("ResponseMetadata") - if not isinstance(metadata, dict): - return {} +def _get_response_attributes(response: "Mapping[str, Any]") -> "Attributes": + metadata = response.get("ResponseMetadata", {}) attributes: "Attributes" = {} # botocore injects HTTP status into `ResponseMetadata` after parsing. @@ -107,18 +102,11 @@ def _get_response_attributes(response: "Any") -> "Attributes": if isinstance(status_code, int) and 100 <= status_code <= 599: attributes[SPANDATA.HTTP_STATUS_CODE] = status_code - retry_attempts = metadata.get("RetryAttempts") - if ( - isinstance(retry_attempts, int) - # avoid emitting `resend_count=True`. - and not isinstance(retry_attempts, bool) - and retry_attempts > 0 - ): + retry_attempts = metadata.get("RetryAttempts", 0) + if retry_attempts > 0: attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] = retry_attempts - headers = metadata.get("HTTPHeaders") - if not isinstance(headers, dict): - headers = {} + headers = metadata.get("HTTPHeaders", {}) request_id = next( ( @@ -157,11 +145,9 @@ def _get_error_type(exception: "BaseException") -> str: # `ClientError` wraps AWS service errors; `Error.Code` identifies the # actual service error, e.g. `AccessDeniedException`. # https://docs.aws.amazon.com/boto3/latest/guide/error-handling.html - error = exception.response.get("Error") - if isinstance(error, dict): - error_code = error.get("Code") - if isinstance(error_code, str) and error_code: - return error_code + error_code: "Optional[str]" = exception.response.get("Error", {}).get("Code") + if error_code: + return error_code # failures before a service response have no AWS error code. # https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/ From 1f4e936dd2f83f4c4686f47e372123e13941d405 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 16:04:32 +0200 Subject: [PATCH 46/72] fix flaky test --- tests/integrations/boto3/test_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 2706488109..9b2d502115 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -108,6 +108,7 @@ def record_client_span(request, **kwargs): assert isinstance(request_client_span, Span) assert not isinstance(request_client_span, StreamedSpan) assert request_client_span.timestamp is None + assert sentry_sdk.get_current_span() is parent if consume == "read": assert body.read() == b"x" @@ -134,6 +135,7 @@ def record_client_span(request, **kwargs): assert sentry_sdk.traces.get_current_span() is parent # type: ignore[attr-defined] else: assert request_client_span.timestamp is not None + assert sentry_sdk.get_current_span() is parent sentry_sdk.flush() if span_streaming: @@ -182,10 +184,8 @@ def record_client_span(request, **kwargs): assert stream_span["parent_span_id"] == client_span["span_id"] assert client_span["span_id"] == request_client_span.span_id end_timestamp = "end_timestamp" if span_streaming else "timestamp" - assert client_span["start_timestamp"] <= http_span["start_timestamp"] - assert http_span["start_timestamp"] <= stream_span["start_timestamp"] - assert http_span[end_timestamp] <= stream_span[end_timestamp] - assert stream_span[end_timestamp] <= client_span[end_timestamp] + for span in (client_span, http_span, stream_span): + assert span[end_timestamp] is not None @pytest.mark.parametrize("span_streaming", [True, False]) From 02a8fa1ac0899949159a81e14b29f4a5576597dc Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 10:15:38 +0200 Subject: [PATCH 47/72] add base `ServiceExtension` class --- .../integrations/boto3/_services/base.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sentry_sdk/integrations/boto3/_services/base.py diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py new file mode 100644 index 0000000000..0c5e3be964 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -0,0 +1,31 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Optional, Tuple + + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext + + +class _ServiceExtension: + """ + Specialize generic botocore instrumentation for an AWS service. + """ + + __slots__ = () + + def get_span_config( + self, ctx: "AwsCallContext" + ) -> "Optional[Tuple[Optional[str], Optional[str]]]": + """Return an optional `(op, origin)` override for the client span.""" + return None + + def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": + """Return service-specific attributes available before the call.""" + return {} + + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + """Return service-specific attributes derived from the response.""" + return {} From ee7e768380aee9e7b2f05afa839b4e6d6656904d Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 11:16:57 +0200 Subject: [PATCH 48/72] feat(boto3): Add service registry --- .../integrations/boto3/_services/registry.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 sentry_sdk/integrations/boto3/_services/registry.py diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py new file mode 100644 index 0000000000..18d1f08553 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -0,0 +1,34 @@ +from functools import lru_cache +from importlib import import_module +from typing import TYPE_CHECKING + +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension +from sentry_sdk.utils import capture_internal_exceptions + +if TYPE_CHECKING: + from typing import Dict, Optional, Tuple + + +# service modules are imported lazily. +# e.g. `s3` -> (`sentry_sdk.integrations.boto3._services.s3`, `_S3Extension) +_SERVICE_EXTENSIONS: "Dict[str, Tuple[str, str]]" = {} + + +@lru_cache(maxsize=None) +def _resolve_service( + service: "str", +) -> "Optional[_ServiceExtension]": + target = _SERVICE_EXTENSIONS.get(service) + if target is None: + return None + + # preserve generic instrumentation when lookup fails. + extension = None + with capture_internal_exceptions(): + module_name, class_name = target + extension_class = getattr(import_module(module_name), class_name) + candidate = extension_class() + if isinstance(candidate, _ServiceExtension): + extension = candidate + + return extension From 4d9ae30c6d02bd24c48186f5afe9cdfcdbbfdb55 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 11:34:18 +0200 Subject: [PATCH 49/72] add missing `__init__.py` file --- sentry_sdk/integrations/boto3/_services/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 sentry_sdk/integrations/boto3/_services/__init__.py diff --git a/sentry_sdk/integrations/boto3/_services/__init__.py b/sentry_sdk/integrations/boto3/_services/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From e90b407379c135b3c0698071833bc4309607fc83 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 13:11:10 +0200 Subject: [PATCH 50/72] simplify `_resolve_service()` logic --- sentry_sdk/integrations/boto3/_services/registry.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index 18d1f08553..57f53c6b9b 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -22,13 +22,12 @@ def _resolve_service( if target is None: return None - # preserve generic instrumentation when lookup fails. - extension = None with capture_internal_exceptions(): module_name, class_name = target extension_class = getattr(import_module(module_name), class_name) candidate = extension_class() if isinstance(candidate, _ServiceExtension): - extension = candidate + return candidate - return extension + # preserve generic instrumentation when lookup fails. + return None From af519a024a10ebfb822f90229065c61ee73b03b8 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 13:12:54 +0200 Subject: [PATCH 51/72] add service extension logic to `_instrumentation.py` --- .../integrations/boto3/_instrumentation.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 74f8cce948..153d51180e 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -29,6 +29,7 @@ from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.integrations.boto3._services.base import _ServiceExtension try: from botocore.awsrequest import AWSRequest @@ -170,6 +171,7 @@ def _get_error_attributes(exception: "BaseException") -> "Attributes": def _start_client_span( ctx: "AwsCallContext", + service_ext: "Optional[_ServiceExtension]" = None, ) -> "Optional[Union[Span, StreamedSpan]]": client = sentry_sdk.get_client() @@ -189,6 +191,32 @@ def _start_client_span( span_op = OP.HTTP_CLIENT span_origin = ORIGIN + if service_ext is not None: + service_span_config = None + with capture_internal_exceptions(): + service_span_config = service_ext.get_span_config(ctx) + + with capture_internal_exceptions(): + if service_span_config is not None: + service_op, service_origin = service_span_config + if isinstance(service_op, str) and service_op: + span_op = service_op + if isinstance(service_origin, str) and service_origin: + span_origin = service_origin + + with capture_internal_exceptions(): + attributes.update(service_ext.get_request_attributes(ctx)) + + # Generic attributes take precedence over service-specific attributes. + attributes.update( + { + SPANDATA.RPC_METHOD: ctx.operation_name, + SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME, + } + ) + with capture_internal_exceptions(): + attributes.update(_get_client_attributes(ctx)) + if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: return None From 98cc2a4d16c2b85db482f25c39678b787b98f004 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 13:17:06 +0200 Subject: [PATCH 52/72] combine span-config extraction and validation in `_start_client_span()` --- sentry_sdk/integrations/boto3/_instrumentation.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 153d51180e..dd05ccd5f7 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -192,13 +192,10 @@ def _start_client_span( span_origin = ORIGIN if service_ext is not None: - service_span_config = None with capture_internal_exceptions(): - service_span_config = service_ext.get_span_config(ctx) - - with capture_internal_exceptions(): - if service_span_config is not None: - service_op, service_origin = service_span_config + config = service_ext.get_span_config(ctx) + if config is not None: + service_op, service_origin = config if isinstance(service_op, str) and service_op: span_op = service_op if isinstance(service_origin, str) and service_origin: From ca7e4497517703e532cb20b2e167078945810020 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 11:19:33 +0200 Subject: [PATCH 53/72] add service extension logic to `_client.py` --- sentry_sdk/integrations/boto3/_client.py | 32 +++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index d30770ebe7..a947d1dc91 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -14,6 +14,9 @@ _set_span_attributes, _start_client_span, ) +from sentry_sdk.integrations.boto3._services.registry import ( + _resolve_service, +) from sentry_sdk.integrations.boto3.consts import IDENTIFIER from sentry_sdk.traces import NoOpStreamedSpan, StreamedSpan from sentry_sdk.utils import capture_internal_exceptions @@ -21,10 +24,13 @@ if TYPE_CHECKING: from typing import Any, Iterator, Optional, Union + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.tracing import Span try: from botocore.client import BaseClient + from botocore.exceptions import ClientError except ImportError: raise DidNotEnable("botocore not installed") @@ -99,9 +105,14 @@ def sentry_patched_make_api_call( with capture_internal_exceptions(): ctx.add_metadata(self) + service_ext: "Optional[_ServiceExtension]" = None + with capture_internal_exceptions(): + # resolve service extension for service-specific enrichment. + service_ext = _resolve_service(ctx.service_name) + span: "Optional[Union[Span, StreamedSpan]]" = None with capture_internal_exceptions(): - span = _start_client_span(ctx) + span = _start_client_span(ctx, service_ext) if span is None: return orig_make_api_call(self, operation_name, api_params) @@ -109,16 +120,31 @@ def sentry_patched_make_api_call( # activate without finishing; a streaming response may outlive the call. span_ctx = _activate_client_span(span) + attributes: "Attributes" = {} try: with span_ctx: try: parsed = orig_make_api_call(self, operation_name, api_params) except BaseException as error: + if service_ext is not None and isinstance(error, ClientError): + with capture_internal_exceptions(): + attributes.update( + service_ext.get_response_attributes(ctx, error.response) + ) with capture_internal_exceptions(): - _set_span_attributes(span, _get_error_attributes(error)) + attributes.update(_get_error_attributes(error)) + with capture_internal_exceptions(): + _set_span_attributes(span, attributes) raise + if service_ext is not None: + with capture_internal_exceptions(): + attributes.update( + service_ext.get_response_attributes(ctx, parsed) + ) + with capture_internal_exceptions(): + attributes.update(_get_response_attributes(parsed)) with capture_internal_exceptions(): - _set_span_attributes(span, _get_response_attributes(parsed)) + _set_span_attributes(span, attributes) except BaseException as error: _finish_span(span, error) raise From f95f4fe0f4dc5afcaac9447cd4a5b0b9e5eae2d1 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Fri, 18 Sep 2026 15:19:42 +0200 Subject: [PATCH 54/72] add tests --- tests/integrations/boto3/test_client.py | 110 ++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 9b2d502115..5d11512fb6 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -16,6 +16,7 @@ _get_response_attributes, _instrument_streaming_body, ) +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from sentry_sdk.traces import StreamedSpan @@ -389,6 +390,115 @@ def _span_attributes(span, span_streaming): return span["attributes"] if span_streaming else span["data"] +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_service_extension_customizes_client_span( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + class TestServiceExtension(_ServiceExtension): + def get_span_config(self, ctx): + return ("aws.test", None) + + def get_request_attributes(self, ctx): + return { + "aws.test.request": ctx.params["Key"], + SPANDATA.RPC_METHOD: "must-not-override", + } + + def get_response_attributes(self, ctx, response): + return { + "aws.test.response": response["ResponseMetadata"]["RequestId"], + SPANDATA.HTTP_STATUS_CODE: 418, + } + + extension = TestServiceExtension() + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._resolve_service", + lambda service_name: extension, + ) + client = client_factory() + api_params = {"Bucket": "bucket", "Key": "foo"} + + with Stubber(client) as stubber: + stubber.add_response( + "head_object", + { + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "request-id", + } + }, + api_params, + ) + spans_by_op = _capture_boto3_spans_by_op( + lambda: client.head_object(**api_params), + capture_items, + span_streaming, + ) + + spans = spans_by_op.get("aws.test", []) + assert len(spans) == 1 + attributes = _span_attributes(spans[0], span_streaming) + assert attributes["aws.test.request"] == "foo" + assert attributes["aws.test.response"] == "request-id" + assert attributes[SPANDATA.RPC_METHOD] == "HeadObject" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + + +@pytest.mark.parametrize("span_streaming", [True, False]) +def test_service_extension_enriches_client_error( + capture_items, + client_factory, + monkeypatch, + span_streaming, +): + class TestServiceExtension(_ServiceExtension): + def get_response_attributes(self, ctx, response): + return { + "aws.test.error": response["Error"]["Code"], + SPANDATA.ERROR_TYPE: "must-not-override", + SPANDATA.HTTP_STATUS_CODE: 418, + } + + monkeypatch.setattr( + "sentry_sdk.integrations.boto3._client._resolve_service", + lambda service_name: TestServiceExtension(), + ) + client = client_factory() + error = ClientError( + { + "Error": {"Code": "AccessDeniedException"}, + "ResponseMetadata": {"HTTPStatusCode": 403}, + }, + "HeadObject", + ) + + def raise_client_error(**kwargs): + raise error + + client.meta.events.register("before-parameter-build", raise_client_error) + + def invoke_failing_client_method(): + with pytest.raises(ClientError) as exc_info: + client.head_object(Bucket="bucket", Key="foo") + assert exc_info.value is error + + spans_by_op = _capture_boto3_spans_by_op( + invoke_failing_client_method, + capture_items, + span_streaming, + ) + spans = spans_by_op.get(OP.HTTP_CLIENT, []) + + _assert_one_failed_span(spans, span_streaming) + attributes = _span_attributes(spans[0], span_streaming) + assert attributes["aws.test.error"] == "AccessDeniedException" + assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 + + @pytest.mark.parametrize( ("response", "expected"), [ From b0b9f0cfb83b6362fc0918d4c368f6f243e90aa5 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 11:04:52 +0200 Subject: [PATCH 55/72] fix(boto3): Remove "hardcoded" origin - otherwise service-specific spans with different origins would be skipped (e.g. DynamoDB) --- sentry_sdk/integrations/boto3/_instrumentation.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index dd05ccd5f7..cf52837b20 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -463,10 +463,9 @@ def _sentry_request_created( if span is None: return - # an ignored streamed span is not activated; avoid enriching its parent. - if isinstance(span, StreamedSpan): - if not (span.get_attributes().get(SPANDATA.SENTRY_ORIGIN) == ORIGIN): - return + # an ignored streamed span is not active; avoid enriching its parent. + if isinstance(span, StreamedSpan) and span.active: + return _set_request_attributes(span, request) # each attempt has a fresh `request.context`; carry the active client span. From 7d645b42cbba616db31c00ec0d51be0bb38d93cb Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 15:12:02 +0200 Subject: [PATCH 56/72] feat(boto3): Add `sentry.kind` attribute to boto span --- sentry_sdk/consts.py | 6 ++++++ sentry_sdk/integrations/boto3/_instrumentation.py | 1 + tests/integrations/boto3/test_client.py | 3 +++ 3 files changed, 10 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 90e89afc95..5b037a66d0 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -1214,6 +1214,12 @@ class SPANDATA: Used in inbound filters. """ + SENTRY_KIND = "sentry.kind" + """ + Used to clarify the relationship between parents and children, or to distinguish between spans, e.g. a `server` and `client` span with the same name. + Example: "client", "server", "producer", "consumer", "internal" + """ + SENTRY_OP = "sentry.op" """ The operation of a span. diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index cf52837b20..9e602aa8a1 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -185,6 +185,7 @@ def _start_client_span( attributes: "Attributes" = { SPANDATA.RPC_METHOD: ctx.operation_name, SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, + SPANDATA.SENTRY_KIND: "client", } with capture_internal_exceptions(): attributes.update(_get_client_attributes(ctx)) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 5d11512fb6..fc2fff6988 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -404,6 +404,7 @@ def get_span_config(self, ctx): def get_request_attributes(self, ctx): return { "aws.test.request": ctx.params["Key"], + SPANDATA.SENTRY_KIND: "producer", SPANDATA.RPC_METHOD: "must-not-override", } @@ -443,6 +444,7 @@ def get_response_attributes(self, ctx, response): attributes = _span_attributes(spans[0], span_streaming) assert attributes["aws.test.request"] == "foo" assert attributes["aws.test.response"] == "request-id" + assert attributes[SPANDATA.SENTRY_KIND] == "producer" assert attributes[SPANDATA.RPC_METHOD] == "HeadObject" assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 @@ -658,6 +660,7 @@ def test_client_call_has_common_attributes( assert attributes[SPANDATA.RPC_SERVICE] == rpc_service assert attributes[SPANDATA.RPC_METHOD] == rpc_method assert attributes[SPANDATA.RPC_SYSTEM_NAME] == AWS_RPC_SYSTEM_NAME + assert attributes[SPANDATA.SENTRY_KIND] == "client" assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" assert attributes[SPANDATA.SERVER_ADDRESS] == server_address assert attributes[SPANDATA.SERVER_PORT] == server_port From 64e14643ece2f06b98b26f357ec190c738acb426 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 15:21:03 +0200 Subject: [PATCH 57/72] ref(boto3): Add comment explaining --- sentry_sdk/integrations/boto3/_instrumentation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 9e602aa8a1..6bbdb33835 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -185,6 +185,8 @@ def _start_client_span( attributes: "Attributes" = { SPANDATA.RPC_METHOD: ctx.operation_name, SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, + # all client call spans are by default "client" spans. + # https://opentelemetry.io/docs/specs/semconv/cloud-providers/aws-sdk/#aws-sdk-spans SPANDATA.SENTRY_KIND: "client", } with capture_internal_exceptions(): From f221d724f598be4d62bdc6c397afab4d77576471 Mon Sep 17 00:00:00 2001 From: Pablo Deputter <71842639+pabloDeputter@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:08:17 +0200 Subject: [PATCH 58/72] Update sentry_sdk/integrations/boto3/_services/base.py ref(boto3): improve docstring for `get_request_attributes()` Co-authored-by: Erica Pisani --- sentry_sdk/integrations/boto3/_services/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py index 0c5e3be964..bb5d648a9f 100644 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -21,7 +21,7 @@ def get_span_config( return None def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": - """Return service-specific attributes available before the call.""" + """Return service-specific attributes available before the request is made.""" return {} def get_response_attributes( From a106f76adb89385c146eb69ec10ed7b5de14aad0 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 13:09:20 +0200 Subject: [PATCH 59/72] fix(boto3): correct `AWS_RPC_SYSTEM_NAME` constant --- sentry_sdk/integrations/boto3/_instrumentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 6bbdb33835..23149a5968 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -211,7 +211,7 @@ def _start_client_span( attributes.update( { SPANDATA.RPC_METHOD: ctx.operation_name, - SPANDATA.RPC_SYSTEM_NAME: _AWS_RPC_SYSTEM_NAME, + SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, } ) with capture_internal_exceptions(): From 032eff31a5872eaa8cccaabea2bcda5641e60358 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 13:39:19 +0200 Subject: [PATCH 60/72] ref(boto3): refactor `get_span_config()` into `get_span_op()` and `get_span_origin()` --- .../integrations/boto3/_instrumentation.py | 15 ++++++----- .../integrations/boto3/_services/base.py | 12 +++++---- tests/integrations/boto3/test_client.py | 26 +++++++++++++++---- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index 23149a5968..d2109cab24 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -196,13 +196,14 @@ def _start_client_span( if service_ext is not None: with capture_internal_exceptions(): - config = service_ext.get_span_config(ctx) - if config is not None: - service_op, service_origin = config - if isinstance(service_op, str) and service_op: - span_op = service_op - if isinstance(service_origin, str) and service_origin: - span_origin = service_origin + service_op = service_ext.get_span_op(ctx) + if service_op is not None: + span_op = service_op + + with capture_internal_exceptions(): + service_origin = service_ext.get_span_origin(ctx) + if service_origin is not None: + span_origin = service_origin with capture_internal_exceptions(): attributes.update(service_ext.get_request_attributes(ctx)) diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py index bb5d648a9f..86cb8dfc8e 100644 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -1,7 +1,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from typing import Any, Optional, Tuple + from typing import Any, Optional from sentry_sdk._types import Attributes from sentry_sdk.integrations.boto3._context import AwsCallContext @@ -14,10 +14,12 @@ class _ServiceExtension: __slots__ = () - def get_span_config( - self, ctx: "AwsCallContext" - ) -> "Optional[Tuple[Optional[str], Optional[str]]]": - """Return an optional `(op, origin)` override for the client span.""" + def get_span_op(self, ctx: "AwsCallContext") -> "Optional[str]": + """Return an optional `sentry.op` override for the client span.""" + return None + + def get_span_origin(self, ctx: "AwsCallContext") -> "Optional[str]": + """Return an optional `sentry.origin` override for the client span.""" return None def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index fc2fff6988..7458393788 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -321,7 +321,12 @@ def respond(request, **kwargs): return request_span_ids -def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streaming): +def _capture_bot^o3_spans_by_op( + invoke_client_method, + capture_items, + span_streaming, + expected_origin=ORIGIN, +): items = capture_items() if span_streaming: @@ -333,14 +338,17 @@ def _capture_boto3_spans_by_op(invoke_client_method, capture_items, span_streami item.payload for item in items if item.type == "span" - and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) == ORIGIN + and item.payload["attributes"].get(SPANDATA.SENTRY_ORIGIN) + == expected_origin ] else: with sentry_sdk.start_transaction(): invoke_client_method() transaction = next(item.payload for item in items if item.type == "transaction") - spans = [span for span in transaction["spans"] if span["origin"] == ORIGIN] + spans = [ + span for span in transaction["spans"] if span["origin"] == expected_origin + ] spans_by_op = {} for span in spans: @@ -398,8 +406,11 @@ def test_service_extension_customizes_client_span( span_streaming, ): class TestServiceExtension(_ServiceExtension): - def get_span_config(self, ctx): - return ("aws.test", None) + def get_span_op(self, ctx): + return "aws.test" + + def get_span_origin(self, ctx): + return "auto.aws.test" def get_request_attributes(self, ctx): return { @@ -437,6 +448,7 @@ def get_response_attributes(self, ctx, response): lambda: client.head_object(**api_params), capture_items, span_streaming, + expected_origin="auto.aws.test", ) spans = spans_by_op.get("aws.test", []) @@ -447,6 +459,10 @@ def get_response_attributes(self, ctx, response): assert attributes[SPANDATA.SENTRY_KIND] == "producer" assert attributes[SPANDATA.RPC_METHOD] == "HeadObject" assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + if span_streaming: + assert attributes[SPANDATA.SENTRY_ORIGIN] == "auto.aws.test" + else: + assert spans[0]["origin"] == "auto.aws.test" @pytest.mark.parametrize("span_streaming", [True, False]) From b1d95516a1715bf2c4a97e65533bfb84114650da Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 29 Sep 2026 14:03:26 +0200 Subject: [PATCH 61/72] ref(boto3): remove attribute-precedence --- sentry_sdk/integrations/boto3/_instrumentation.py | 10 ---------- tests/integrations/boto3/test_client.py | 3 +-- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index d2109cab24..c486bfb005 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -208,16 +208,6 @@ def _start_client_span( with capture_internal_exceptions(): attributes.update(service_ext.get_request_attributes(ctx)) - # Generic attributes take precedence over service-specific attributes. - attributes.update( - { - SPANDATA.RPC_METHOD: ctx.operation_name, - SPANDATA.RPC_SYSTEM_NAME: AWS_RPC_SYSTEM_NAME, - } - ) - with capture_internal_exceptions(): - attributes.update(_get_client_attributes(ctx)) - if has_span_streaming_enabled(client.options): if sentry_sdk.traces.get_current_span() is None: return None diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index 7458393788..f11e064740 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -321,7 +321,7 @@ def respond(request, **kwargs): return request_span_ids -def _capture_bot^o3_spans_by_op( +def _capture_boto3_spans_by_op( invoke_client_method, capture_items, span_streaming, @@ -416,7 +416,6 @@ def get_request_attributes(self, ctx): return { "aws.test.request": ctx.params["Key"], SPANDATA.SENTRY_KIND: "producer", - SPANDATA.RPC_METHOD: "must-not-override", } def get_response_attributes(self, ctx, response): From c2f4f6ba4c2f4ff1a381f38f34bc5c5e86f21986 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 10:41:47 +0200 Subject: [PATCH 62/72] ref(boto3): remove lazy-loading approach; replace with static map --- .../integrations/boto3/_services/registry.py | 32 ++++++------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index 57f53c6b9b..e8616d5d86 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -1,33 +1,21 @@ -from functools import lru_cache -from importlib import import_module from typing import TYPE_CHECKING -from sentry_sdk.integrations.boto3._services.base import _ServiceExtension -from sentry_sdk.utils import capture_internal_exceptions - if TYPE_CHECKING: - from typing import Dict, Optional, Tuple + from typing import Dict, Optional + + from sentry_sdk.integrations.boto3._services.base import _ServiceExtension -# service modules are imported lazily. -# e.g. `s3` -> (`sentry_sdk.integrations.boto3._services.s3`, `_S3Extension) -_SERVICE_EXTENSIONS: "Dict[str, Tuple[str, str]]" = {} +# when py 3.15 drops, we might want to take a look at using +# a lazy-loading approach using the new `lazy` keyword. +# e.g. {"s3": _S3Extension} +_SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = {} -@lru_cache(maxsize=None) def _resolve_service( - service: "str", + service_name: "str", ) -> "Optional[_ServiceExtension]": - target = _SERVICE_EXTENSIONS.get(service) - if target is None: - return None - - with capture_internal_exceptions(): - module_name, class_name = target - extension_class = getattr(import_module(module_name), class_name) - candidate = extension_class() - if isinstance(candidate, _ServiceExtension): - return candidate - + if service_name in _SERVICE_EXTENSIONS: + return _SERVICE_EXTENSIONS[service_name] # preserve generic instrumentation when lookup fails. return None From a928e74d68aafa95f1619db68037aa19e870a09d Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 10:50:29 +0200 Subject: [PATCH 63/72] fix(boto3): make logic clearer --- sentry_sdk/integrations/boto3/_client.py | 6 +++++- sentry_sdk/integrations/boto3/_services/registry.py | 6 ++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index a947d1dc91..66ae7b5f9f 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -108,7 +108,11 @@ def sentry_patched_make_api_call( service_ext: "Optional[_ServiceExtension]" = None with capture_internal_exceptions(): # resolve service extension for service-specific enrichment. - service_ext = _resolve_service(ctx.service_name) + service_ext = ( + _resolve_service(ctx.service_name) + if ctx.service_name is not None + else None + ) span: "Optional[Union[Span, StreamedSpan]]" = None with capture_internal_exceptions(): diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index e8616d5d86..7cad0c64a9 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -8,14 +8,12 @@ # when py 3.15 drops, we might want to take a look at using # a lazy-loading approach using the new `lazy` keyword. -# e.g. {"s3": _S3Extension} +# e.g. {"s3": _S3Extension()} _SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = {} def _resolve_service( service_name: "str", ) -> "Optional[_ServiceExtension]": - if service_name in _SERVICE_EXTENSIONS: - return _SERVICE_EXTENSIONS[service_name] # preserve generic instrumentation when lookup fails. - return None + return _SERVICE_EXTENSIONS.get(service_name) From 2d85548b96cace67f0c007ecc2c50235c15d9c9e Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 13:25:27 +0200 Subject: [PATCH 64/72] ref(boto3): change comment's and simplify logic --- .../integrations/boto3/_services/base.py | 13 ++++++++----- .../integrations/boto3/_services/registry.py | 19 ++++++++++++++++--- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_services/base.py b/sentry_sdk/integrations/boto3/_services/base.py index 86cb8dfc8e..0d9cb1c24d 100644 --- a/sentry_sdk/integrations/boto3/_services/base.py +++ b/sentry_sdk/integrations/boto3/_services/base.py @@ -9,25 +9,28 @@ class _ServiceExtension: """ - Specialize generic botocore instrumentation for an AWS service. + Optional hooks for adding service-specific behavior to AWS client + span; non-overridden methods keep the generic instrumentation. + Services without a registered extension in `_SERVICE_EXTENSIONS` continue + to use the generic instrumentation. """ __slots__ = () def get_span_op(self, ctx: "AwsCallContext") -> "Optional[str]": - """Return an optional `sentry.op` override for the client span.""" + """Return an optional `sentry.op` override, or `None` to keep the default.""" return None def get_span_origin(self, ctx: "AwsCallContext") -> "Optional[str]": - """Return an optional `sentry.origin` override for the client span.""" + """Return an optional `sentry.origin` override, or `None` to keep the default.""" return None def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": - """Return service-specific attributes available before the request is made.""" + """Return request attributes to add before the AWS request is made.""" return {} def get_response_attributes( self, ctx: "AwsCallContext", response: "Any" ) -> "Attributes": - """Return service-specific attributes derived from the response.""" + """Return response attributes to add after the AWS request is made.""" return {} diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index 7cad0c64a9..4e7b868f54 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -1,3 +1,13 @@ +"""Registry for the optional service extensions. + +The registry maps botocore service names, such as ``s3``, to extension +classes. It is intentionally static: the number of extensions is small, and +loading service modules dynamically would add complexity for little benefit. + +Not every AWS service needs an extension. When a service is not in this map, +the caller receives ``None`` and keeps the generic instrumentation. +""" + from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -6,14 +16,17 @@ from sentry_sdk.integrations.boto3._services.base import _ServiceExtension +# add a ServiceExtension here when one is implemented. for example: +# _SERVICE_EXTENSIONS = {"s3": _S3Extension()} # when py 3.15 drops, we might want to take a look at using # a lazy-loading approach using the new `lazy` keyword. -# e.g. {"s3": _S3Extension()} _SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = {} def _resolve_service( - service_name: "str", + service_name: "Optional[str]", ) -> "Optional[_ServiceExtension]": - # preserve generic instrumentation when lookup fails. + """Return the extension for a service, or `None` for generic instrumentation.""" + if service_name is None: + return None return _SERVICE_EXTENSIONS.get(service_name) From 9d2bd9612308de049d32224283075f9ca43b7030 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 13:39:34 +0200 Subject: [PATCH 65/72] ref(boto3): remove defensive checks --- sentry_sdk/integrations/boto3/_client.py | 37 +++++++----------- sentry_sdk/integrations/boto3/_context.py | 47 ++++++++--------------- 2 files changed, 30 insertions(+), 54 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_client.py b/sentry_sdk/integrations/boto3/_client.py index 66ae7b5f9f..32015c802e 100644 --- a/sentry_sdk/integrations/boto3/_client.py +++ b/sentry_sdk/integrations/boto3/_client.py @@ -22,10 +22,9 @@ from sentry_sdk.utils import capture_internal_exceptions if TYPE_CHECKING: - from typing import Any, Iterator, Optional, Union + from typing import Any, Dict, Iterator, Optional, Union from sentry_sdk._types import Attributes - from sentry_sdk.integrations.boto3._services.base import _ServiceExtension from sentry_sdk.tracing import Span try: @@ -40,12 +39,12 @@ def _activate_client_span( span: "Union[Span, StreamedSpan]", ) -> "Iterator[Union[Span, StreamedSpan]]": """ - Activate the boto span temporarily during `_make_api_call()` without ending it. + Activate the client span temporarily during `_make_api_call()` without ending it. Botocore returns a `StreamingBody` before its bytes are consumed. Using the context manager would finish it as soon as `_make_api_call()` returns, so restore the caller's span here and let the `StreamingBody` wrapper finish - the boto span when body is consumed/closed. + the client span when the body is consumed or closed. faulty: desired: boto3 [_make_api_call] boto3 [_make_api_call------] @@ -80,13 +79,13 @@ def _patch_botocore_client() -> None: def sentry_patched_init(self: "BaseClient", *args: "Any", **kwargs: "Any") -> None: orig_init(self, *args, **kwargs) - meta = self.meta - meta.events.register("request-created", _sentry_request_created) - # run after other `before-sign` handlers so existing baggage is preserved. - meta.events.register_last("before-sign", _sentry_before_sign) + with capture_internal_exceptions(): + self.meta.events.register("request-created", _sentry_request_created) + # run after other `before-sign` handlers so existing baggage is preserved. + self.meta.events.register_last("before-sign", _sentry_before_sign) def sentry_patched_make_api_call( - self: "BaseClient", operation_name: str, api_params: "Any" + self: "BaseClient", operation_name: str, api_params: "Dict[str, Any]" ) -> "Any": """ Track a single API call, including retries, serialization, and endpoint @@ -99,23 +98,13 @@ def sentry_patched_make_api_call( if client.get_integration(IDENTIFIER) is None: return orig_make_api_call(self, operation_name, api_params) - ctx = AwsCallContext(operation_name, api_params) - - # add optional metadata to context. - with capture_internal_exceptions(): - ctx.add_metadata(self) - - service_ext: "Optional[_ServiceExtension]" = None - with capture_internal_exceptions(): - # resolve service extension for service-specific enrichment. - service_ext = ( - _resolve_service(ctx.service_name) - if ctx.service_name is not None - else None - ) - span: "Optional[Union[Span, StreamedSpan]]" = None with capture_internal_exceptions(): + ctx = AwsCallContext(operation_name, api_params) + with capture_internal_exceptions(): + # add optional metadata to the context, e.g. service-name, region-name, etc. + ctx.add_metadata(self) + service_ext = _resolve_service(ctx.service_name) span = _start_client_span(ctx, service_ext) if span is None: diff --git a/sentry_sdk/integrations/boto3/_context.py b/sentry_sdk/integrations/boto3/_context.py index 83867d20b4..4ab95eb366 100644 --- a/sentry_sdk/integrations/boto3/_context.py +++ b/sentry_sdk/integrations/boto3/_context.py @@ -23,40 +23,27 @@ class AwsCallContext: "params", ) - def __init__(self, operation_name: str, params: "Any") -> None: - self.operation_name: str = operation_name - self.params: "Dict[str, Any]" = {} + def __init__(self, operation_name: str, params: "Dict[str, Any]") -> None: + self.operation_name: "str" = operation_name + self.params: "Dict[str, Any]" = dict(params) self.service_name: "Optional[str]" = None self.service_id: "Optional[str]" = None self.service_id_hyphenized: "Optional[str]" = None self.region_name: "Optional[str]" = None self.endpoint_url: "Optional[str]" = None - if isinstance(params, dict): - with capture_internal_exceptions(): - self.params = dict(params) - def add_metadata(self, client: "BaseClient") -> None: - def _get_attr(obj: "Any", name: str) -> "Any": - if obj is None: - return None - - with capture_internal_exceptions(): - return getattr(obj, name) - - client_meta = _get_attr(client, "meta") - service_model = _get_attr(client_meta, "service_model") - - # botocore's internal identifier, e.g. `apigateway`. - self.service_name = _get_attr(service_model, "service_name") - - # modeled AWS service identity used in span names, e.g. `API Gateway`. - service_id = _get_attr(service_model, "service_id") - if service_id is not None: - with capture_internal_exceptions(): - self.service_id = str(service_id) - with capture_internal_exceptions(): - self.service_id_hyphenized = service_id.hyphenize() - - self.region_name = _get_attr(client_meta, "region_name") - self.endpoint_url = _get_attr(client_meta, "endpoint_url") + with capture_internal_exceptions(): + service_model = client.meta.service_model + # botocore's internal identifier, e.g. `apigateway`. + self.service_name = service_model.service_name + service_id = service_model.service_id + # modeled AWS service identity used in span names, e.g. `API Gateway`. + self.service_id = str(service_id) + self.service_id_hyphenized = service_id.hyphenize() + + with capture_internal_exceptions(): + self.region_name = client.meta.region_name + + with capture_internal_exceptions(): + self.endpoint_url = client.meta.endpoint_url From b65c0ca59fe0766403c4c54f743faa37509fa8c4 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 14:47:18 +0200 Subject: [PATCH 66/72] ref(boto3): remove defensive checks and cleanup comments --- .../integrations/boto3/_instrumentation.py | 84 +++++++------------ 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_instrumentation.py b/sentry_sdk/integrations/boto3/_instrumentation.py index c486bfb005..3c180a025c 100644 --- a/sentry_sdk/integrations/boto3/_instrumentation.py +++ b/sentry_sdk/integrations/boto3/_instrumentation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, cast from urllib.parse import urlsplit import sentry_sdk @@ -83,7 +83,7 @@ def _get_client_attributes( attributes: "Attributes" = {} # `rpc.service` is deprecated in OTel, but js still uses it. - if ctx.service_id: + if ctx.service_id is not None: attributes[SPANDATA.RPC_SERVICE] = ctx.service_id if ctx.region_name: @@ -100,7 +100,7 @@ def _get_response_attributes(response: "Mapping[str, Any]") -> "Attributes": # botocore injects HTTP status into `ResponseMetadata` after parsing. # https://github.com/boto/botocore/blob/358f8eec8c76201bb1a7a35644abcbc9036de7ed/botocore/parsers.py#L273-L284 status_code = metadata.get("HTTPStatusCode") - if isinstance(status_code, int) and 100 <= status_code <= 599: + if status_code is not None: attributes[SPANDATA.HTTP_STATUS_CODE] = status_code retry_attempts = metadata.get("RetryAttempts", 0) @@ -109,33 +109,19 @@ def _get_response_attributes(response: "Mapping[str, Any]") -> "Attributes": headers = metadata.get("HTTPHeaders", {}) - request_id = next( - ( - value - for value in ( - metadata.get("RequestId"), - headers.get("x-amzn-requestid"), - headers.get("x-amzn-request-id"), - headers.get("x-amz-request-id"), - ) - if isinstance(value, str) and value - ), - None, + request_id = ( + metadata.get("RequestId") + or headers.get("x-amzn-requestid") + or headers.get("x-amzn-request-id") + or headers.get("x-amz-request-id") ) - if request_id is not None: + if request_id: attributes[SPANDATA.AWS_REQUEST_ID] = request_id # S3's `HostId` is the extended request ID returned in `x-amz-id-2`. # https://docs.aws.amazon.com/AmazonS3/latest/developerguide/get-request-ids.html - extended_request_id = next( - ( - value - for value in (metadata.get("HostId"), headers.get("x-amz-id-2")) - if isinstance(value, str) and value - ), - None, - ) - if extended_request_id is not None: + extended_request_id = metadata.get("HostId") or headers.get("x-amz-id-2") + if extended_request_id: attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] = extended_request_id return attributes @@ -175,10 +161,8 @@ def _start_client_span( ) -> "Optional[Union[Span, StreamedSpan]]": client = sentry_sdk.get_client() - if client.get_integration(IDENTIFIER) is None: - return None - # use unknown if `service_id_hyphenized` is not set so span name can still be created. + # use "unknown" if `service_id_hyphenized` is not set so span name can still be created. # e.g. "aws.unknown.GetObject" service_name = ctx.service_id_hyphenized or "unknown" span_name = f"aws.{service_name}.{ctx.operation_name}" @@ -191,6 +175,9 @@ def _start_client_span( } with capture_internal_exceptions(): attributes.update(_get_client_attributes(ctx)) + + # `sentry.span_op` and `sentry.span_origin` are set to generic defaults; + # a service extension can override them with `get_span_op()` and `get_span_origin()`. span_op = OP.HTTP_CLIENT span_origin = ORIGIN @@ -236,7 +223,7 @@ def _start_client_span( with capture_internal_exceptions(): _set_span_attributes(span, attributes) with capture_internal_exceptions(): - if ctx.service_id_hyphenized: + if ctx.service_id_hyphenized is not None: span.set_tag("aws.service_id", ctx.service_id_hyphenized) span.set_tag("aws.operation_name", ctx.operation_name) return span @@ -273,10 +260,11 @@ def _instrument_streaming_body( if isinstance(span, StreamedSpan): streaming_span = sentry_sdk.traces.start_span( name=span.name, - # keep stream span under the boto span after `_make_api_call()` returns. + # keep the stream span under the client span after + # `_make_api_call()` returns. parent_span=span, # the body may outlive the api call, so keep it inactive. Otherwise it - # 1. could restore the already-finished boto span when it ends; 2. make + # 1. could restore the already-finished client span when it ends; 2. make # unrelated new spans attach to the stream span since it's the current span. active=False, attributes={ @@ -300,7 +288,8 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: return finished = True - # finish stream span before boto span, and only once across read/close. + # finish the stream span before the client span, and only once across + # read and close. if error is not None: with capture_internal_exceptions(): attributes = _get_error_attributes(error) @@ -311,13 +300,8 @@ def finish_span(error: "Optional[BaseException]" = None) -> None: _finish_span(span, error) def content_length_reached() -> bool: - content_length = getattr(body, "_content_length", None) - amount_read = getattr(body, "_amount_read", None) - return ( - content_length is not None - and amount_read is not None - and amount_read >= int(content_length) - ) + content_length = body._content_length # type: ignore[attr-defined] + return content_length is not None and body._amount_read >= int(content_length) # type: ignore[attr-defined] def sentry_streaming_body_read(*args: "Any", **kwargs: "Any") -> bytes: nonlocal read_in_progress @@ -381,17 +365,15 @@ def _set_request_attributes( client = sentry_sdk.get_client() parsed_url = None - if request.url is not None: - with capture_internal_exceptions(): - parsed_url = parse_url(request.url, sanitize=False) + with capture_internal_exceptions(): + parsed_url = parse_url(cast(str, request.url), sanitize=False) # overwrite server attributes when actual request URL is resolved. _set_span_attributes(span, _get_server_attributes(request.url)) if isinstance(span, StreamedSpan): span.set_attributes(get_url_attributes(client, parsed_url)) - if request.method is not None: - span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, request.method) + span.set_attribute(SPANDATA.HTTP_REQUEST_METHOD, cast(str, request.method)) return if parsed_url is not None: @@ -399,24 +381,21 @@ def _set_request_attributes( span.set_data(SPANDATA.HTTP_QUERY, parsed_url.query) span.set_data(SPANDATA.HTTP_FRAGMENT, parsed_url.fragment) - if request.method is not None: - span.set_data(SPANDATA.HTTP_METHOD, request.method) + span.set_data(SPANDATA.HTTP_METHOD, request.method) def _add_request_breadcrumb(request: "AWSRequest") -> None: client = sentry_sdk.get_client() parsed_url = None - if request.url is not None: - with capture_internal_exceptions(): - parsed_url = parse_url(request.url, sanitize=False) + with capture_internal_exceptions(): + parsed_url = parse_url(cast(str, request.url), sanitize=False) breadcrumb: "dict[str, Any]" = {} if has_span_streaming_enabled(client.options): breadcrumb.update(get_url_attributes(client, parsed_url)) - if request.method is not None: - breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method + breadcrumb[SPANDATA.HTTP_REQUEST_METHOD] = request.method else: if parsed_url is not None: breadcrumb.update( @@ -427,8 +406,7 @@ def _add_request_breadcrumb(request: "AWSRequest") -> None: } ) - if request.method is not None: - breadcrumb[SPANDATA.HTTP_METHOD] = request.method + breadcrumb[SPANDATA.HTTP_METHOD] = request.method add_http_breadcrumb(None, breadcrumb) From 1270b49e0c8f672440074d6f2c959eadc74b305b Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 15:25:21 +0200 Subject: [PATCH 67/72] cleanup tests --- tests/integrations/boto3/test_client.py | 336 +++++------------------- 1 file changed, 73 insertions(+), 263 deletions(-) diff --git a/tests/integrations/boto3/test_client.py b/tests/integrations/boto3/test_client.py index f11e064740..c309549316 100644 --- a/tests/integrations/boto3/test_client.py +++ b/tests/integrations/boto3/test_client.py @@ -12,16 +12,13 @@ import sentry_sdk from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations.boto3 import Boto3Integration -from sentry_sdk.integrations.boto3._instrumentation import ( - _get_response_attributes, - _instrument_streaming_body, -) from sentry_sdk.integrations.boto3._services.base import _ServiceExtension +from sentry_sdk.integrations.boto3._services.registry import _SERVICE_EXTENSIONS from sentry_sdk.integrations.boto3.consts import AWS_RPC_SYSTEM_NAME, ORIGIN from sentry_sdk.integrations.stdlib import StdlibIntegration from sentry_sdk.traces import StreamedSpan from sentry_sdk.tracing import Span -from tests.integrations.boto3.aws_mock import Body +from tests.integrations.boto3.aws_mock import Body, MockResponse session = boto3.Session( # type: ignore[attr-defined] aws_access_key_id="-", @@ -189,57 +186,6 @@ def record_client_span(request, **kwargs): assert span[end_timestamp] is not None -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_streaming_body_instrumentation_setup_failure_finishes_stream_span( - sentry_init, - capture_items, - span_streaming, -): - sentry_init( - traces_sample_rate=1.0, - trace_lifecycle="stream" if span_streaming else "static", - integrations=[Boto3Integration()], - server_name="", - ) - - class _RawStreamLookupFailingBody(StreamingBody): - @property - def _raw_stream(self): - raise RuntimeError("raw stream lookup failed") - - @_raw_stream.setter - def _raw_stream(self, raw_stream): - self._raw_stream_value = raw_stream - - body = _RawStreamLookupFailingBody(Body(b"x"), "1") - - def invoke(): - if not span_streaming: - with sentry_sdk.start_span( - name="client", op=OP.HTTP_CLIENT, origin=ORIGIN - ) as span: - with pytest.raises(RuntimeError, match="raw stream lookup failed"): - _instrument_streaming_body(span, {"Body": body}) - return - - span = sentry_sdk.traces.start_span( # type: ignore[attr-defined] - name="client", - attributes={ - SPANDATA.SENTRY_OP: OP.HTTP_CLIENT, - SPANDATA.SENTRY_ORIGIN: ORIGIN, - }, - active=False, - ) - with pytest.raises(RuntimeError, match="raw stream lookup failed"): - _instrument_streaming_body(span, {"Body": body}) - - spans_by_op = _capture_boto3_spans_by_op(invoke, capture_items, span_streaming) - stream_spans = spans_by_op.get(OP.HTTP_CLIENT_STREAM, []) - - assert len(stream_spans) == 1 - _assert_span_finished(stream_spans[0], span_streaming) - - def test_non_body_stream_does_not_delay_client_span(sentry_init, capture_items): sentry_init( traces_sample_rate=1.0, @@ -424,11 +370,7 @@ def get_response_attributes(self, ctx, response): SPANDATA.HTTP_STATUS_CODE: 418, } - extension = TestServiceExtension() - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._client._resolve_service", - lambda service_name: extension, - ) + monkeypatch.setitem(_SERVICE_EXTENSIONS, "s3", TestServiceExtension()) client = client_factory() api_params = {"Bucket": "bucket", "Key": "foo"} @@ -439,6 +381,7 @@ def get_response_attributes(self, ctx, response): "ResponseMetadata": { "HTTPStatusCode": 200, "RequestId": "request-id", + "HostId": "extended-request-id", } }, api_params, @@ -458,157 +401,14 @@ def get_response_attributes(self, ctx, response): assert attributes[SPANDATA.SENTRY_KIND] == "producer" assert attributes[SPANDATA.RPC_METHOD] == "HeadObject" assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" + _assert_span_finished(spans[0], span_streaming) if span_streaming: assert attributes[SPANDATA.SENTRY_ORIGIN] == "auto.aws.test" else: assert spans[0]["origin"] == "auto.aws.test" -@pytest.mark.parametrize("span_streaming", [True, False]) -def test_service_extension_enriches_client_error( - capture_items, - client_factory, - monkeypatch, - span_streaming, -): - class TestServiceExtension(_ServiceExtension): - def get_response_attributes(self, ctx, response): - return { - "aws.test.error": response["Error"]["Code"], - SPANDATA.ERROR_TYPE: "must-not-override", - SPANDATA.HTTP_STATUS_CODE: 418, - } - - monkeypatch.setattr( - "sentry_sdk.integrations.boto3._client._resolve_service", - lambda service_name: TestServiceExtension(), - ) - client = client_factory() - error = ClientError( - { - "Error": {"Code": "AccessDeniedException"}, - "ResponseMetadata": {"HTTPStatusCode": 403}, - }, - "HeadObject", - ) - - def raise_client_error(**kwargs): - raise error - - client.meta.events.register("before-parameter-build", raise_client_error) - - def invoke_failing_client_method(): - with pytest.raises(ClientError) as exc_info: - client.head_object(Bucket="bucket", Key="foo") - assert exc_info.value is error - - spans_by_op = _capture_boto3_spans_by_op( - invoke_failing_client_method, - capture_items, - span_streaming, - ) - spans = spans_by_op.get(OP.HTTP_CLIENT, []) - - _assert_one_failed_span(spans, span_streaming) - attributes = _span_attributes(spans[0], span_streaming) - assert attributes["aws.test.error"] == "AccessDeniedException" - assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" - assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 - - -@pytest.mark.parametrize( - ("response", "expected"), - [ - ({}, {}), - ( - { - "ResponseMetadata": { - "RequestId": "request-id", - "HostId": "extended-request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 0, - } - }, - { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - }, - ), - ( - { - "ResponseMetadata": { - "RequestId": "request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 2, - } - }, - { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, - }, - ), - ], -) -def test_get_response_attributes(response, expected): - assert _get_response_attributes(response) == expected - - -@pytest.mark.parametrize( - "header_name", - ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"], -) -def test_get_response_attributes_reads_request_id_header(header_name): - response = { - "ResponseMetadata": { - "HTTPHeaders": {header_name: "request-id"}, - } - } - - assert _get_response_attributes(response) == {SPANDATA.AWS_REQUEST_ID: "request-id"} - - -def test_get_response_attributes_reads_extended_request_id_header(): - response = { - "ResponseMetadata": { - "HTTPHeaders": {"x-amz-id-2": "extended-request-id"}, - } - } - - assert _get_response_attributes(response) == { - SPANDATA.AWS_EXTENDED_REQUEST_ID: "extended-request-id" - } - - -@pytest.mark.parametrize( - ("field", "value", "attribute"), - [ - ("RequestId", 123, SPANDATA.AWS_REQUEST_ID), - ("RequestId", "", SPANDATA.AWS_REQUEST_ID), - ("HTTPStatusCode", "200", SPANDATA.HTTP_STATUS_CODE), - ("HTTPStatusCode", True, SPANDATA.HTTP_STATUS_CODE), - ("HTTPStatusCode", 999, SPANDATA.HTTP_STATUS_CODE), - ], -) -def test_get_response_attributes_ignores_malformed_field(field, value, attribute): - metadata = { - "RequestId": "request-id", - "HTTPStatusCode": 200, - "RetryAttempts": 2, - } - metadata[field] = value - - attributes = _get_response_attributes({"ResponseMetadata": metadata}) - expected = { - SPANDATA.AWS_REQUEST_ID: "request-id", - SPANDATA.HTTP_STATUS_CODE: 200, - SPANDATA.HTTP_REQUEST_RESEND_COUNT: 2, - } - expected.pop(attribute) - assert attributes == expected - - @pytest.mark.parametrize( ( "service_name", @@ -668,6 +468,13 @@ def test_client_call_has_common_attributes( api_params, capture_items, span_streaming, + response={ + "ResponseMetadata": { + "HTTPStatusCode": 200, + "RequestId": "request-id", + "RetryAttempts": 0, + } + }, ) attributes = _span_attributes(span, span_streaming) @@ -679,6 +486,11 @@ def test_client_call_has_common_attributes( assert attributes[SPANDATA.CLOUD_REGION] == "eu-north-1" assert attributes[SPANDATA.SERVER_ADDRESS] == server_address assert attributes[SPANDATA.SERVER_PORT] == server_port + assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 + assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" + assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes + assert SPANDATA.ERROR_TYPE not in attributes + _assert_span_finished(span, span_streaming) def test_client_call_attributes_are_available_at_span_creation( @@ -705,10 +517,13 @@ def test_client_call_attributes_are_available_at_span_creation( client = session.client("s3") items = capture_items("span") - with Stubber(client) as stubber: - stubber.add_response("head_object", {}, {"Bucket": "bucket", "Key": "foo"}) - with sentry_sdk.traces.start_span(name="parent"): - client.head_object(Bucket="bucket", Key="foo") + with MockResponse(client, 200, {}, b""): + with sentry_sdk.traces.start_span(name="parent") as parent: + response = client.head_object(Bucket="bucket", Key="foo") + assert response["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sentry_sdk.traces.get_current_span() is parent + assert SPANDATA.RPC_METHOD not in parent.get_attributes() + assert SPANDATA.HTTP_REQUEST_METHOD not in parent.get_attributes() sentry_sdk.flush() client_spans = [ @@ -719,60 +534,29 @@ def test_client_call_attributes_are_available_at_span_creation( assert client_spans == [] -def test_client_call_omits_missing_region( - sentry_init, - capture_items, - monkeypatch, -): - sentry_init( - traces_sample_rate=1.0, - integrations=[Boto3Integration()], - trace_lifecycle="stream", - server_name="", - ) - client = session.client("s3") - monkeypatch.setattr(type(client.meta), "region_name", property(lambda _: None)) - - span = _capture_stubbed_client_span( - client, - "head_object", - {"Bucket": "bucket", "Key": "foo"}, - capture_items, - span_streaming=True, - ) - - assert SPANDATA.CLOUD_REGION not in span["attributes"] - - @pytest.mark.parametrize("span_streaming", [True, False]) -def test_client_call_has_response_attributes( - capture_items, - client_factory, - span_streaming, +@pytest.mark.parametrize( + "request_id_header", ["x-amzn-requestid", "x-amzn-request-id", "x-amz-request-id"] +) +def test_client_call_has_response_header_attributes( + capture_items, client_factory, span_streaming, request_id_header ): client = client_factory() - span = _capture_stubbed_client_span( - client, - "head_object", - {"Bucket": "bucket", "Key": "foo"}, - capture_items, - span_streaming, - response={ - "ResponseMetadata": { - "HTTPStatusCode": 200, - "RequestId": "request-id", - "HostId": "extended-request-id", - "RetryAttempts": 0, - } - }, - ) - attributes = _span_attributes(span, span_streaming) + headers = {request_id_header: "request-id", "x-amz-id-2": "extended-request-id"} + with MockResponse(client, 200, headers, b""): + spans_by_op = _capture_boto3_spans_by_op( + lambda: client.head_object(Bucket="bucket", Key="foo"), + capture_items, + span_streaming, + ) + spans = spans_by_op[OP.HTTP_CLIENT] + assert len(spans) == 1 + attributes = _span_attributes(spans[0], span_streaming) assert attributes[SPANDATA.HTTP_STATUS_CODE] == 200 assert attributes[SPANDATA.AWS_REQUEST_ID] == "request-id" assert attributes[SPANDATA.AWS_EXTENDED_REQUEST_ID] == "extended-request-id" assert SPANDATA.HTTP_REQUEST_RESEND_COUNT not in attributes - assert SPANDATA.ERROR_TYPE not in attributes @pytest.mark.parametrize("span_streaming", [True, False]) @@ -827,11 +611,24 @@ def attempt_failed_head_object_call(): @pytest.mark.parametrize("span_streaming", [True, False]) +@pytest.mark.parametrize("with_service_extension", [False, True]) def test_client_error_has_response_attributes_and_is_unchanged( capture_items, client_factory, + monkeypatch, span_streaming, + with_service_extension, ): + class TestServiceExtension(_ServiceExtension): + def get_response_attributes(self, ctx, response): + return { + "aws.test.error": response["Error"]["Code"], + SPANDATA.ERROR_TYPE: "must-not-override", + SPANDATA.HTTP_STATUS_CODE: 418, + } + + if with_service_extension: + monkeypatch.setitem(_SERVICE_EXTENSIONS, "s3", TestServiceExtension()) client = client_factory() original_exception = ClientError( { @@ -869,6 +666,8 @@ def invoke_failing_client_method(): assert attributes[SPANDATA.HTTP_STATUS_CODE] == 403 assert attributes[SPANDATA.HTTP_REQUEST_RESEND_COUNT] == 1 assert attributes[SPANDATA.ERROR_TYPE] == "AccessDeniedException" + if with_service_extension: + assert attributes["aws.test.error"] == "AccessDeniedException" assert "Error.Message" not in attributes assert "exception.message" not in attributes assert "error.message" not in attributes @@ -923,23 +722,31 @@ def invoke_failing_client_method(): @pytest.mark.tests_internal_exceptions @pytest.mark.parametrize("span_streaming", [True, False]) -def test_response_attribute_extraction_failure_does_not_change_response( +@pytest.mark.parametrize( + "failing_instrumentation", + [ + "_start_client_span", + "_get_response_attributes", + ], +) +def test_instrumentation_failure_does_not_change_response( capture_items, client_factory, monkeypatch, span_streaming, + failing_instrumentation, ): client = client_factory() api_params = {"Bucket": "bucket", "Key": "foo"} original_response = {"ResponseMetadata": {"HTTPStatusCode": 200}} returned_responses = [] - def fail_attribute_extraction(response): - raise RuntimeError("attribute extraction failed") + def fail_instrumentation(*args, **kwargs): + raise RuntimeError("instrumentation failed") monkeypatch.setattr( - "sentry_sdk.integrations.boto3._client._get_response_attributes", - fail_attribute_extraction, + f"sentry_sdk.integrations.boto3._client.{failing_instrumentation}", + fail_instrumentation, ) def invoke_client_method(): @@ -954,8 +761,11 @@ def invoke_client_method(): client_spans = spans_by_op.get(OP.HTTP_CLIENT, []) assert returned_responses == [original_response] assert returned_responses[0] is original_response - assert len(client_spans) == 1 - _assert_span_finished(client_spans[0], span_streaming) + if failing_instrumentation == "_get_response_attributes": + assert len(client_spans) == 1 + _assert_span_finished(client_spans[0], span_streaming) + else: + assert client_spans == [] @pytest.mark.tests_internal_exceptions From 91c9bdb7e0e7144bb6c6c913c2eb8e77f2adba80 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 15:36:47 +0200 Subject: [PATCH 68/72] remove AI comment --- sentry_sdk/integrations/boto3/_services/registry.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index 4e7b868f54..b387db0f26 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -1,13 +1,3 @@ -"""Registry for the optional service extensions. - -The registry maps botocore service names, such as ``s3``, to extension -classes. It is intentionally static: the number of extensions is small, and -loading service modules dynamically would add complexity for little benefit. - -Not every AWS service needs an extension. When a service is not in this map, -the caller receives ``None`` and keeps the generic instrumentation. -""" - from typing import TYPE_CHECKING if TYPE_CHECKING: From f2a2bc6c0175dfb361f508b359429b111e636d9a Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 16:39:47 +0200 Subject: [PATCH 69/72] feat(attributes): Add s3 relevant attributes to `SPANDATA` --- sentry_sdk/consts.py | 48 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/sentry_sdk/consts.py b/sentry_sdk/consts.py index 5b037a66d0..3b8a1a6855 100644 --- a/sentry_sdk/consts.py +++ b/sentry_sdk/consts.py @@ -402,6 +402,48 @@ class SPANDATA: Example: "79b9da39-b7ae-508a-a6bc-864b2829c622" """ + AWS_S3_BUCKET = "aws.s3.bucket" + """ + The S3 bucket name the request refers to. + Example: "ot-demo-test" + """ + + AWS_S3_COPY_SOURCE = "aws.s3.copy_source" + """ + The source object (in the form bucket/key) for the copy operation. + Example: "someFile.yml" + """ + + AWS_S3_DELETE = "aws.s3.delete" + """ + The delete request container that specifies the objects to be deleted. + Example: "Objects=[{Key=string,VersionId=string},{Key=string,VersionId=string}],Quiet=boolean" + """ + + AWS_S3_KEY = "aws.s3.key" + """ + The S3 object key the request refers to. Corresponds to the --key parameter of the S3 API operations. + Example: "someFile.yml" + """ + + AWS_S3_OBJECT_SIZE = "aws.s3.object_size" + """ + The size of the S3 object in bytes. + Example: 434234 + """ + + AWS_S3_PART_NUMBER = "aws.s3.part_number" + """ + The part number of the part being uploaded in a multipart-upload operation. This is a positive integer between 1 and 10,000. + Example: 3456 + """ + + AWS_S3_UPLOAD_ID = "aws.s3.upload_id" + """ + Upload ID that identifies the multipart upload. + Example: "dfRtDYWFbkRONycy.Yxwh66Yjlx.cph0gtNBtJ" + """ + CACHE_HIT = "cache.hit" """ A boolean indicating whether the requested data was found in the cache. @@ -922,6 +964,12 @@ class SPANDATA: Example: ?foo=bar&bar=baz """ + HTTP_RESPONSE_BODY_SIZE = "http.response.body.size" + """ + The encoded body size of the response (in bytes). + Example: 123 + """ + HTTP_STATUS_CODE = "http.response.status_code" """ The HTTP status code as an integer. From 0e948cb093fea5f1ceb46cacd67537ad69f51843 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Wed, 30 Sep 2026 11:20:38 +0200 Subject: [PATCH 70/72] feat(boto3): Register s3 to `_SERVICE_EXTENSIONS` --- sentry_sdk/integrations/boto3/_services/registry.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/boto3/_services/registry.py b/sentry_sdk/integrations/boto3/_services/registry.py index b387db0f26..991da042d9 100644 --- a/sentry_sdk/integrations/boto3/_services/registry.py +++ b/sentry_sdk/integrations/boto3/_services/registry.py @@ -1,5 +1,7 @@ from typing import TYPE_CHECKING +from sentry_sdk.integrations.boto3._services.s3 import _S3Extension + if TYPE_CHECKING: from typing import Dict, Optional @@ -10,7 +12,10 @@ # _SERVICE_EXTENSIONS = {"s3": _S3Extension()} # when py 3.15 drops, we might want to take a look at using # a lazy-loading approach using the new `lazy` keyword. -_SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = {} +# e.g. {"s3": _S3Extension()} +_SERVICE_EXTENSIONS: "Dict[str, _ServiceExtension]" = { + "s3": _S3Extension(), +} def _resolve_service( From a3629db22c1c612c9375a3eb41a2927a36a9a303 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 16:47:26 +0200 Subject: [PATCH 71/72] feat(boto3): Add attribute extraction helpers --- .../boto3/_services/_attribute_extraction.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 sentry_sdk/integrations/boto3/_services/_attribute_extraction.py diff --git a/sentry_sdk/integrations/boto3/_services/_attribute_extraction.py b/sentry_sdk/integrations/boto3/_services/_attribute_extraction.py new file mode 100644 index 0000000000..ff57dc5065 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/_attribute_extraction.py @@ -0,0 +1,36 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable, Optional, Sequence, Tuple + + from sentry_sdk._types import Attributes + + _Converter = Callable[[Any], Optional[Any]] + # e.g. ("Limit", "aws.dynamodb.limit", _as_integer) converts + # {"Limit": 10} into {"aws.dynamodb.limit": 10} using `_extract_attributes()` + _AttributeSpec = Tuple[str, str, _Converter] + + +def _as_integer(value: "Any") -> "Optional[int]": + if isinstance(value, int) and not isinstance(value, bool): + return value + return None + + +def _as_string(value: "Any") -> "Optional[str]": + return value if isinstance(value, str) and value else None + + +def _extract_attributes( + source: "Any", specs: "Sequence[_AttributeSpec]" +) -> "Attributes": + if not isinstance(source, dict): + return {} + + attributes = {} + for param, attribute, convert in specs: + value = convert(source.get(param)) + # an unexpected type results in that attribute being omitted. + if value is not None: + attributes[attribute] = value + return attributes From 7c39a6c507c5a6bcd851101885cb64fd86a0c553 Mon Sep 17 00:00:00 2001 From: Pablo Deputter Date: Tue, 22 Sep 2026 16:47:36 +0200 Subject: [PATCH 72/72] feat(boto3): Add S3 extensio --- sentry_sdk/integrations/boto3/_services/s3.py | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 sentry_sdk/integrations/boto3/_services/s3.py diff --git a/sentry_sdk/integrations/boto3/_services/s3.py b/sentry_sdk/integrations/boto3/_services/s3.py new file mode 100644 index 0000000000..ed4f201d30 --- /dev/null +++ b/sentry_sdk/integrations/boto3/_services/s3.py @@ -0,0 +1,109 @@ +import json +from typing import TYPE_CHECKING + +from sentry_sdk.consts import SPANDATA +from sentry_sdk.integrations.boto3._services._attribute_extraction import ( + _as_integer, + _as_string, + _extract_attributes, +) +from sentry_sdk.integrations.boto3._services.base import _ServiceExtension + +if TYPE_CHECKING: + from typing import Any, Optional, Sequence + + from sentry_sdk._types import Attributes + from sentry_sdk.integrations.boto3._context import AwsCallContext + from sentry_sdk.integrations.boto3._services._attribute_extraction import ( + _AttributeSpec, + ) + +_RESPONSE_BODY_SIZE_OPERATIONS = frozenset( + ( + "GetObject", + "GetObjectAnnotation", + ) +) + +_RESPONSE_OBJECT_SIZE_FIELDS = { + "GetObjectAttributes": "ObjectSize", + "PutObject": "Size", +} + + +def _json_dict(value: "Any") -> "Optional[str]": + if not isinstance(value, dict): + return None + + try: + return json.dumps( + value, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + except (TypeError, ValueError): + return None + + +_REQUEST_ATTRIBUTES: "Sequence[_AttributeSpec]" = ( + # s3-specific attributes defined by OTel SemConv. Specified as a tuple of + # (param_name, attribute_name, converter_func). the `converter_func` is + # used to 1. validate the value (otherwise omitted) and 2. convert it to + # the appropriate type. + # https://opentelemetry.io/docs/specs/semconv/object-stores/s3/ + ("Bucket", SPANDATA.AWS_S3_BUCKET, _as_string), + ("CopySource", SPANDATA.AWS_S3_COPY_SOURCE, _as_string), + ("Delete", SPANDATA.AWS_S3_DELETE, _json_dict), + ("Key", SPANDATA.AWS_S3_KEY, _as_string), + ("PartNumber", SPANDATA.AWS_S3_PART_NUMBER, _as_integer), + ("UploadId", SPANDATA.AWS_S3_UPLOAD_ID, _as_string), +) + + +class _S3Extension(_ServiceExtension): + __slots__ = () + + def get_request_attributes(self, ctx: "AwsCallContext") -> "Attributes": + attributes: "Attributes" = _extract_attributes(ctx.params, _REQUEST_ATTRIBUTES) + + if ctx.operation_name == "CompleteMultipartUpload": + object_size = _as_integer(ctx.params.get("MpuObjectSize")) + if object_size is not None and object_size >= 0: + attributes[SPANDATA.AWS_S3_OBJECT_SIZE] = object_size + + return attributes + + def get_response_attributes( + self, ctx: "AwsCallContext", response: "Any" + ) -> "Attributes": + if not isinstance(response, dict): + return {} + + attributes: "Attributes" = {} + operation_name = ctx.operation_name + + if operation_name in _RESPONSE_BODY_SIZE_OPERATIONS: + # `ContentLength` is the size of the HTTP body returned, which may be a range. + content_length = _as_integer(response.get("ContentLength")) + if content_length is not None and content_length >= 0: + attributes[SPANDATA.HTTP_RESPONSE_BODY_SIZE] = content_length + + # these fields report the total S3 object size, not the HTTP body size. + object_size_field = _RESPONSE_OBJECT_SIZE_FIELDS.get(operation_name) + if object_size_field is not None: + object_size = _as_integer(response.get(object_size_field)) + if object_size is not None and object_size >= 0: + attributes[SPANDATA.AWS_S3_OBJECT_SIZE] = object_size + + if ( + operation_name == "HeadObject" + and "Range" not in ctx.params + and "PartNumber" not in ctx.params + ): + # an un-ranged `HEAD` has no body, so `ContentLength` is the object size. + object_size = _as_integer(response.get("ContentLength")) + if object_size is not None and object_size >= 0: + attributes[SPANDATA.AWS_S3_OBJECT_SIZE] = object_size + + return attributes