Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
antonpirker merged 9 commits into from
Mar 12, 2024
Merged

Discard open spans after 10 minutes #2801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
99b1ad0
Discard open spans after 10 minutes
antonpirker Mar 12, 2024
7239adf
Merge branch 'master' into antonpirker/otel-memory-leak
antonpirker Mar 12, 2024
f058dfb
Apply suggestions from code review
antonpirker Mar 12, 2024
da48aa1
Made function private
antonpirker Mar 12, 2024
4ee9535
Clarifying comments
antonpirker Mar 12, 2024
d426e1a
Update sentry_sdk/integrations/opentelemetry/span_processor.py
antonpirker Mar 12, 2024
b5cc262
Fixed prune
antonpirker Mar 12, 2024
abc865c
Merge branch 'master' into antonpirker/otel-memory-leak
antonpirker Mar 12, 2024
2aa4247
Linting
antonpirker Mar 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 47 additions & 3 deletions sentry_sdk/integrations/opentelemetry/span_processor.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
# The mapping from otel span ids to sentry spans
otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

# The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
open_spans = {} # type: dict[int, set[str]]

def __new__(cls):
# type: () -> SentrySpanProcessor
if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
# type: (Event, Hint) -> Event
return link_trace_context_to_error_event(event, self.otel_span_map)

def _prune_old_spans(self):
# type: (SentrySpanProcessor) -> None
"""
Prune spans that have been open for too long.
"""
current_time_minutes = int(time() / 60)
for span_start_minutes in list(
self.open_spans.keys()
): # making a list because we change the dict
# prune empty open spans buckets
if self.open_spans[span_start_minutes] == set():
self.open_spans.pop(span_start_minutes)

# prune old buckets
elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
for span_id in self.open_spans.pop(span_start_minutes):
self.otel_span_map.pop(span_id, None)

def on_start(self, otel_span, parent_context=None):
# type: (OTelSpan, Optional[SpanContext]) -> None
hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
sentry_span = sentry_parent_span.start_child(
span_id=trace_data["span_id"],
description=otel_span.name,
start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
start_timestamp=utc_from_timestamp(
otel_span.start_time / 1e9
), # OTel spans have nanosecond precision
instrumenter=INSTRUMENTER.OTEL,
)
else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
parent_span_id=parent_span_id,
trace_id=trace_data["trace_id"],
baggage=trace_data["baggage"],
start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
start_timestamp=utc_from_timestamp(
otel_span.start_time / 1e9
), # OTel spans have nanosecond precision
instrumenter=INSTRUMENTER.OTEL,
)

self.otel_span_map[trace_data["span_id"]] = sentry_span

span_start_in_minutes = int(
otel_span.start_time / 1e9 / 60
) # OTel spans have nanosecond precision
self.open_spans.setdefault(span_start_in_minutes, set()).add(
trace_data["span_id"]
)
self._prune_old_spans()

def on_end(self, otel_span):
# type: (OTelSpan) -> None
hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
else:
self._update_span_with_otel_data(sentry_span, otel_span)

sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
sentry_span.finish(
end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
) # OTel spans have nanosecond precision

span_start_in_minutes = int(
otel_span.start_time / 1e9 / 60
) # OTel spans have nanosecond precision
self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
self._prune_old_spans()

def _is_sentry_span(self, hub, otel_span):
# type: (Hub, OTelSpan) -> bool
Expand Down
92 changes: 92 additions & 0 deletions tests/integrations/opentelemetry/test_span_processor.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
assert "contexts" in event
assert "trace" in event["contexts"]
assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
otel_span = MagicMock()
otel_span.name = "Sample OTel Span"
otel_span.start_time = time.time_ns()
span_context = SpanContext(
trace_id=int("1234567890abcdef1234567890abcdef", 16),
span_id=int("1234567890abcdef", 16),
is_remote=True,
)
otel_span.get_span_context.return_value = span_context
otel_span.parent = MagicMock()
otel_span.parent.span_id = int("abcdef1234567890", 16)

parent_context = {}
fake_client = MagicMock()
fake_client.options = {"instrumenter": "otel"}
fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

current_hub = MagicMock()
current_hub.client = fake_client

fake_hub = MagicMock()
fake_hub.current = current_hub

with mock.patch(
"sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
):
span_processor = SentrySpanProcessor()

span_processor.otel_span_map = {
"111111111abcdef": MagicMock(), # should stay
"2222222222abcdef": MagicMock(), # should go
"3333333333abcdef": MagicMock(), # should go
}
current_time_minutes = int(time.time() / 60)
span_processor.open_spans = {
current_time_minutes - 3: {"111111111abcdef"}, # should stay
current_time_minutes
- 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
}

span_processor.on_start(otel_span, parent_context)
assert sorted(list(span_processor.otel_span_map.keys())) == [
"111111111abcdef",
"1234567890abcdef",
]
assert sorted(list(span_processor.open_spans.values())) == [
{"111111111abcdef"},
{"1234567890abcdef"},
]
Comment thread
antonpirker marked this conversation as resolved.


def test_pruning_old_spans_on_end():
otel_span = MagicMock()
otel_span.name = "Sample OTel Span"
otel_span.start_time = time.time_ns()
span_context = SpanContext(
trace_id=int("1234567890abcdef1234567890abcdef", 16),
span_id=int("1234567890abcdef", 16),
is_remote=True,
)
otel_span.get_span_context.return_value = span_context
otel_span.parent = MagicMock()
otel_span.parent.span_id = int("abcdef1234567890", 16)

fake_sentry_span = MagicMock(spec=Span)
fake_sentry_span.set_context = MagicMock()
fake_sentry_span.finish = MagicMock()

span_processor = SentrySpanProcessor()
span_processor._get_otel_context = MagicMock()
span_processor._update_span_with_otel_data = MagicMock()

span_processor.otel_span_map = {
"111111111abcdef": MagicMock(), # should stay
"2222222222abcdef": MagicMock(), # should go
"3333333333abcdef": MagicMock(), # should go
"1234567890abcdef": fake_sentry_span, # should go (because it is closed)
}
current_time_minutes = int(time.time() / 60)
span_processor.open_spans = {
current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
current_time_minutes - 3: {"111111111abcdef"}, # should stay
current_time_minutes
- 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
}

span_processor.on_end(otel_span)
assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.
, 'i'); if (__m === '*' || __re.test(location.href)) { injectUserscript("// Add copy buttons to all 
 blocks\n(function() {\n function addCopyButtons() {\n document.querySelectorAll('pre code').forEach(function(codeBlock) {\n if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;\n codeBlock.parentElement.setAttribute('data-copy-added', 'true');\n \n var btn = document.createElement('button');\n btn.textContent = 'Copy';\n btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';\n btn.onmouseover = function() { this.style.opacity = '1'; };\n btn.onmouseout = function() { this.style.opacity = '0.7'; };\n btn.onclick = function() {\n navigator.clipboard.writeText(codeBlock.textContent).then(function() {\n btn.textContent = 'Copied!';\n setTimeout(function() { btn.textContent = 'Copy'; }, 1500);\n });\n };\n codeBlock.parentElement.style.position = 'relative';\n codeBlock.parentElement.appendChild(btn);\n });\n }\n \n addCopyButtons();\n \n // Re-run on dynamic content\n var observer = new MutationObserver(addCopyButtons);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Add Copy Buttons to Code Blocks");
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Force GitHub README to respect dark mode\n(function() {\n var style = document.createElement('style');\n style.textContent = '\n .markdown-body {\n color-scheme: dark light;\n }\n .markdown-body pre { background: #161b22 !important; }\n .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; }\n .markdown-body table th, .markdown-body table td { border-color: #30363d !important; }\n .markdown-body img { background: #0d1117; }\n .markdown-body blockquote { border-left-color: #8b949e; }\n .markdown-body hr { border-color: #30363d; }\n ';\n document.head.appendChild(style);\n})();", "GitHub Dark Mode README Fix");
}
} catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Highlight search terms from Google/DuckDuckGo/Bing referrer\n(function() {\n var ref = document.referrer;\n var terms = [];\n \n if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) {\n var url = new URL(ref);\n var q = url.searchParams.get('q') || url.searchParams.get('p');\n if (q) {\n terms = q.split(/\\s+/).filter(function(t) { return t.length > 2; });\n }\n }\n \n if (terms.length === 0) return;\n \n var style = document.createElement('style');\n style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }';\n document.head.appendChild(style);\n \n function highlight(node) {\n if (node.nodeType === 3) { // text node\n var text = node.textContent;\n var found = false;\n terms.forEach(function(term) {\n var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\') + ')', 'gi');\n if (regex.test(text)) {\n found = true;\n var frag = document.createDocumentFragment();\n var parts = text.split(regex);\n parts.forEach(function(part, i) {\n if (i % 2 === 0) {\n frag.appendChild(document.createTextNode(part));\n } else {\n var span = document.createElement('span');\n span.className = 'userscript-highlight';\n span.textContent = part;\n frag.appendChild(span);\n }\n });\n node.parentNode.replaceChild(frag, node);\n }\n });\n } else if (node.nodeType === 1 && node.childNodes) { // element\n var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT'];\n if (!skipTags.includes(node.tagName)) {\n Array.from(node.childNodes).forEach(highlight);\n }\n }\n }\n \n highlight(document.body);\n \n // Re-highlight on dynamic content\n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1 || node.nodeType === 3) highlight(node);\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Highlight Search Terms");
}
} catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Strip utm_, fbclid, gclid, etc. from all links on page\n(function() {\n var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content',\n 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid',\n 'ref', 'ref_src', 'source', 'medium', 'campaign'];\n \n function cleanUrl(url) {\n try {\n var u = new URL(url, window.location.origin);\n var changed = false;\n trackingParams.forEach(function(p) {\n if (u.searchParams.has(p)) {\n u.searchParams.delete(p);\n changed = true;\n }\n });\n return changed ? u.toString() : url;\n } catch (e) {\n return url;\n }\n }\n \n function cleanLinks() {\n document.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n \n cleanLinks();\n \n var observer = new MutationObserver(function(mutations) {\n mutations.forEach(function(m) {\n m.addedNodes.forEach(function(node) {\n if (node.nodeType === 1) {\n if (node.tagName === 'A') cleanLinks();\n node.querySelectorAll('a[href]').forEach(function(a) {\n var clean = cleanUrl(a.href);\n if (clean !== a.href) a.href = clean;\n });\n }\n });\n });\n });\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "Remove Tracking Parameters from Links");
}
} catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); }
})();
(function(){
try {
var __m = "youtube.com";
var __re = new RegExp('^' + "youtube\\.com" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Auto-enable theater mode on YouTube\n(function() {\n function tryTheater() {\n var btn = document.querySelector('button[aria-label=\"Theater mode\"], ytd-player #player button[title=\"Theater mode\"]');\n if (btn && !btn.classList.contains('activated')) {\n btn.click();\n }\n }\n \n // Try immediately\n tryTheater();\n \n // Try after navigation (SPA)\n var lastUrl = location.href;\n setInterval(function() {\n if (location.href !== lastUrl) {\n lastUrl = location.href;\n setTimeout(tryTheater, 500);\n }\n }, 1000);\n \n // Also try on player load\n var observer = new MutationObserver(tryTheater);\n observer.observe(document.body, { childList: true, subtree: true });\n})();", "YouTube Theater Mode Default");
}
} catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Remove or un-stick sticky/fixed headers that block content\n(function() {\n function unstick() {\n document.querySelectorAll('header, nav, [role=\"banner\"], .header, .navbar, .sticky, .fixed-top, [style*=\"position: fixed\"], [style*=\"position:sticky\"]').forEach(function(el) {\n if (el.style.position === 'fixed' || el.style.position === 'sticky' || \n getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') {\n el.style.position = 'static';\n el.style.top = 'auto';\n el.style.zIndex = 'auto';\n }\n });\n }\n \n unstick();\n \n var observer = new MutationObserver(unstick);\n observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] });\n})();", "Kill Sticky Headers");
}
} catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); }
})();
(function(){
try {
var __m = "*";
var __re = new RegExp('^' + ".*" + '

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.


, 'i');
if (__m === '*' || __re.test(location.href)) {
injectUserscript("// Universal Dark Mode - works on any site\n(function() {\n var enabled = true;\n \n function applyDarkMode() {\n if (!enabled) return;\n \n // Create style element if it doesn't exist\n var style = document.getElementById('universal-dark-mode-style');\n if (!style) {\n style = document.createElement('style');\n style.id = 'universal-dark-mode-style';\n document.head.appendChild(style);\n }\n \n // Dark mode CSS - inverts colors but preserves images/video\n style.textContent = '\n /* Invert everything except media */\n html {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #1a1a2e !important;\n }\n \n /* Restore images, videos, iframes, canvas */\n img, video, iframe, canvas, svg, picture, [style*=\"background-image\"] {\n filter: invert(1) hue-rotate(180deg) !important;\n }\n \n /* Preserve specific elements that should not be inverted */\n .no-dark-mode, .no-dark-mode *,\n [data-theme=\"light\"], [data-theme=\"light\"],\n .ace_editor, .ace_editor *,\n .CodeMirror, .CodeMirror *,\n .monaco-editor, .monaco-editor *,\n .markdown-body pre, .markdown-body pre *,\n .highlight, .highlight *,\n pre code, pre code * {\n filter: none !important;\n }\n \n /* Fix common UI elements */\n .modal, .popup, .dropdown-menu, .tooltip, .popover {\n filter: invert(1) hue-rotate(180deg) !important;\n background: #2d2d44 !important;\n border-color: #444 !important;\n }\n \n /* Scrollbars */\n ::-webkit-scrollbar { background: #1a1a2e !important; }\n ::-webkit-scrollbar-thumb { background: #444 !important; }\n ::-webkit-scrollbar-thumb:hover { background: #555 !important; }\n \n /* Selection */\n ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; }\n ';\n }\n \n function removeDarkMode() {\n var style = document.getElementById('universal-dark-mode-style');\n if (style) style.remove();\n }\n \n // Toggle with Alt+Shift+D\n document.addEventListener('keydown', function(e) {\n if (e.altKey && e.shiftKey && e.key === 'D') {\n e.preventDefault();\n enabled = !enabled;\n if (enabled) {\n applyDarkMode();\n console.log('[Universal Dark Mode] Enabled');\n } else {\n removeDarkMode();\n console.log('[Universal Dark Mode] Disabled');\n }\n }\n });\n \n // Apply on load\n applyDarkMode();\n \n // Re-apply on dynamic content\n var observer = new MutationObserver(function(mutations) {\n if (enabled && !document.getElementById('universal-dark-mode-style')) {\n applyDarkMode();\n }\n });\n observer.observe(document.head, { childList: true });\n \n console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle');\n})();", "Universal Dark Mode");
}
} catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); }
})();
})();

Skip to content
Discard open spans after 10 minutes#2801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

 Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and
privacy statement. We’ll occasionally send you account related emails.

Already on GitHub?
Sign in
to your account

 Merged

antonpirker
merged 9 commits into

from

Mar 12, 2024
 Merged


Discard open spans after 10 minutes
#2801
Show file tree
Hide file tree
 Changes from all commits
Commits

Show all changes


9 commits


Select commit
Hold shift + click to select a range
99b1ad0

Discard open spans after 10 minutes


antonpirker Mar 12, 2024
7239adf

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
f058dfb

Apply suggestions from code review


antonpirker Mar 12, 2024
da48aa1

Made function private


antonpirker Mar 12, 2024
4ee9535

Clarifying comments


antonpirker Mar 12, 2024
d426e1a

Update sentry_sdk/integrations/opentelemetry/span_processor.py


antonpirker Mar 12, 2024
b5cc262

Fixed prune


antonpirker Mar 12, 2024
abc865c

Merge branch 'master' into antonpirker/otel-memory-leak


antonpirker Mar 12, 2024
2aa4247

Linting


antonpirker Mar 12, 2024

File filter

Filter by extension
Filter by extension
Conversations

Failed to load comments. 
Loading
Jump to

Jump to file


Failed to load files. 
Loading
 Diff view

 Diff view


50 changes: 47 additions & 3 deletions
sentry_sdk/integrations/opentelemetry/span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
from time import time

from opentelemetry.context import get_value # type: ignore
from opentelemetry.sdk.trace import SpanProcessor # type: ignore
from opentelemetry.semconv.trace import SpanAttributes # type: ignore
Expand DownExpand Up@@ -33,6 +35,7 @@
 from sentry_sdk._types import Event, Hint

OPEN_TELEMETRY_CONTEXT = "otel"
SPAN_MAX_TIME_OPEN_MINUTES = 10


def link_trace_context_to_error_event(event, otel_span_map):
Expand DownExpand Up@@ -76,6 +79,9 @@ class SentrySpanProcessor(SpanProcessor): # type: ignore
 # The mapping from otel span ids to sentry spans
 otel_span_map = {} # type: Dict[str, Union[Transaction, SentrySpan]]

 # The currently open spans. Elements will be discarded after SPAN_MAX_TIME_OPEN_MINUTES
 open_spans = {} # type: dict[int, set[str]]

 def __new__(cls):
 # type: () -> SentrySpanProcessor
 if not hasattr(cls, "instance"):
Expand All@@ -90,6 +96,24 @@ def global_event_processor(event, hint):
 # type: (Event, Hint) -> Event
 return link_trace_context_to_error_event(event, self.otel_span_map)

 def _prune_old_spans(self):
 # type: (SentrySpanProcessor) -> None
 """
 Prune spans that have been open for too long.
 """
 current_time_minutes = int(time() / 60)
 for span_start_minutes in list(
 self.open_spans.keys()
 ): # making a list because we change the dict
 # prune empty open spans buckets
 if self.open_spans[span_start_minutes] == set():
 self.open_spans.pop(span_start_minutes)

 # prune old buckets
 elif current_time_minutes - span_start_minutes > SPAN_MAX_TIME_OPEN_MINUTES:
 for span_id in self.open_spans.pop(span_start_minutes):
 self.otel_span_map.pop(span_id, None)

 def on_start(self, otel_span, parent_context=None):
 # type: (OTelSpan, Optional[SpanContext]) -> None
 hub = Hub.current
Expand DownExpand Up@@ -125,7 +149,9 @@ def on_start(self, otel_span, parent_context=None):
 sentry_span = sentry_parent_span.start_child(
 span_id=trace_data["span_id"],
 description=otel_span.name,
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )
 else:
Expand All@@ -135,12 +161,22 @@ def on_start(self, otel_span, parent_context=None):
 parent_span_id=parent_span_id,
 trace_id=trace_data["trace_id"],
 baggage=trace_data["baggage"],
 start_timestamp=utc_from_timestamp(otel_span.start_time / 1e9),
 start_timestamp=utc_from_timestamp(
 otel_span.start_time / 1e9
 ), # OTel spans have nanosecond precision
 instrumenter=INSTRUMENTER.OTEL,
 )

 self.otel_span_map[trace_data["span_id"]] = sentry_span

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).add(
 trace_data["span_id"]
 )
 self._prune_old_spans()

 def on_end(self, otel_span):
 # type: (OTelSpan) -> None
 hub = Hub.current
Expand DownExpand Up@@ -173,7 +209,15 @@ def on_end(self, otel_span):
 else:
 self._update_span_with_otel_data(sentry_span, otel_span)

 sentry_span.finish(end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9))
 sentry_span.finish(
 end_timestamp=utc_from_timestamp(otel_span.end_time / 1e9)
 ) # OTel spans have nanosecond precision

 span_start_in_minutes = int(
 otel_span.start_time / 1e9 / 60
 ) # OTel spans have nanosecond precision
 self.open_spans.setdefault(span_start_in_minutes, set()).discard(span_id)
 self._prune_old_spans()

 def _is_sentry_span(self, hub, otel_span):
 # type: (Hub, OTelSpan) -> bool
Expand Down

92 changes: 92 additions & 0 deletions
tests/integrations/opentelemetry/test_span_processor.py

View file


Open in desktop

Original file line numberDiff line numberDiff line change
Expand Up@@ -531,3 +531,95 @@ def test_link_trace_context_to_error_event():
 assert "contexts" in event
 assert "trace" in event["contexts"]
 assert event["contexts"]["trace"] == fake_trace_context


def test_pruning_old_spans_on_start():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 parent_context = {}
 fake_client = MagicMock()
 fake_client.options = {"instrumenter": "otel"}
 fake_client.dsn = "https://1234567890abcdef@o123456.ingest.sentry.io/123456"

 current_hub = MagicMock()
 current_hub.client = fake_client

 fake_hub = MagicMock()
 fake_hub.current = current_hub

 with mock.patch(
 "sentry_sdk.integrations.opentelemetry.span_processor.Hub", fake_hub
 ):
 span_processor = SentrySpanProcessor()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_start(otel_span, parent_context)
 assert sorted(list(span_processor.otel_span_map.keys())) == [
 "111111111abcdef",
 "1234567890abcdef",
 ]
 assert sorted(list(span_processor.open_spans.values())) == [
 {"111111111abcdef"},
 {"1234567890abcdef"},
 ]
Comment thread
antonpirker marked this conversation as resolved.



def test_pruning_old_spans_on_end():
 otel_span = MagicMock()
 otel_span.name = "Sample OTel Span"
 otel_span.start_time = time.time_ns()
 span_context = SpanContext(
 trace_id=int("1234567890abcdef1234567890abcdef", 16),
 span_id=int("1234567890abcdef", 16),
 is_remote=True,
 )
 otel_span.get_span_context.return_value = span_context
 otel_span.parent = MagicMock()
 otel_span.parent.span_id = int("abcdef1234567890", 16)

 fake_sentry_span = MagicMock(spec=Span)
 fake_sentry_span.set_context = MagicMock()
 fake_sentry_span.finish = MagicMock()

 span_processor = SentrySpanProcessor()
 span_processor._get_otel_context = MagicMock()
 span_processor._update_span_with_otel_data = MagicMock()

 span_processor.otel_span_map = {
 "111111111abcdef": MagicMock(), # should stay
 "2222222222abcdef": MagicMock(), # should go
 "3333333333abcdef": MagicMock(), # should go
 "1234567890abcdef": fake_sentry_span, # should go (because it is closed)
 }
 current_time_minutes = int(time.time() / 60)
 span_processor.open_spans = {
 current_time_minutes: {"1234567890abcdef"}, # should go (because it is closed)
 current_time_minutes - 3: {"111111111abcdef"}, # should stay
 current_time_minutes
 - 11: {"2222222222abcdef", "3333333333abcdef"}, # should go
 }

 span_processor.on_end(otel_span)
 assert sorted(list(span_processor.otel_span_map.keys())) == ["111111111abcdef"]
 assert sorted(list(span_processor.open_spans.values())) == [{"111111111abcdef"}]
Comment thread
antonpirker marked this conversation as resolved.