') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); Use plotly.js `base64` API to store and pass typed arrays declared by numpy, pandas, etc. by archmoj · Pull Request #4470 · plotly/plotly.py · GitHub
Skip to content

Use plotly.js base64 API to store and pass typed arrays declared by numpy, pandas, etc. - #4470

Merged
marthacryan merged 128 commits into
masterfrom
pass-b64
Oct 21, 2024
Merged

Use plotly.js base64 API to store and pass typed arrays declared by numpy, pandas, etc. #4470
marthacryan merged 128 commits into
masterfrom
pass-b64

Conversation

@archmoj

@archmojarchmoj commented Dec 21, 2023•
edited by marthacryan
Loading

Copy link
Copy Markdown
Contributor

This PR make use of this plotly.js PR and changes the default behavior to pass numpy arrays as plotly.js base64 spec.

Code Changes

  • Adds to_typed_array_spec function to use for converting numpy arrays to the "typed array spec" type that plotly.js expects when receiving base64 encoded arrays.
  • Updates the validate_coerce function of the DataArrayValidator class to base64 encode numpy arrays and store them in the "typed array spec" format (by calling to_typed_array_spec)
  • Adds thorough testing to validators
  • Adds performance (size and speed) tests

LiamConnors
LiamConnors reviewed Jan 11, 2024
View reviewed changes
Comment threadpackages/python/plotly/_plotly_utils/basevalidators.py Outdated
@archmoj@LiamConnors
Update packages/python/plotly/_plotly_utils/basevalidators.py
f200767
Co-authored-by: Liam Connors <liam@plot.ly>
LiamConnors
LiamConnors reviewed Jan 17, 2024
View reviewed changes
Comment threaddoc/python/b64.md Outdated
Comment threaddoc/python/b64.md Outdated
Comment threaddoc/python/b64.md Outdated
Comment threaddoc/python/b64.md Outdated
archmojand others added 3 commits February 29, 2024 11:10
@archmoj
Merge remote-tracking branch 'origin/master' into pass-b64_dev
5fb9ee3
 - fixed Conflicts in
packages/python/plotly/plotly/tests/test_optional/test_figure_factory/test_figure_factory.py
packages/python/plotly/plotly/tests/test_optional/test_px/test_px_input.py
@archmoj
Merge branch 'master' into pass-b64
b31dc27
@archmoj
Merge branch 'master' into pass-b64
0323215
@archmojarchmoj mentioned this pull request May 8, 2024
Merged
6 tasks
@marthacryan
Revert changes to tests that check data field
9c5d112
@marthacryan
Merge with master
5e50d8c
@marthacryan
Remove performance tests
8fff9c5
@marthacryan
Update base64 tests to reflect new approach
bd0a2d3
@marthacryan
Fix failing tests
b3ed838
- Removes a speed comparison that is no longer needed because the base64 encoding makes it faster
- Updates a test to use numpy arrays to account for plotly express automatically converting numerical lists to pandas arrays.
- Remove unnecessary enumerate
@marthacryan
Merge with master
7bd7993
@marthacryan
Revert changes to to_json_plotly tests
cd8e0be
archmoj
archmoj commented Oct 17, 2024
View reviewed changes
Comment threadpackages/python/plotly/plotly/tests/test_optional/test_utils/test_utils.py
with self.assertRaises(TypeError):
_json.dumps({"a": {1}}, cls=utils.PlotlyJSONEncoder)

def test_fast_track_finite_arrays(self):

@archmojarchmojOct 17, 2024

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marthacryan
Could we revert this change now or we should drop the test?

@marthacryanmarthacryanOct 17, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this because it was failing. Presumably it's just faster now?

ndrezn
ndrezn reviewed Oct 17, 2024
View reviewed changes

@ndreznndrezn left a comment •
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Psyched for this to land! Killer work team.
ezgif-4-d5a21d0496

emilykl
emilykl reviewed Oct 17, 2024
View reviewed changes
Comment threadtest/percy/compare-pandas.py Outdated
if any(l1 != l2 for l1, l2 in zip(fig1, fig2)):
print("".join(difflib.unified_diff(fig1, fig2)))
raise ValueError(f"Pandas 1/2 difference in {filename}")
if filename not in [

@emilyklemilyklOct 17, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marthacryan / @archmoj What is the reason for excluding these particular files?

emilykl
emilykl reviewed Oct 17, 2024
View reviewed changes
Comment threadpackages/python/plotly/_plotly_utils/utils.py Outdated
are skipped for conversion to the typed array spec
"""
skipped_keys = ["geojson", "layer", "range"]
return any(skipped_key in key for skipped_key in skipped_keys)

@emilyklemilyklOct 17, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@marthacryan Shouldn't the test be skipped_key == key rather than skipped_key in key ?

Or we really do want to be checking that the skipped key is a substring of key?

@emilyklemilyklOct 17, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also I think the docstring comment is no longer accurate

emilykl
emilykl reviewed Oct 17, 2024
View reviewed changes
Comment threadpackages/python/plotly/_plotly_utils/utils.py Outdated
v = copy_to_readonly_numpy_array(v)

np = get_module("numpy", should_load=False)
if not isinstance(v, np.ndarray):

@emilyklemilyklOct 17, 2024•
edited
Loading

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Won't this line raise an error if numpy is not installed? (Since get_module will return None in that case.)

Is that ok?

@marthacryanmarthacryan mentioned this pull request Oct 17, 2024
Open
@marthacryan
marthacryan merged commit 8c75004 into masterOct 21, 2024
@marthacryan
marthacryan deleted the pass-b64 branch October 21, 2024 15:47
@archmojarchmoj mentioned this pull request Oct 21, 2024
Merged
5 tasks
@ParfaitGParfaitG mentioned this pull request Feb 19, 2025
Closed
@alippai

alippai commented Mar 20, 2025

Copy link
Copy Markdown

Should this work for SVG render mode too? (render_mode="svg" in px.line)

@gvwilson

gvwilson commented Mar 20, 2025

Copy link
Copy Markdown
Contributor

cc @marthacryan@emilykl for an answer

@k8culverk8culver mentioned this pull request Apr 8, 2025
Open
@jnumainvillejnumainville mentioned this pull request May 14, 2025
Merged
@emilyklemilykl mentioned this pull request Oct 1, 2025
Closed
@evnchnevnchn mentioned this pull request Jun 5, 2026
Merged
6 tasks
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@emilyklemilyklemilykl left review comments
@alexcjohnsonalexcjohnsonAwaiting requested review from alexcjohnson
@marthacryanmarthacryanAwaiting requested review from marthacryan
+2 more reviewers
@LiamConnorsLiamConnorsLiamConnors left review comments
@ndreznndreznndrezn left review comments
Reviewers whose approvals may not affect merge requirements

Assignees

@marthacryanmarthacryan

Labels

featuresomething newP1needed for current cycle

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

8 participants

@archmoj@marthacryan@gvwilson@alippai@alexcjohnson@emilykl@LiamConnors@ndrezn