Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all
 blocks
(function() {
function addCopyButtons() {
document.querySelectorAll('pre code').forEach(function(codeBlock) {
if (codeBlock.parentElement.hasAttribute('data-copy-added')) return;
codeBlock.parentElement.setAttribute('data-copy-added', 'true');
var btn = document.createElement('button');
btn.textContent = 'Copy';
btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;';
btn.onmouseover = function() { this.style.opacity = '1'; };
btn.onmouseout = function() { this.style.opacity = '0.7'; };
btn.onclick = function() {
navigator.clipboard.writeText(codeBlock.textContent).then(function() {
btn.textContent = 'Copied!';
setTimeout(function() { btn.textContent = 'Copy'; }, 1500);
});
};
codeBlock.parentElement.style.position = 'relative';
codeBlock.parentElement.appendChild(btn);
});
}
addCopyButtons();
// Re-run on dynamic content
var observer = new MutationObserver(addCopyButtons);
observer.observe(document.body, { childList: true, subtree: true });
})();
}
} catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); }
})();
(function(){
try {
var __m = "github.com";
var __re = new RegExp('^' + "github\\.com" + '
JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); JSON encoding refactor and orjson encoding by jonmmease · Pull Request #2955 · plotly/plotly.py · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
40b9af1
WIP accelerated encoding with orjson
jonmmease Dec 5, 2020
f79e318
support fig to dict in io without cloning
jonmmease Dec 5, 2020
55720de
Merge branch 'master' into orjson_encoding
jonmmease Dec 5, 2020
7b3593a
fix clone default
jonmmease Dec 5, 2020
da915d6
Add pio.json.config object to configure default encoder
jonmmease Dec 5, 2020
7b235ef
default_encoder to default_engine
jonmmease Dec 5, 2020
7895b6a
blacken
jonmmease Dec 5, 2020
ce05a68
Handle Dash objects in to_json
jonmmease Dec 6, 2020
4ef6510
add JSON encoding tests
jonmmease Dec 31, 2020
101ba85
add testing of from_plotly_json
jonmmease Dec 31, 2020
67d3670
Better error message when orjson not installed and orjson engine requ…
jonmmease Dec 31, 2020
02c00da
Add orjson as optional testing dependency
jonmmease Dec 31, 2020
99ea6a1
Replace Python 3.5 CI tests with 3.8
jonmmease Dec 31, 2020
d44ec26
Try only install orjson with Python 3.6+
jonmmease Dec 31, 2020
b7d8422
Don't test orjson engine when orjson not installed
jonmmease Dec 31, 2020
ddcd6f5
Try new 3.8.7 docker image since prior guess doesn't exist
jonmmease Dec 31, 2020
33359f3
greater than!
jonmmease Dec 31, 2020
c7c1819
Bump scikit image version for Python 3.8 compatibility
jonmmease Dec 31, 2020
a8d52ab
Try to help Python 2 from getting confused about which json module to…
jonmmease Dec 31, 2020
619838f
Update pandas for Python 3
jonmmease Dec 31, 2020
7c7a272
Revert 3.8 CI updates. Too much for this PR
jonmmease Dec 31, 2020
1708703
Doh
jonmmease Dec 31, 2020
66cab10
Don't skip copying during serialization
jonmmease Dec 31, 2020
56a8945
Rename new JSON functions:
jonmmease Jan 2, 2021
0a51020
Ensure cleaned numpy arrays are contiguous
jonmmease Jan 2, 2021
4e9d64e
Use to_json_plotly in html and orca logic
jonmmease Jan 8, 2021
d4068de
Add orjson documentation dependency
jonmmease Jan 8, 2021
58b7192
Handle pandas Timestamp scalars in orjson engine
jonmmease Jan 8, 2021
974fcba
Rework date and string encoding, add and fix tests
jonmmease Jan 8, 2021
a651a63
default JSON engine to "auto"
jonmmease Jan 8, 2021
af1d88d
Fix expected JSON in html export (no spaces)
jonmmease Jan 8, 2021
1d6acc3
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 8, 2021
d51fd94
blacken
jonmmease Jan 8, 2021
042c54c
Fix expected JSON in matplotlylib test
jonmmease Jan 8, 2021
ddc1b8f
Fix expected JSON in html repr test
jonmmease Jan 8, 2021
d7928b0
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 13, 2021
76cc625
Don't drop timezones during serialization, just let Plotly.js ignore …
jonmmease Jan 13, 2021
453461d
Merge branch 'numpy_date_serialization' into orjson_encoding
jonmmease Jan 13, 2021
84ba4b5
no need to skip legacy tests now
jonmmease Jan 13, 2021
340aed3
Only try `datetime_as_string` on datetime kinded numpy arrays
jonmmease Jan 13, 2021
6cea61d
Don't store object or unicode numpy arrays in figure. Coerce to lists
jonmmease Jan 21, 2021
93815c1
Try orjson encoding without cleaning first
jonmmease Jan 21, 2021
242d1fa
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Jan 21, 2021
8a3a4b3
blacken
jonmmease Jan 21, 2021
1de750a
remove scratch file
jonmmease Jan 21, 2021
81f73d5
Remove unused clone
jonmmease Jan 21, 2021
80be8bd
Remove the new "json" encoder
jonmmease Jan 22, 2021
cb54f88
Reorder dict cleaning for performance
jonmmease Jan 22, 2021
1fbfa0d
Merge remote-tracking branch 'origin/master' into orjson_encoding
jonmmease Apr 29, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/requirements.txt
Original file line numberDiff line numberDiff line change
Expand Up@@ -31,3 +31,4 @@ umap-learn==0.5.1
pooch
wget
nbconvert==5.6.1
orjson
79 changes: 35 additions & 44 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,7 +53,7 @@ def to_scalar_or_list(v):
return v


def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
def copy_to_readonly_numpy_array_or_list(v, kind=None, force_numeric=False):
"""
Convert an array-like value into a read-only numpy array

Expand DownExpand Up@@ -89,7 +89,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):

# u: unsigned int, i: signed int, f: float
numeric_kinds = {"u", "i", "f"}
kind_default_dtypes = {"u": "uint32", "i": "int32", "f": "float64", "O": "object"}
kind_default_dtypes = {
"u": "uint32",
"i": "int32",
"f": "float64",
"O": "object",
"U": "U",
}

# Handle pandas Series and Index objects
if pd and isinstance(v, (pd.Series, pd.Index)):
Expand All@@ -113,18 +119,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
return copy_to_readonly_numpy_array(
return copy_to_readonly_numpy_array_or_list(
np.array(v), kind=kind, force_numeric=force_numeric
)
else:
# v is not homogenous array
v_list = [to_scalar_or_list(e) for e in v]

# Lookup dtype for requested kind, if any
dtype = kind_default_dtypes.get(first_kind, None)

# construct new array from list
new_v = np.array(v_list, order="C", dtype=dtype)
return [to_scalar_or_list(e) for e in v]
elif v.dtype.kind in numeric_kinds:
# v is a homogenous numeric array
if kind and v.dtype.kind not in kind:
Expand All@@ -135,6 +135,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# Either no kind was requested or requested kind is satisfied
new_v = np.ascontiguousarray(v.copy())
elif v.dtype.kind == "O":
if kind:
dtype = kind_default_dtypes.get(first_kind, None)
return np.array(v, dtype=dtype)
else:
return v.tolist()
else:
# v is a non-numeric homogenous array
new_v = v.copy()
Expand All@@ -149,12 +155,12 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
if "U" not in kind:
# Force non-numeric arrays to have object type
# --------------------------------------------
# Here we make sure that non-numeric arrays have the object
# datatype. This works around cases like np.array([1, 2, '3']) where
# Here we make sure that non-numeric arrays become lists
# This works around cases like np.array([1, 2, '3']) where
# numpy converts the integers to strings and returns array of dtype
# '<U21'
if new_v.dtype.kind not in ["u", "i", "f", "O", "M"]:
new_v = np.array(v, dtype="object")
return v.tolist()

# Set new array to be read-only
# -----------------------------
Expand DownExpand Up@@ -191,7 +197,7 @@ def is_homogeneous_array(v):
if v_numpy.shape == ():
return False
else:
return True
return True # v_numpy.dtype.kind in ["u", "i", "f", "M", "U"]
return False


Expand DownExpand Up@@ -393,7 +399,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
elif is_simple_array(v):
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -598,7 +604,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els[:10])

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
v = copy_to_readonly_numpy_array_or_list(v)
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -754,7 +760,7 @@ def validate_coerce(self, v):
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
try:
v_array = copy_to_readonly_numpy_array(v, force_numeric=True)
v_array = copy_to_readonly_numpy_array_or_list(v, force_numeric=True)
except (ValueError, TypeError, OverflowError):
self.raise_invalid_val(v)

Expand DownExpand Up@@ -881,7 +887,7 @@ def validate_coerce(self, v):
pass
elif self.array_ok and is_homogeneous_array(v):
np = get_module("numpy")
v_array = copy_to_readonly_numpy_array(
v_array = copy_to_readonly_numpy_array_or_list(
v, kind=("i", "u"), force_numeric=True
)

Expand DownExpand Up@@ -1042,26 +1048,7 @@ def validate_coerce(self, v):
if invalid_els:
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
np = get_module("numpy")

# If not strict, let numpy cast elements to strings
v = copy_to_readonly_numpy_array(v, kind="U")

# Check no_blank
if self.no_blank:
invalid_els = v[v == ""][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

# Check values
if self.values:
invalid_inds = np.logical_not(np.isin(v, self.values))
invalid_els = v[invalid_inds][:10].tolist()
if invalid_els:
self.raise_invalid_elements(invalid_els)

elif is_simple_array(v):
if is_simple_array(v) or is_homogeneous_array(v):
if not self.strict:
v = [StringValidator.to_str_or_unicode_or_none(e) for e in v]

Expand DownExpand Up@@ -1338,8 +1325,12 @@ def validate_coerce(self, v, should_raise=True):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v)
if self.numbers_allowed() and v.dtype.kind in ["u", "i", "f"]:
v = copy_to_readonly_numpy_array_or_list(v)
if (
not isinstance(v, list)
and self.numbers_allowed()
and v.dtype.kind in ["u", "i", "f"]
):
# Numbers are allowed and we have an array of numbers.
# All good
pass
Expand All@@ -1353,9 +1344,9 @@ def validate_coerce(self, v, should_raise=True):

# ### Check that elements have valid colors types ###
elif self.numbers_allowed() or invalid_els:
v = copy_to_readonly_numpy_array(validated_v, kind="O")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="O")
else:
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
elif self.array_ok and is_simple_array(v):
validated_v = [self.validate_coerce(e, should_raise=False) for e in v]

Expand DownExpand Up@@ -1870,7 +1861,7 @@ def validate_coerce(self, v):
self.raise_invalid_elements(invalid_els)

if is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(validated_v, kind="U")
v = copy_to_readonly_numpy_array_or_list(validated_v, kind="U")
else:
v = to_scalar_or_list(v)
else:
Expand DownExpand Up@@ -1918,7 +1909,7 @@ def validate_coerce(self, v):
# Pass None through
pass
elif self.array_ok and is_homogeneous_array(v):
v = copy_to_readonly_numpy_array(v, kind="O")
v = copy_to_readonly_numpy_array_or_list(v, kind="O")
elif self.array_ok and is_simple_array(v):
v = to_scalar_or_list(v)
return v
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -32,15 +32,29 @@ def test_validator_acceptance_simple(val, validator):


@pytest.mark.parametrize(
"val",
[np.array([2, 3, 4]), pd.Series(["a", "b", "c"]), np.array([[1, 2, 3], [4, 5, 6]])],
"val", [np.array([2, 3, 4]), np.array([[1, 2, 3], [4, 5, 6]])],
)
def test_validator_acceptance_homogeneous(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(validator.present(coerce_val), val)


# Accept object array as list
@pytest.mark.parametrize(
"val",
[
["A", "B", "C"],
np.array(["A", "B", "C"], dtype="object"),
pd.Series(["a", "b", "c"]),
],
)
def test_validator_accept_object_array_as_list(val, validator):
coerce_val = validator.validate_coerce(val)
assert isinstance(coerce_val, list)
assert coerce_val == list(val)


# ### Rejection ###
@pytest.mark.parametrize("val", ["Hello", 23, set(), {}])
def test_rejection(val, validator):
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,7 +126,7 @@ def test_rejection_by_element_aok(val, validator_aok):
[],
["bar12"],
("foo", "bar012", "baz"),
np.array([]),
np.array([], dtype="object"),
np.array(["bar12"]),
np.array(["foo", "bar012", "baz"]),
],
Expand All@@ -135,7 +135,7 @@ def test_acceptance_aok(val, validator_aok_re):
# Values should be accepted and returned unchanged
coerce_val = validator_aok_re.validate_coerce(val)
if isinstance(val, (np.ndarray, pd.Series)):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == list(np.array(val))
elif isinstance(val, (list, tuple)):
assert validator_aok_re.present(coerce_val) == tuple(val)
else:
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -149,13 +149,10 @@ def test_color_validator_object(color_validator, color_object_pandas):
res = color_validator.validate_coerce(color_object_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_pandas)
assert res == color_object_pandas.tolist()


def test_color_validator_categorical(color_validator, color_categorical_pandas):
Expand All@@ -164,13 +161,10 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):

# Check type
assert color_categorical_pandas.dtype == "category"
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))
assert res == color_categorical_pandas.tolist()


def test_data_array_validator_dates_series(
Expand All@@ -180,13 +174,10 @@ def test_data_array_validator_dates_series(
res = data_array_validator.validate_coerce(datetime_pandas)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array)
assert res == dates_array.tolist()


def test_data_array_validator_dates_dataframe(
Expand All@@ -197,10 +188,7 @@ def test_data_array_validator_dates_dataframe(
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
assert res == dates_array.reshape(len(dates_array), 1).tolist()
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,8 +138,7 @@ def test_acceptance_aok_scalars(val, validator_aok):
def test_acceptance_aok_list(val, validator_aok):
coerce_val = validator_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert isinstance(coerce_val, np.ndarray)
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
assert coerce_val == val.tolist()
elif isinstance(val, list):
assert validator_aok.present(val) == tuple(val)
else:
Expand DownExpand Up@@ -178,9 +177,7 @@ def test_rejection_aok_values(val, validator_aok_values):
)
def test_acceptance_no_blanks_aok(val, validator_no_blanks_aok):
coerce_val = validator_no_blanks_aok.validate_coerce(val)
if isinstance(val, np.ndarray):
assert np.array_equal(coerce_val, np.array(val, dtype=coerce_val.dtype))
elif isinstance(val, list):
if isinstance(val, (list, np.ndarray)):
assert validator_no_blanks_aok.present(coerce_val) == tuple(val)
else:
assert coerce_val == val
Expand Down
Original file line numberDiff line numberDiff line change
Expand Up@@ -126,10 +126,7 @@ def test_color_validator_object(color_validator, color_object_xarray):
res = color_validator.validate_coerce(color_object_xarray)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"
assert isinstance(res, list)

# Check values
np.testing.assert_array_equal(res, color_object_xarray)
assert res == list(color_object_xarray)
2 changes: 2 additions & 0 deletions packages/python/plotly/_plotly_utils/utils.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -61,8 +61,10 @@ def encode(self, o):
# We catch false positive cases (e.g. strings such as titles, labels etc.)
# but this is ok since the intention is to skip the decoding / reencoding
# step when it's completely safe

if not ("NaN" in encoded_o or "Infinity" in encoded_o):
return encoded_o

# now:
# 1. `loads` to switch Infinity, -Infinity, NaN to None
# 2. `dumps` again so you get 'null' instead of extended JSON
Expand Down
15 changes: 15 additions & 0 deletions packages/python/plotly/plotly/basedatatypes.py
Original file line numberDiff line numberDiff line change
Expand Up@@ -3297,6 +3297,7 @@ def to_dict(self):
# Frame key is only added if there are any frames
res = {"data": data, "layout": layout}
frames = deepcopy([frame._props for frame in self._frame_objs])

if frames:
res["frames"] = frames

Expand DownExpand Up@@ -3413,6 +3414,13 @@ def to_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
str
Expand DownExpand Up@@ -3469,6 +3477,13 @@ def write_json(self, *args, **kwargs):
remove_uids: bool (default True)
True if trace UIDs should be omitted from the JSON representation

engine: str (default None)
The JSON encoding engine to use. One of:
- "json" for an encoder based on the built-in Python json module
- "orjson" for a fast encoder the requires the orjson package
If not specified, the default encoder is set to the current value of
plotly.io.json.config.default_encoder.

Returns
-------
None
Expand Down
Loading