Accept objects for encoded typedarrays in data_array valType

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,

.circleci/config.yml

@@ -292,6 +292,25 @@ jobs:
  paths:
  - plotly.js
 
+ make-baselines-b64:
+ parallelism: 4
+ docker:
+ - image: circleci/python:3.8.9
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Install kaleido, plotly.io and required fonts
+ command: .circleci/env_image.sh
+ - run:
+ name: Create all png files
+ command: .circleci/test.sh make-baselines-b64
+ - persist_to_workspace:
+ root: ~/
+ paths:
+ - plotly.js
+
  test-baselines:
  docker:
  - image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
  path: build
  destination: /
 
+ test-baselines-b64:
+ docker:
+ - image: circleci/node:16.9.0
+ working_directory: ~/plotly.js
+ steps:
+ - attach_workspace:
+ at: ~/
+ - run:
+ name: Compare pixels
+ command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
+ - store_artifacts:
+ path: build
+ destination: /
+
  test-baselines-mathjax3:
  docker:
  - image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
  - test-baselines-mathjax3:
  requires:
  - make-baselines-mathjax3
+ - make-baselines-b64:
+ requires:
+ - install-and-cibuild
+ - test-baselines-b64:
+ requires:
+ - make-baselines-b64
  - make-baselines:
  requires:
  - install-and-cibuild

.circleci/env_image.sh

@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
 sudo fc-cache -f && \
 # install kaleido & plotly
 sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
+# install numpy i.e. to convert arrays to typed arrays
+sudo python3 -m pip install numpy==1.24.2

.circleci/test.sh

@@ -106,6 +106,12 @@ case $1 in
  exit $EXIT_STATE
  ;;
 
+ make-baselines-b64)
+ SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
+ python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
+ exit $EXIT_STATE
+ ;;
+
  make-baselines)
  SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
  python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

.eslintrc

@@ -14,8 +14,12 @@
  "Float32Array": true,
  "Float64Array": true,
  "Uint8Array": true,
+ "Int8Array": true,
+ "Uint8ClampedArray": true,
  "Int16Array": true,
+ "Uint16Array": true,
  "Int32Array": true,
+ "Uint32Array": true,
  "ArrayBuffer": true,
  "DataView": true,
  "SVGElement": false

devtools/test_dashboard/server.js

@@ -23,6 +23,7 @@ if(!strict) {
  config.devtool = 'eval';
 }
 
+var mockFolder = constants.pathToTestImageMocks;
 
 // mock list
 getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
 
 function getMockFiles() {
  return new Promise(function(resolve, reject) {
- fs.readdir(constants.pathToTestImageMocks, function(err, files) {
+ fs.readdir(mockFolder, function(err, files) {
  if(err) {
  reject(err);
  } else {
@@ -116,7 +117,7 @@ function getMockFiles() {
 
 function readFiles(files) {
  var promises = files.map(function(file) {
- var filePath = path.join(constants.pathToTestImageMocks, file);
+ var filePath = path.join(mockFolder, file);
  return readFilePromise(filePath);
  });
 

draftlogs/5230_add.md

@@ -0,0 +1 @@
+ - Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

package.json

@@ -52,6 +52,7 @@
  "test-requirejs": "node tasks/test_requirejs.js",
  "test-plain-obj": "node tasks/test_plain_obj.js",
  "test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
+ "b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
  "mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
  "mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
  "strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
  "@turf/area": "^6.4.0",
  "@turf/bbox": "^6.4.0",
  "@turf/centroid": "^6.0.2",
+ "base64-arraybuffer": "^1.0.2",
  "canvas-fit": "^1.5.0",
  "color-alpha": "1.0.4",
  "color-normalize": "1.5.0",

src/components/colorscale/helpers.js

@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
  var container = containerStr ?
  Lib.nestedProperty(trace, containerStr).get() || {} :
  trace;
+
  var color = container[colorKey || 'color'];
+ if(color && color._inputArray) color = color._inputArray;
 
  var isArrayWithOneNumber = false;
  if(Lib.isArrayOrTypedArray(color)) {

src/lib/array.js

@@ -1,4 +1,7 @@
 'use strict';
+var b64decode = require('base64-arraybuffer').decode;
+
+var isPlainObject = require('./is_plain_object');
 
 var isArray = Array.isArray;
 
@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
  return out;
 };
 
+var typedArrays = {
+ u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
+ Uint8ClampedArray, // not supported in numpy?
+
+ i1: typeof Int8Array === 'undefined' ? undefined :
+ Int8Array,
+
+ u1: typeof Uint8Array === 'undefined' ? undefined :
+ Uint8Array,
+
+ i2: typeof Int16Array === 'undefined' ? undefined :
+ Int16Array,
+
+ u2: typeof Uint16Array === 'undefined' ? undefined :
+ Uint16Array,
+
+ i4: typeof Int32Array === 'undefined' ? undefined :
+ Int32Array,
+
+ u4: typeof Uint32Array === 'undefined' ? undefined :
+ Uint32Array,
+
+ f4: typeof Float32Array === 'undefined' ? undefined :
+ Float32Array,
+
+ f8: typeof Float64Array === 'undefined' ? undefined :
+ Float64Array,
+
+ /* TODO: potentially add Big Int
+
+ i8: typeof BigInt64Array === 'undefined' ? undefined :
+ BigInt64Array,
+
+ u8: typeof BigUint64Array === 'undefined' ? undefined :
+ BigUint64Array,
+ */
+};
+
+typedArrays.uint8c = typedArrays.u1c;
+typedArrays.uint8 = typedArrays.u1;
+typedArrays.int8 = typedArrays.i1;
+typedArrays.uint16 = typedArrays.u2;
+typedArrays.int16 = typedArrays.i2;
+typedArrays.uint32 = typedArrays.u4;
+typedArrays.int32 = typedArrays.i4;
+typedArrays.float32 = typedArrays.f4;
+typedArrays.float64 = typedArrays.f8;
+
+function isArrayBuffer(a) {
+ return a.constructor === ArrayBuffer;
+}
+exports.isArrayBuffer = isArrayBuffer;
+
+exports.decodeTypedArraySpec = function(vIn) {
+ var out = [];
+ var v = coerceTypedArraySpec(vIn);
+ var dtype = v.dtype;
+
+ var T = typedArrays[dtype];
+ if(!T) throw new Error('Error in dtype: "' + dtype + '"');
+ var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
+
+ var buffer = v.bdata;
+ if(!isArrayBuffer(buffer)) {
+ buffer = b64decode(buffer);
+ }
+ var shape = v.shape === undefined ?
+ // detect 1-d length
+ [buffer.byteLength / BYTES_PER_ELEMENT] :
+ // convert number to string and split to array
+ ('' + v.shape).split(',');
+
+ shape.reverse(); // i.e. to match numpy order
+ var ndim = shape.length;
+
+ var nj, j;
+ var ni = +shape[0];
+
+ var rowBytes = BYTES_PER_ELEMENT * ni;
+ var pos = 0;
+
+ if(ndim === 1) {
+ out = new T(buffer);
+ } else if(ndim === 2) {
+ nj = +shape[1];
+ for(j = 0; j < nj; j++) {
+ out[j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ } else if(ndim === 3) {
+ nj = +shape[1];
+ var nk = +shape[2];
+ for(var k = 0; k < nk; k++) {
+ out[k] = [];
+ for(j = 0; j < nj; j++) {
+ out[k][j] = new T(buffer, pos, ni);
+ pos += rowBytes;
+ }
+ }
+ } else {
+ throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
+ }
+
+ // attach bdata, dtype & shape to array for json export
+ out.bdata = v.bdata;
+ out.dtype = v.dtype;
+ out.shape = shape.reverse().join(',');
+
+ vIn._inputArray = out;
+
+ return out;
+};
+
+exports.isTypedArraySpec = function(v) {
+ return (
+ isPlainObject(v) &&
+ v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
+
+ v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
+
+ (v.shape === undefined || (
+ v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
+ ))
+ );
+};
+
+function coerceTypedArraySpec(v) {
+ return {
+ bdata: v.bdata,
+ dtype: v.dtype,
+ shape: v.shape
+ };
+}
+
 /*
  * TypedArray-compatible concatenation of n arrays
  * if all arrays are the same type it will preserve that type,