Skip to content

Commit 7536e78

Browse files
archmojarchmoj
authored
Merge pull request #5230 from plotly/handle-coded-typed-arrays
Accept objects for encoded typedarrays in data_array valType
2 parents 571fddf + cae76aa commit 7536e78

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

74 files changed

+888
-168
lines changed

Diff for: ‎.circleci/config.yml

+39
Original file line numberDiff line numberDiff line change
@@ -292,6 +292,25 @@ jobs:
292292
paths:
293293
- plotly.js
294294

295+
make-baselines-b64:
296+
parallelism: 4
297+
docker:
298+
- image: circleci/python:3.8.9
299+
working_directory: ~/plotly.js
300+
steps:
301+
- attach_workspace:
302+
at: ~/
303+
- run:
304+
name: Install kaleido, plotly.io and required fonts
305+
command: .circleci/env_image.sh
306+
- run:
307+
name: Create all png files
308+
command: .circleci/test.sh make-baselines-b64
309+
- persist_to_workspace:
310+
root: ~/
311+
paths:
312+
- plotly.js
313+
295314
test-baselines:
296315
docker:
297316
- image: circleci/node:16.9.0
@@ -320,6 +339,20 @@ jobs:
320339
path: build
321340
destination: /
322341

342+
test-baselines-b64:
343+
docker:
344+
- image: circleci/node:16.9.0
345+
working_directory: ~/plotly.js
346+
steps:
347+
- attach_workspace:
348+
at: ~/
349+
- run:
350+
name: Compare pixels
351+
command: .circleci/test.sh test-image ; find build -maxdepth 1 -type f -delete
352+
- store_artifacts:
353+
path: build
354+
destination: /
355+
323356
test-baselines-mathjax3:
324357
docker:
325358
- image: circleci/node:16.9.0
@@ -512,6 +545,12 @@ workflows:
512545
- test-baselines-mathjax3:
513546
requires:
514547
- make-baselines-mathjax3
548+
- make-baselines-b64:
549+
requires:
550+
- install-and-cibuild
551+
- test-baselines-b64:
552+
requires:
553+
- make-baselines-b64
515554
- make-baselines:
516555
requires:
517556
- install-and-cibuild

Diff for: ‎.circleci/env_image.sh

+2
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,5 @@ sudo cp -r .circleci/fonts/ /usr/share/ && \
66
sudo fc-cache -f && \
77
# install kaleido & plotly
88
sudo python3 -m pip install kaleido==0.2.1 plotly==5.5.0 --progress-bar off
9+
# install numpy i.e. to convert arrays to typed arrays
10+
sudo python3 -m pip install numpy==1.24.2

Diff for: ‎.circleci/test.sh

+6
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,12 @@ case $1 in
106106
exit $EXIT_STATE
107107
;;
108108

109+
make-baselines-b64)
110+
SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
111+
python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
112+
exit $EXIT_STATE
113+
;;
114+
109115
make-baselines)
110116
SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
111117
python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?

Diff for: ‎.eslintrc

+4
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,12 @@
1414
"Float32Array": true,
1515
"Float64Array": true,
1616
"Uint8Array": true,
17+
"Int8Array": true,
18+
"Uint8ClampedArray": true,
1719
"Int16Array": true,
20+
"Uint16Array": true,
1821
"Int32Array": true,
22+
"Uint32Array": true,
1923
"ArrayBuffer": true,
2024
"DataView": true,
2125
"SVGElement": false

Diff for: ‎devtools/test_dashboard/server.js

+3-2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ if(!strict) {
2323
config.devtool = 'eval';
2424
}
2525

26+
var mockFolder = constants.pathToTestImageMocks;
2627

2728
// mock list
2829
getMockFiles()
@@ -104,7 +105,7 @@ compiler.run(function(devtoolsErr, devtoolsStats) {
104105

105106
function getMockFiles() {
106107
return new Promise(function(resolve, reject) {
107-
fs.readdir(constants.pathToTestImageMocks, function(err, files) {
108+
fs.readdir(mockFolder, function(err, files) {
108109
if(err) {
109110
reject(err);
110111
} else {
@@ -116,7 +117,7 @@ function getMockFiles() {
116117

117118
function readFiles(files) {
118119
var promises = files.map(function(file) {
119-
var filePath = path.join(constants.pathToTestImageMocks, file);
120+
var filePath = path.join(mockFolder, file);
120121
return readFilePromise(filePath);
121122
});
122123

Diff for: ‎draftlogs/5230_add.md

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- Accept objects for encoded typedarrays in data_array valType [[#5230](https://github.com/plotly/plotly.js/pull/5230)]

Diff for: ‎package-lock.json

+14
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎package.json

+2
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
"test-requirejs": "node tasks/test_requirejs.js",
5353
"test-plain-obj": "node tasks/test_plain_obj.js",
5454
"test": "npm run test-jasmine -- --nowatch && npm run test-bundle && npm run test-image && npm run test-export && npm run test-syntax && npm run lint",
55+
"b64": "python3 test/image/generate_b64_mocks.py && node devtools/test_dashboard/server.js",
5556
"mathjax3": "node devtools/test_dashboard/server.js --mathjax3",
5657
"mathjax3chtml": "node devtools/test_dashboard/server.js --mathjax3chtml",
5758
"strict": "node devtools/test_dashboard/server.js --strict",
@@ -77,6 +78,7 @@
7778
"@turf/area": "^6.4.0",
7879
"@turf/bbox": "^6.4.0",
7980
"@turf/centroid": "^6.0.2",
81+
"base64-arraybuffer": "^1.0.2",
8082
"canvas-fit": "^1.5.0",
8183
"color-alpha": "1.0.4",
8284
"color-normalize": "1.5.0",

Diff for: ‎src/components/colorscale/helpers.js

+2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ function hasColorscale(trace, containerStr, colorKey) {
1313
var container = containerStr ?
1414
Lib.nestedProperty(trace, containerStr).get() || {} :
1515
trace;
16+
1617
var color = container[colorKey || 'color'];
18+
if(color && color._inputArray) color = color._inputArray;
1719

1820
var isArrayWithOneNumber = false;
1921
if(Lib.isArrayOrTypedArray(color)) {

Diff for: ‎src/lib/array.js

+137
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
'use strict';
2+
var b64decode = require('base64-arraybuffer').decode;
3+
4+
var isPlainObject = require('./is_plain_object');
25

36
var isArray = Array.isArray;
47

@@ -48,6 +51,140 @@ exports.ensureArray = function(out, n) {
4851
return out;
4952
};
5053

54+
var typedArrays = {
55+
u1c: typeof Uint8ClampedArray === 'undefined' ? undefined :
56+
Uint8ClampedArray, // not supported in numpy?
57+
58+
i1: typeof Int8Array === 'undefined' ? undefined :
59+
Int8Array,
60+
61+
u1: typeof Uint8Array === 'undefined' ? undefined :
62+
Uint8Array,
63+
64+
i2: typeof Int16Array === 'undefined' ? undefined :
65+
Int16Array,
66+
67+
u2: typeof Uint16Array === 'undefined' ? undefined :
68+
Uint16Array,
69+
70+
i4: typeof Int32Array === 'undefined' ? undefined :
71+
Int32Array,
72+
73+
u4: typeof Uint32Array === 'undefined' ? undefined :
74+
Uint32Array,
75+
76+
f4: typeof Float32Array === 'undefined' ? undefined :
77+
Float32Array,
78+
79+
f8: typeof Float64Array === 'undefined' ? undefined :
80+
Float64Array,
81+
82+
/* TODO: potentially add Big Int
83+
84+
i8: typeof BigInt64Array === 'undefined' ? undefined :
85+
BigInt64Array,
86+
87+
u8: typeof BigUint64Array === 'undefined' ? undefined :
88+
BigUint64Array,
89+
*/
90+
};
91+
92+
typedArrays.uint8c = typedArrays.u1c;
93+
typedArrays.uint8 = typedArrays.u1;
94+
typedArrays.int8 = typedArrays.i1;
95+
typedArrays.uint16 = typedArrays.u2;
96+
typedArrays.int16 = typedArrays.i2;
97+
typedArrays.uint32 = typedArrays.u4;
98+
typedArrays.int32 = typedArrays.i4;
99+
typedArrays.float32 = typedArrays.f4;
100+
typedArrays.float64 = typedArrays.f8;
101+
102+
function isArrayBuffer(a) {
103+
return a.constructor === ArrayBuffer;
104+
}
105+
exports.isArrayBuffer = isArrayBuffer;
106+
107+
exports.decodeTypedArraySpec = function(vIn) {
108+
var out = [];
109+
var v = coerceTypedArraySpec(vIn);
110+
var dtype = v.dtype;
111+
112+
var T = typedArrays[dtype];
113+
if(!T) throw new Error('Error in dtype: "' + dtype + '"');
114+
var BYTES_PER_ELEMENT = T.BYTES_PER_ELEMENT;
115+
116+
var buffer = v.bdata;
117+
if(!isArrayBuffer(buffer)) {
118+
buffer = b64decode(buffer);
119+
}
120+
var shape = v.shape === undefined ?
121+
// detect 1-d length
122+
[buffer.byteLength / BYTES_PER_ELEMENT] :
123+
// convert number to string and split to array
124+
('' + v.shape).split(',');
125+
126+
shape.reverse(); // i.e. to match numpy order
127+
var ndim = shape.length;
128+
129+
var nj, j;
130+
var ni = +shape[0];
131+
132+
var rowBytes = BYTES_PER_ELEMENT * ni;
133+
var pos = 0;
134+
135+
if(ndim === 1) {
136+
out = new T(buffer);
137+
} else if(ndim === 2) {
138+
nj = +shape[1];
139+
for(j = 0; j < nj; j++) {
140+
out[j] = new T(buffer, pos, ni);
141+
pos += rowBytes;
142+
}
143+
} else if(ndim === 3) {
144+
nj = +shape[1];
145+
var nk = +shape[2];
146+
for(var k = 0; k < nk; k++) {
147+
out[k] = [];
148+
for(j = 0; j < nj; j++) {
149+
out[k][j] = new T(buffer, pos, ni);
150+
pos += rowBytes;
151+
}
152+
}
153+
} else {
154+
throw new Error('ndim: ' + ndim + 'is not supported with the shape:"' + v.shape + '"');
155+
}
156+
157+
// attach bdata, dtype & shape to array for json export
158+
out.bdata = v.bdata;
159+
out.dtype = v.dtype;
160+
out.shape = shape.reverse().join(',');
161+
162+
vIn._inputArray = out;
163+
164+
return out;
165+
};
166+
167+
exports.isTypedArraySpec = function(v) {
168+
return (
169+
isPlainObject(v) &&
170+
v.hasOwnProperty('dtype') && (typeof v.dtype === 'string') &&
171+
172+
v.hasOwnProperty('bdata') && (typeof v.bdata === 'string' || isArrayBuffer(v.bdata)) &&
173+
174+
(v.shape === undefined || (
175+
v.hasOwnProperty('shape') && (typeof v.shape === 'string' || typeof v.shape === 'number')
176+
))
177+
);
178+
};
179+
180+
function coerceTypedArraySpec(v) {
181+
return {
182+
bdata: v.bdata,
183+
dtype: v.dtype,
184+
shape: v.shape
185+
};
186+
}
187+
51188
/*
52189
* TypedArray-compatible concatenation of n arrays
53190
* if all arrays are the same type it will preserve that type,

0 commit comments

Comments
 (0)