Skip to content

Commit 8fd75fb

Browse files
jasnellgibfahn
jasnell
authored and
gibfahn
committed
util: graduate TextEncoder/TextDecoder, tests
Add tests ported from Web Platform Tests. Graduate TextEncoder / TextDecoder from experimental PR-URL: #15743 Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Refael Ackermann <refack@gmail.com> Reviewed-By: Anna Henningsen <anna@addaleax.net> Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com> Reviewed-By: Timothy Gu <timothygu99@gmail.com>
1 parent f00ba6b commit 8fd75fb

12 files changed

Lines changed: 563 additions & 28 deletions

‎doc/api/util.md‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -551,8 +551,6 @@ see [Custom promisified functions][].
551551
added: v8.3.0
552552
-->
553553

554-
> Stability: 1 - Experimental
555-
556554
An implementation of the [WHATWG Encoding Standard][]`TextDecoder` API.
557555

558556
```js
@@ -690,8 +688,6 @@ mark.
690688
added: v8.3.0
691689
-->
692690

693-
> Stability: 1 - Experimental
694-
695691
An implementation of the [WHATWG Encoding Standard][]`TextEncoder` API. All
696692
instances of `TextEncoder` only support UTF-8 encoding.
697693

‎lib/internal/encoding.js‎

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,6 @@ const kEncoding = Symbol('encoding');
1010
constkDecoder=Symbol('decoder');
1111
constkEncoder=Symbol('encoder');
1212

13-
letwarned=false;
14-
constexperimental=
15-
'The WHATWG Encoding Standard implementation is an experimental API. It '+
16-
'should not yet be used in production applications.';
17-
1813
const{
1914
getConstructorOf,
2015
customInspectSymbol: inspect
@@ -289,11 +284,6 @@ function getEncodingFromLabel(label) {
289284

290285
classTextEncoder{
291286
constructor(){
292-
if(!warned){
293-
warned=true;
294-
process.emitWarning(experimental,'ExperimentalWarning');
295-
}
296-
297287
this[kEncoder]=true;
298288
}
299289

@@ -353,11 +343,6 @@ function makeTextDecoderICU() {
353343

354344
classTextDecoder{
355345
constructor(encoding='utf-8',options={}){
356-
if(!warned){
357-
warned=true;
358-
process.emitWarning(experimental,'ExperimentalWarning');
359-
}
360-
361346
encoding=`${encoding}`;
362347
if(typeofoptions!=='object')
363348
thrownewerrors.Error('ERR_INVALID_ARG_TYPE','options','object');
@@ -430,11 +415,6 @@ function makeTextDecoderJS() {
430415

431416
classTextDecoder{
432417
constructor(encoding='utf-8',options={}){
433-
if(!warned){
434-
warned=true;
435-
process.emitWarning(experimental,'ExperimentalWarning');
436-
}
437-
438418
encoding=`${encoding}`;
439419
if(typeofoptions!=='object')
440420
thrownewerrors.Error('ERR_INVALID_ARG_TYPE','options','object');

‎test/parallel/test-whatwg-encoding-fatal-streaming.js‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
'use strict';
2+
3+
// From: https://github.com/w3c/web-platform-tests/blob/d74324b53c/encoding/textdecoder-fatal-streaming.html
4+
5+
constcommon=require('../common');
6+
7+
if(!common.hasIntl)
8+
common.skip('missing Intl');
9+
10+
constassert=require('assert');
11+
const{
12+
TextDecoder
13+
}=require('util');
14+
15+
16+
{
17+
[
18+
{encoding: 'utf-8',sequence: [0xC0]},
19+
{encoding: 'utf-16le',sequence: [0x00]},
20+
{encoding: 'utf-16be',sequence: [0x00]}
21+
].forEach((testCase)=>{
22+
constdata=newUint8Array([testCase.sequence]);
23+
common.expectsError(
24+
()=>{
25+
constdecoder=newTextDecoder(testCase.encoding,{fatal: true});
26+
decoder.decode(data);
27+
},{
28+
code: 'ERR_ENCODING_INVALID_ENCODED_DATA',
29+
type: TypeError,
30+
message:
31+
`The encoded data was not valid for encoding ${testCase.encoding}`
32+
}
33+
);
34+
35+
assert.strictEqual(
36+
newTextDecoder(testCase.encoding).decode(data),
37+
'\uFFFD'
38+
);
39+
});
40+
}
41+
42+
{
43+
constdecoder=newTextDecoder('utf-16le',{fatal: true});
44+
constodd=newUint8Array([0x00]);
45+
consteven=newUint8Array([0x00,0x00]);
46+
47+
assert.strictEqual(decoder.decode(odd,{stream: true}),'');
48+
assert.strictEqual(decoder.decode(odd),'\u0000');
49+
50+
common.expectsError(
51+
()=>{
52+
decoder.decode(even,{stream: true});
53+
decoder.decode(odd);
54+
},{
55+
code: 'ERR_ENCODING_INVALID_ENCODED_DATA',
56+
type: TypeError,
57+
message:
58+
'The encoded data was not valid for encoding utf-16le'
59+
}
60+
);
61+
62+
common.expectsError(
63+
()=>{
64+
decoder.decode(odd,{stream: true});
65+
decoder.decode(even);
66+
},{
67+
code: 'ERR_ENCODING_INVALID_ENCODED_DATA',
68+
type: TypeError,
69+
message:
70+
'The encoded data was not valid for encoding utf-16le'
71+
}
72+
);
73+
74+
assert.strictEqual(decoder.decode(even,{stream: true}),'\u0000');
75+
assert.strictEqual(decoder.decode(even),'\u0000');
76+
}

‎test/parallel/test-whatwg-encoding-internals.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
'use strict';
33

44
require('../common');
5+
56
constassert=require('assert');
67
const{ getEncodingFromLabel }=require('internal/encoding');
78

‎test/parallel/test-whatwg-encoding-surrogates-utf8.js‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
'use strict';
2+
3+
// From: https://github.com/w3c/web-platform-tests/blob/fa9436d12c/encoding/api-surrogates-utf8.html
4+
5+
require('../common');
6+
7+
constassert=require('assert');
8+
const{
9+
TextDecoder,
10+
TextEncoder
11+
}=require('util');
12+
13+
constbadStrings=[
14+
{
15+
input: 'abc123',
16+
expected: [0x61,0x62,0x63,0x31,0x32,0x33],
17+
decoded: 'abc123',
18+
name: 'Sanity check'
19+
},
20+
{
21+
input: '\uD800',
22+
expected: [0xef,0xbf,0xbd],
23+
decoded: '\uFFFD',
24+
name: 'Surrogate half (low)'
25+
},
26+
{
27+
input: '\uDC00',
28+
expected: [0xef,0xbf,0xbd],
29+
decoded: '\uFFFD',
30+
name: 'Surrogate half (high)'
31+
},
32+
{
33+
input: 'abc\uD800123',
34+
expected: [0x61,0x62,0x63,0xef,0xbf,0xbd,0x31,0x32,0x33],
35+
decoded: 'abc\uFFFD123',
36+
name: 'Surrogate half (low), in a string'
37+
},
38+
{
39+
input: 'abc\uDC00123',
40+
expected: [0x61,0x62,0x63,0xef,0xbf,0xbd,0x31,0x32,0x33],
41+
decoded: 'abc\uFFFD123',
42+
name: 'Surrogate half (high), in a string'
43+
},
44+
{
45+
input: '\uDC00\uD800',
46+
expected: [0xef,0xbf,0xbd,0xef,0xbf,0xbd],
47+
decoded: '\uFFFD\uFFFD',
48+
name: 'Wrong order'
49+
}
50+
];
51+
52+
badStrings.forEach((t)=>{
53+
constencoded=newTextEncoder().encode(t.input);
54+
assert.deepStrictEqual([].slice.call(encoded),t.expected);
55+
assert.strictEqual(newTextDecoder('utf-8').decode(encoded),t.decoded);
56+
});

‎test/parallel/test-whatwg-encoding-textdecoder-fatal.js‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
'use strict';
2+
3+
// From: https://github.com/w3c/web-platform-tests/blob/39a67e2fff/encoding/textdecoder-fatal.html
4+
5+
constcommon=require('../common');
6+
7+
if(!common.hasIntl)
8+
common.skip('missing Intl');
9+
10+
constassert=require('assert');
11+
const{
12+
TextDecoder
13+
}=require('util');
14+
15+
constbad=[
16+
{encoding: 'utf-8',input: [0xFF],name: 'invalid code'},
17+
{encoding: 'utf-8',input: [0xC0],name: 'ends early'},
18+
{encoding: 'utf-8',input: [0xE0],name: 'ends early 2'},
19+
{encoding: 'utf-8',input: [0xC0,0x00],name: 'invalid trail'},
20+
{encoding: 'utf-8',input: [0xC0,0xC0],name: 'invalid trail 2'},
21+
{encoding: 'utf-8',input: [0xE0,0x00],name: 'invalid trail 3'},
22+
{encoding: 'utf-8',input: [0xE0,0xC0],name: 'invalid trail 4'},
23+
{encoding: 'utf-8',input: [0xE0,0x80,0x00],name: 'invalid trail 5'},
24+
{encoding: 'utf-8',input: [0xE0,0x80,0xC0],name: 'invalid trail 6'},
25+
{encoding: 'utf-8',input: [0xFC,0x80,0x80,0x80,0x80,0x80],
26+
name: '> 0x10FFFF'},
27+
{encoding: 'utf-8',input: [0xFE,0x80,0x80,0x80,0x80,0x80],
28+
name: 'obsolete lead byte'},
29+
// Overlong encodings
30+
{encoding: 'utf-8',input: [0xC0,0x80],name: 'overlong U+0000 - 2 bytes'},
31+
{encoding: 'utf-8',input: [0xE0,0x80,0x80],
32+
name: 'overlong U+0000 - 3 bytes'},
33+
{encoding: 'utf-8',input: [0xF0,0x80,0x80,0x80],
34+
name: 'overlong U+0000 - 4 bytes'},
35+
{encoding: 'utf-8',input: [0xF8,0x80,0x80,0x80,0x80],
36+
name: 'overlong U+0000 - 5 bytes'},
37+
{encoding: 'utf-8',input: [0xFC,0x80,0x80,0x80,0x80,0x80],
38+
name: 'overlong U+0000 - 6 bytes'},
39+
{encoding: 'utf-8',input: [0xC1,0xBF],name: 'overlong U+007F - 2 bytes'},
40+
{encoding: 'utf-8',input: [0xE0,0x81,0xBF],
41+
name: 'overlong U+007F - 3 bytes'},
42+
{encoding: 'utf-8',input: [0xF0,0x80,0x81,0xBF],
43+
name: 'overlong U+007F - 4 bytes'},
44+
{encoding: 'utf-8',input: [0xF8,0x80,0x80,0x81,0xBF],
45+
name: 'overlong U+007F - 5 bytes'},
46+
{encoding: 'utf-8',input: [0xFC,0x80,0x80,0x80,0x81,0xBF],
47+
name: 'overlong U+007F - 6 bytes'},
48+
{encoding: 'utf-8',input: [0xE0,0x9F,0xBF],
49+
name: 'overlong U+07FF - 3 bytes'},
50+
{encoding: 'utf-8',input: [0xF0,0x80,0x9F,0xBF],
51+
name: 'overlong U+07FF - 4 bytes'},
52+
{encoding: 'utf-8',input: [0xF8,0x80,0x80,0x9F,0xBF],
53+
name: 'overlong U+07FF - 5 bytes'},
54+
{encoding: 'utf-8',input: [0xFC,0x80,0x80,0x80,0x9F,0xBF],
55+
name: 'overlong U+07FF - 6 bytes'},
56+
{encoding: 'utf-8',input: [0xF0,0x8F,0xBF,0xBF],
57+
name: 'overlong U+FFFF - 4 bytes'},
58+
{encoding: 'utf-8',input: [0xF8,0x80,0x8F,0xBF,0xBF],
59+
name: 'overlong U+FFFF - 5 bytes'},
60+
{encoding: 'utf-8',input: [0xFC,0x80,0x80,0x8F,0xBF,0xBF],
61+
name: 'overlong U+FFFF - 6 bytes'},
62+
{encoding: 'utf-8',input: [0xF8,0x84,0x8F,0xBF,0xBF],
63+
name: 'overlong U+10FFFF - 5 bytes'},
64+
{encoding: 'utf-8',input: [0xFC,0x80,0x84,0x8F,0xBF,0xBF],
65+
name: 'overlong U+10FFFF - 6 bytes'},
66+
// UTF-16 surrogates encoded as code points in UTF-8
67+
{encoding: 'utf-8',input: [0xED,0xA0,0x80],name: 'lead surrogate'},
68+
{encoding: 'utf-8',input: [0xED,0xB0,0x80],name: 'trail surrogate'},
69+
{encoding: 'utf-8',input: [0xED,0xA0,0x80,0xED,0xB0,0x80],
70+
name: 'surrogate pair'},
71+
{encoding: 'utf-16le',input: [0x00],name: 'truncated code unit'},
72+
// Mismatched UTF-16 surrogates are exercised in utf16-surrogates.html
73+
// FIXME: Add legacy encoding cases
74+
];
75+
76+
bad.forEach((t)=>{
77+
common.expectsError(
78+
()=>{
79+
newTextDecoder(t.encoding,{fatal: true})
80+
.decode(newUint8Array(t.input));
81+
},{
82+
code: 'ERR_ENCODING_INVALID_ENCODED_DATA',
83+
type: TypeError
84+
}
85+
);
86+
});
87+
88+
{
89+
assert('fatal'innewTextDecoder());
90+
assert.strictEqual(typeofnewTextDecoder().fatal,'boolean');
91+
assert(!newTextDecoder().fatal);
92+
assert(newTextDecoder('utf-8',{fatal: true}).fatal);
93+
}

‎test/parallel/test-whatwg-encoding-textdecoder-ignorebom.js‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
'use strict';
2+
3+
// From: https://github.com/w3c/web-platform-tests/blob/7f567fa29c/encoding/textdecoder-ignorebom.html
4+
5+
constcommon=require('../common');
6+
7+
constassert=require('assert');
8+
const{
9+
TextDecoder
10+
}=require('util');
11+
12+
constcases=[
13+
{
14+
encoding: 'utf-8',
15+
bytes: [0xEF,0xBB,0xBF,0x61,0x62,0x63],
16+
skipNoIntl: false
17+
},
18+
{
19+
encoding: 'utf-16le',
20+
bytes: [0xFF,0xFE,0x61,0x00,0x62,0x00,0x63,0x00],
21+
skipNoIntl: false
22+
},
23+
{
24+
encoding: 'utf-16be',
25+
bytes: [0xFE,0xFF,0x00,0x61,0x00,0x62,0x00,0x63],
26+
skipNoIntl: true
27+
}
28+
];
29+
30+
cases.forEach((testCase)=>{
31+
if(testCase.skipNoIntl&&!common.hasIntl){
32+
console.log(`skipping ${testCase.encoding} because missing Intl`);
33+
return;// skipping
34+
}
35+
constBOM='\uFEFF';
36+
letdecoder=newTextDecoder(testCase.encoding,{ignoreBOM: true});
37+
constbytes=newUint8Array(testCase.bytes);
38+
assert.strictEqual(decoder.decode(bytes),`${BOM}abc`);
39+
decoder=newTextDecoder(testCase.encoding,{ignoreBOM: false});
40+
assert.strictEqual(decoder.decode(bytes),'abc');
41+
decoder=newTextDecoder(testCase.encoding);
42+
assert.strictEqual(decoder.decode(bytes),'abc');
43+
});
44+
45+
{
46+
assert('ignoreBOM'innewTextDecoder());
47+
assert.strictEqual(typeofnewTextDecoder().ignoreBOM,'boolean');
48+
assert(!newTextDecoder().ignoreBOM);
49+
assert(newTextDecoder('utf-8',{ignoreBOM: true}).ignoreBOM);
50+
}

0 commit comments

Comments
 (0)