File tree

10 files changed

+67
-80
lines changed

10 files changed

+67
-80
lines changed
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"include": ["src/**/*.ts"],
3-
"extension": [".ts"],
2+
"include": ["src/**/*.ts", "src/**/*.mts"],
3+
"extension": [".ts", ".mtx"],
44
"reporter": [],
55
"sourceMap": true,
66
"instrument": true
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
"source.fixAll.eslint": true
77
},
88
"cSpell.words": [
9-
"tsdoc"
9+
"tsdoc",
10+
"whatwg"
1011
]
1112
}
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# This is the revision history of @msgpack/msgpack
22

3+
## NEXT
4+
5+
* Drop IE11 support ([#221](https://.com/msgpack/msgpack-javascript/pull/221))
6+
* It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://.com/msgpack/msgpack-javascript/issues/219)
7+
*
8+
39
## 2.8.0 2022-09-02
410

511
* Let `Encoder#encode()` return a copy of the internal buffer, instead of the reference of the buffer (fix #212).
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/util
55
import Benchmark from "benchmark";
66

77
for (const baseStr of ["A", "あ", "🌏"]) {
8-
const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => {
8+
const dataSet = [10, 100, 500, 1_000].map((n) => {
99
return baseStr.repeat(n);
1010
});
1111

@@ -14,7 +14,7 @@ for (const baseStr of ["A", "あ", "🌏"]) {
1414
const bytes = new Uint8Array(new ArrayBuffer(byteLength));
1515
utf8EncodeJs(str, bytes, 0);
1616

17-
console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`);
17+
console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`);
1818

1919
const suite = new Benchmark.Suite();
2020

Original file line numberDiff line numberDiff line change
@@ -5,15 +5,15 @@ import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8";
55
import Benchmark from "benchmark";
66

77
for (const baseStr of ["A", "あ", "🌏"]) {
8-
const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => {
8+
const dataSet = [10, 30, 50, 100].map((n) => {
99
return baseStr.repeat(n);
1010
});
1111

1212
for (const str of dataSet) {
1313
const byteLength = utf8Count(str);
1414
const buffer = new Uint8Array(byteLength);
1515

16-
console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`);
16+
console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`);
1717

1818
const suite = new Benchmark.Suite();
1919

Original file line numberDiff line numberDiff line change
@@ -16,12 +16,7 @@
1616
"prepublishOnly": "run-p 'test:dist:*' && npm run test:browser",
1717
"clean": "rimraf build dist dist.*",
1818
"test": "mocha 'test/**/*.test.ts'",
19-
"test:purejs": "TEXT_ENCODING=never mocha 'test/**/*.test.ts'",
20-
"test:te": "TEXT_ENCODING=force mocha 'test/**/*.test.ts'",
21-
"test:dist:purejs": "TS_NODE_PROJECT=tsconfig.test-dist-es5-purejs.json npm run test:purejs -- --reporter=dot",
22-
"test:cover": "npm run cover:clean && npm-run-all 'test:cover:*' && npm run cover:report",
23-
"test:cover:purejs": "npx nyc --no-clean npm run test:purejs",
24-
"test:cover:te": "npx nyc --no-clean npm run test:te",
19+
"test:cover": "npm run cover:clean && npx nyc --no-clean npm run 'test' && npm run cover:report",
2520
"test:deno": "deno test test/deno_test.ts",
2621
"test:fuzz": "npm exec --yes -- jsfuzz@git+https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/jsfuzz.git --fuzzTime 60 --no-versifier test/decode.jsfuzz.js corpus",
2722
"cover:clean": "rimraf .nyc_output coverage/",
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { prettyByte } from "./utils/prettyByte";
22
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
33
import { getInt64, getUint64, UINT32_MAX } from "./utils/int";
4-
import { utf8DecodeJs, TEXT_DECODER_THRESHOLD, utf8DecodeTD } from "./utils/utf8";
4+
import { utf8Decode } from "./utils/utf8";
55
import { createDataView, ensureUint8Array } from "./utils/typedArrays";
66
import { CachedKeyDecoder, KeyDecoder } from "./CachedKeyDecoder";
77
import { DecodeError } from "./DecodeError";
@@ -38,18 +38,16 @@ const HEAD_BYTE_REQUIRED = -1;
3838
const EMPTY_VIEW = new DataView(new ArrayBuffer(0));
3939
const EMPTY_BYTES = new Uint8Array(EMPTY_VIEW.buffer);
4040

41-
// IE11: Hack to support IE11.
42-
// IE11: Drop this hack and just use RangeError when IE11 is obsolete.
43-
export const DataViewIndexOutOfBoundsError: typeof Error = (() => {
44-
try {
45-
// IE11: The spec says it should throw RangeError,
46-
// IE11: but in IE11 it throws TypeError.
47-
EMPTY_VIEW.getInt8(0);
48-
} catch (e: any) {
49-
return e.constructor;
41+
try {
42+
// IE11: The spec says it should throw RangeError,
43+
// IE11: but in IE11 it throws TypeError.
44+
EMPTY_VIEW.getInt8(0);
45+
} catch (e) {
46+
if (!(e instanceof RangeError)) {
47+
throw new Error("This module is not supported in the current JavaScript engine because DataView does not throw RangeError on out-of-bounds access");
5048
}
51-
throw new Error("never reached");
52-
})();
49+
}
50+
export const DataViewIndexOutOfBoundsError = RangeError;
5351

5452
const MORE_DATA = new DataViewIndexOutOfBoundsError("Insufficient data");
5553

@@ -507,10 +505,8 @@ export class Decoder<ContextType = undefined> {
507505
let object: string;
508506
if (this.stateIsMapKey() && this.keyDecoder?.canBeCached(byteLength)) {
509507
object = this.keyDecoder.decode(this.bytes, offset, byteLength);
510-
} else if (byteLength > TEXT_DECODER_THRESHOLD) {
511-
object = utf8DecodeTD(this.bytes, offset, byteLength);
512508
} else {
513-
object = utf8DecodeJs(this.bytes, offset, byteLength);
509+
object = utf8Decode(this.bytes, offset, byteLength);
514510
}
515511
this.pos += headerOffset + byteLength;
516512
return object;
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { utf8EncodeJs, utf8Count, TEXT_ENCODER_THRESHOLD, utf8EncodeTE } from "./utils/utf8";
1+
import { utf8Count, utf8Encode } from "./utils/utf8";
22
import { ExtensionCodec, ExtensionCodecType } from "./ExtensionCodec";
33
import { setInt64, setUint64 } from "./utils/int";
44
import { ensureUint8Array } from "./utils/typedArrays";
@@ -177,21 +177,12 @@ export class Encoder<ContextType = undefined> {
177177

178178
private encodeString(object: string) {
179179
const maxHeaderSize = 1 + 4;
180-
const strLength = object.length;
181-
182-
if (strLength > TEXT_ENCODER_THRESHOLD) {
183-
const byteLength = utf8Count(object);
184-
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
185-
this.writeStringHeader(byteLength);
186-
utf8EncodeTE(object, this.bytes, this.pos);
187-
this.pos += byteLength;
188-
} else {
189-
const byteLength = utf8Count(object);
190-
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
191-
this.writeStringHeader(byteLength);
192-
utf8EncodeJs(object, this.bytes, this.pos);
193-
this.pos += byteLength;
194-
}
180+
181+
const byteLength = utf8Count(object);
182+
this.ensureBufferSizeToWrite(maxHeaderSize + byteLength);
183+
this.writeStringHeader(byteLength);
184+
utf8Encode(object, this.bytes, this.pos);
185+
this.pos += byteLength;
195186
}
196187

197188
private encodeObject(object: unknown, depth: number) {
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,3 @@
1-
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
2-
import { UINT32_MAX } from "./int";
3-
4-
const TEXT_ENCODING_AVAILABLE =
5-
(typeof process === "undefined" || process?.env?.["TEXT_ENCODING"] !== "never") &&
6-
typeof TextEncoder !== "undefined" &&
7-
typeof TextDecoder !== "undefined";
81

92
export function utf8Count(str: string): number {
103
const strLength = str.length;
@@ -89,22 +82,30 @@ export function utf8EncodeJs(str: string, output: Uint8Array, outputOffset: numb
8982
}
9083
}
9184

92-
const sharedTextEncoder = TEXT_ENCODING_AVAILABLE ? new TextEncoder() : undefined;
93-
export const TEXT_ENCODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE
94-
? UINT32_MAX
95-
: typeof process !== "undefined" && process?.env?.["TEXT_ENCODING"] !== "force"
96-
? 200
97-
: 0;
85+
// TextEncoder and TextDecoder are standardized in whatwg encoding:
86+
// https://encoding.spec.whatwg.org/
87+
// and available in all the modern browsers:
88+
// https://caniuse.com/textencoder
89+
// They are available in Node.js since v12 LTS as well:
90+
// https://nodejs.org/api/globals.html#textencoder
9891

99-
function utf8EncodeTEencode(str: string, output: Uint8Array, outputOffset: number): void {
100-
output.set(sharedTextEncoder!.encode(str), outputOffset);
101-
}
92+
const sharedTextEncoder = new TextEncoder();
93+
94+
// This threshold should be determined by benchmarking, which might vary in engines and input data.
95+
// Run `npx ts-node benchmark/encode-string.ts` for details.
96+
const TEXT_ENCODER_THRESHOLD = 50;
10297

103-
function utf8EncodeTEencodeInto(str: string, output: Uint8Array, outputOffset: number): void {
104-
sharedTextEncoder!.encodeInto(str, output.subarray(outputOffset));
98+
export function utf8EncodeTE(str: string, output: Uint8Array, outputOffset: number): void {
99+
sharedTextEncoder.encodeInto(str, output.subarray(outputOffset));
105100
}
106101

107-
export const utf8EncodeTE = sharedTextEncoder?.encodeInto ? utf8EncodeTEencodeInto : utf8EncodeTEencode;
102+
export function utf8Encode(str: string, output: Uint8Array, outputOffset: number): void {
103+
if (str.length > TEXT_ENCODER_THRESHOLD) {
104+
utf8EncodeTE(str, output, outputOffset);
105+
} else {
106+
utf8EncodeJs(str, output, outputOffset);
107+
}
108+
}
108109

109110
const CHUNK_SIZE = 0x1_000;
110111

@@ -157,14 +158,21 @@ export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength:
157158
return result;
158159
}
159160

160-
const sharedTextDecoder = TEXT_ENCODING_AVAILABLE ? new TextDecoder() : null;
161-
export const TEXT_DECODER_THRESHOLD = !TEXT_ENCODING_AVAILABLE
162-
? UINT32_MAX
163-
: typeof process !== "undefined" && process?.env?.["TEXT_DECODER"] !== "force"
164-
? 200
165-
: 0;
161+
const sharedTextDecoder = new TextDecoder();
162+
163+
// This threshold should be determined by benchmarking, which might vary in engines and input data.
164+
// Run `npx ts-node benchmark/decode-string.ts` for details.
165+
const TEXT_DECODER_THRESHOLD = 200;
166166

167167
export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
168168
const stringBytes = bytes.subarray(inputOffset, inputOffset + byteLength);
169-
return sharedTextDecoder!.decode(stringBytes);
169+
return sharedTextDecoder.decode(stringBytes);
170+
}
171+
172+
export function utf8Decode(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
173+
if (byteLength > TEXT_DECODER_THRESHOLD) {
174+
return utf8DecodeTD(bytes, inputOffset, byteLength);
175+
} else {
176+
return utf8DecodeJs(bytes, inputOffset, byteLength);
177+
}
170178
}
This file was deleted.

0 commit comments

Comments
 (0)