Merged
Show file tree
Hide file tree
Changes from 1 commit
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Failed to load files.
PrevPrevious commit
Next Next commit
tweaks
  • Loading branch information
@gfx
gfx committedMar 3, 2023
commit d6c13193d9bf87eafb7a0c91aa417a85165aa8eb
Original file line numberDiff line numberDiff line change
Expand Up@@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/util
import Benchmark from "benchmark";

for (const baseStr of ["A", "あ", "🌏"]) {
const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => {
const dataSet = [10, 100, 500, 1_000].map((n) => {
return baseStr.repeat(n);
});

Expand All@@ -14,7 +14,7 @@ for (const baseStr of ["A", "あ", "🌏"]) {
const bytes = new Uint8Array(new ArrayBuffer(byteLength));
utf8EncodeJs(str, bytes, 0);

console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`);
console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`);

const suite = new Benchmark.Suite();

Expand Down
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
/* eslint-disable @typescript-eslint/no-unnecessary-condition */
import { UINT32_MAX } from "./int";

export function utf8Count(str: string): number {
const strLength = str.length;
Expand DownExpand Up@@ -88,9 +86,14 @@ export function utf8EncodeJs(str: string, output: Uint8Array, outputOffset: numb
// https://encoding.spec.whatwg.org/
// and available in all the modern browsers:
// https://caniuse.com/textencoder
// They are available in Node.js since v12 LTS as well:
// https://nodejs.org/api/globals.html#textencoder

const sharedTextEncoder = new TextEncoder();
const TEXT_ENCODER_THRESHOLD = 200;

// This threshold should be determined by benchmarking, which might vary in engines and input data.
// Run `npx ts-node benchmark/encode-string.ts` for details.
const TEXT_ENCODER_THRESHOLD = 50;

export function utf8EncodeTE(str: string, output: Uint8Array, outputOffset: number): void {
sharedTextEncoder.encodeInto(str, output.subarray(outputOffset));
Expand DownExpand Up@@ -156,6 +159,9 @@ export function utf8DecodeJs(bytes: Uint8Array, inputOffset: number, byteLength:
}

const sharedTextDecoder = new TextDecoder();

// This threshold should be determined by benchmarking, which might vary in engines and input data.
// Run `npx ts-node benchmark/decode-string.ts` for details.
const TEXT_DECODER_THRESHOLD = 200;

export function utf8DecodeTD(bytes: Uint8Array, inputOffset: number, byteLength: number): string {
Expand Down