diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index a9ba028c..00000000 --- a/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -.eslintrc.js diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index db8a0909..00000000 --- a/.eslintrc.js +++ /dev/null @@ -1,85 +0,0 @@ -module.exports = { - root: true, - extends: [ - // https://eslint.org/docs/rules/ - "eslint:recommended", - // https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/eslint-plugin/src/configs/recommended.ts - "plugin:@typescript-eslint/recommended", - - // https://github.com/benmosher/eslint-plugin-import - "plugin:import/recommended", - "plugin:import/typescript", - - // https://prettier.io/docs/en/integrating-with-linters.html - // > Make sure to put it last, so it gets the chance to override other configs. - "prettier", - ], - plugins: [ - "@typescript-eslint", - ], - parser: "@typescript-eslint/parser", - parserOptions: { - project: "./tsconfig.json", - }, - settings: {}, - rules: { - "no-constant-condition": ["warn", { checkLoops: false }], - "no-useless-escape": "warn", - "no-console": "warn", - "no-var": "warn", - "no-return-await": "warn", - "prefer-const": "warn", - "guard-for-in": "warn", - "curly": "warn", - "no-param-reassign": "warn", - "prefer-spread": "warn", - - "import/no-unresolved": "off", // cannot handle `paths` in tsconfig - "import/no-cycle": "error", - "import/no-default-export": "warn", - - "@typescript-eslint/await-thenable": "warn", - "@typescript-eslint/array-type": ["warn", { default: "generic" }], - "@typescript-eslint/naming-convention": [ - "warn", - { "selector": "default", "format": ["camelCase", "UPPER_CASE", "PascalCase"], "leadingUnderscore": "allow" }, - { "selector": "typeLike", "format": ["PascalCase"], "leadingUnderscore": "allow" }, - ], - "@typescript-eslint/restrict-plus-operands": ["warn", { "checkCompoundAssignments": true }], - "@typescript-eslint/no-throw-literal": "warn", - "@typescript-eslint/unbound-method": "warn", - "@typescript-eslint/explicit-module-boundary-types": "warn", - "@typescript-eslint/no-extra-semi": "warn", - "@typescript-eslint/no-extra-non-null-assertion": "warn", - "@typescript-eslint/no-unused-vars": ["warn", { "argsIgnorePattern": "^_" }], - "@typescript-eslint/no-use-before-define": "warn", - "@typescript-eslint/no-for-in-array": "warn", - "@typescript-eslint/no-unsafe-argument": "warn", - "@typescript-eslint/no-unsafe-call": "warn", - // "@typescript-eslint/no-unsafe-member-access": "warn", // too strict, especially for testing - "@typescript-eslint/no-unnecessary-condition": ["warn", { "allowConstantLoopConditions": true }], - "@typescript-eslint/no-unnecessary-type-constraint": "warn", - "@typescript-eslint/no-implied-eval": "warn", - "@typescript-eslint/no-non-null-asserted-optional-chain": "warn", - "@typescript-eslint/no-invalid-void-type": "warn", - "@typescript-eslint/no-loss-of-precision": "warn", - "@typescript-eslint/no-confusing-void-expression": "warn", - "@typescript-eslint/prefer-for-of": "warn", - "@typescript-eslint/prefer-includes": "warn", - "@typescript-eslint/prefer-string-starts-ends-with": "warn", - "@typescript-eslint/prefer-readonly": "warn", - "@typescript-eslint/prefer-regexp-exec": "warn", - "@typescript-eslint/prefer-nullish-coalescing": "warn", - "@typescript-eslint/prefer-optional-chain": "warn", - "@typescript-eslint/prefer-ts-expect-error": "warn", - "@typescript-eslint/prefer-regexp-exec": "warn", - - "@typescript-eslint/indent": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-empty-interface": "off", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-var-requires": "off", - "@typescript-eslint/no-non-null-assertion": "off", - "@typescript-eslint/ban-ts-comment": "off", - }, -}; diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..eecf70db --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.vscode/*.json linguist-language=jsonc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 322fd616..101c3dcd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,45 +5,104 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: nodejs: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: node-version: - - '12' - - '14' - - '16' - - '17' - + - '18' + - '20' + - '22' + - '24' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: cache: npm node-version: ${{ matrix.node-version }} - - - run: npm install -g npm - - run: npm install -g nyc codecov + - run: npm install -g nyc - run: npm ci - run: npm run test:cover - - run: codecov -f coverage/*.json + - uses: codecov/codecov-action@v5 + with: + files: coverage/coverage-final.json + token: ${{ secrets.CODECOV_TOKEN }} browser: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: matrix: browser: [ChromeHeadless, FirefoxHeadless] - steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: cache: npm - node-version: '14' + node-version: '22' - run: npm install -g npm - run: npm ci - run: npm run test:browser -- --browsers ${{ matrix.browser }} + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + cache: npm + node-version: '22' + - run: npm ci + - run: npx tsc + - run: npm run lint + + deno: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: "v2.x" + - run: npm ci + - run: npm run test:deno + + bun: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + - run: bun install + - run: npm run test:bun + + node_with_strip_types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + cache: npm + node-version: '24' + - run: npm ci + - run: npm run test:node_with_strip_types + + timeline: + runs-on: ubuntu-latest + permissions: + actions: read + needs: + - nodejs + - browser + - lint + - deno + - bun + steps: + - uses: Kesin11/actions-timeline@v2 + diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c1c41171..578c6153 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,7 +2,8 @@ name: "CodeQL" on: push: - branches: [main] + branches: + - main pull_request: workflow_dispatch: schedule: @@ -13,14 +14,17 @@ jobs: name: Analyze runs-on: ubuntu-latest + permissions: + security-events: write + steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: typescript - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index cfa7d452..79f9e5c7 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -7,18 +7,22 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: fuzzing: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Node.js - uses: actions/setup-node@v2 + uses: actions/setup-node@v4 with: cache: npm - node-version: "16" + node-version: "20" + # npm@9 may fail with https://github.com/npm/cli/issues/6723 + # npm@10 may fail with "GitFetcher requires an Arborist constructor to pack a tarball" + - run: npm install -g npm@8 - run: npm ci - run: npm run test:fuzz diff --git a/.gitignore b/.gitignore index a2b46560..7a381633 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,12 @@ benchmark/sandbox.ts # v8 profiler logs isolate-*.log +# tsimp +.tsimp/ + +# deno +deno.lock + # flamebearer flamegraph.html diff --git a/.mocharc.js b/.mocharc.js index 35d19d66..cc57238c 100644 --- a/.mocharc.js +++ b/.mocharc.js @@ -1,7 +1,6 @@ 'use strict'; require("ts-node/register"); -require("tsconfig-paths/register"); module.exports = { diff: true, diff --git a/.nycrc.json b/.nycrc.json index 69f58e8d..89492926 100644 --- a/.nycrc.json +++ b/.nycrc.json @@ -1,6 +1,6 @@ { - "include": ["src/**/*.ts"], - "extension": [".ts"], + "include": ["src/**/*.ts", "src/**/*.mts"], + "extension": [".ts", ".mtx"], "reporter": [], "sourceMap": true, "instrument": true diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..fc2d6922 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,35 @@ +// For configurations: +// https://code.visualstudio.com/Docs/editor/debugging +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run the current Mocha test file", + "type": "node", + "sourceMaps": true, + "request": "launch", + "internalConsoleOptions": "openOnSessionStart", + "runtimeExecutable": "npx", + "program": "mocha", + "args": [ + "--colors", + "${relativeFile}" + ], + "cwd": "${workspaceFolder}" + }, + { + "name": "Run the current TypeScript file", + "type": "node", + "sourceMaps": true, + "request": "launch", + "internalConsoleOptions": "openOnSessionStart", + "args": [ + "--nolazy", + "-r", + "ts-node/register", + "${relativeFile}" + ], + "cwd": "${workspaceFolder}" + }, + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 9a09c382..de744665 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,13 @@ "files.eol": "\n", "editor.tabSize": 2, "editor.codeActionsOnSave": { - "source.fixAll.eslint": true - } + "source.fixAll.eslint": "explicit" + }, + "cSpell.words": [ + "instanceof", + "tsdoc", + "typeof", + "whatwg" + ], + "makefile.configureOnOpen": false } diff --git a/CHANGELOG.md b/CHANGELOG.md index e8fc7433..4d8d5b43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,100 @@ # This is the revision history of @msgpack/msgpack +## 3.1.2 2025-05-25 + +https://github.com/msgpack/msgpack-javascript/compare/v3.1.1...v3.1.2 + +* Make sure this library works with `node --experimental-strip-types` + +## 3.1.1 2025-03-12 + +https://github.com/msgpack/msgpack-javascript/compare/v3.1.0...v3.1.1 + +* Stop using `Symbol.dispose`, which is not yet supported in some environments ([#268](https://github.com/msgpack/msgpack-javascript/pull/268) by @rijenkii) + + +## 3.1.0 2025-02-21 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.1...v3.1.0 + +* Added support for nonstandard map keys in the decoder ([#266](https://github.com/msgpack/msgpack-javascript/pull/266) by @PejmanNik) + +## 3.0.1 2025-02-11 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0...v3.0.1 + +* Implement a tiny polyfill to Symbol.dispose ([#261](https://github.com/msgpack/msgpack-javascript/pull/261) to fix #260) + + +## 3.0.0 2025-02-07 + +https://github.com/msgpack/msgpack-javascript/compare/v2.8.0...v3.0.0 + +* Set the compile target to ES2020, dropping support for the dists with the ES5 target +* Fixed a bug that `encode()` and `decode()` were not re-entrant in reusing instances ([#257](https://github.com/msgpack/msgpack-javascript/pull/257)) +* Allowed the data alignment to support zero-copy decoding ([#248](https://github.com/msgpack/msgpack-javascript/pull/248), thanks to @EddiG) +* Added an option `rawStrings: boolean` to decoders ([#235](https://github.com/msgpack/msgpack-javascript/pull/235), thanks to @jasonpaulos) +* Optimized GC load by reusing stack states ([#228](https://github.com/msgpack/msgpack-javascript/pull/228), thanks to @sergeyzenchenko) +* Added an option `useBigInt64` to map JavaScript's BigInt to MessagePack's int64 and uint64 ([#223](https://github.com/msgpack/msgpack-javascript/pull/223)) +* Drop IE11 support ([#221](https://github.com/msgpack/msgpack-javascript/pull/221)) + * It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://github.com/msgpack/msgpack-javascript/issues/219) +* Change the interfaces of `Encoder` and `Decoder`, and describe the interfaces in README.md ([#224](https://github.com/msgpack/msgpack-javascript/pull/224)): + * `new Encoder(options: EncoderOptions)`: it takes the same named-options as `encode()` + * `new Decoder(options: DecoderOptions)`: it takes the same named-options as `decode()` + +## 3.0.0-beta6 2025-02-07 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta5...v3.0.0-beta6 + +* Set the compile target to ES2020, dropping support for the dists with the ES5 target + +## 3.0.0-beta5 2025-02-06 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta4...v3.0.0-beta5 + +* Fixed a bug that `encode()` and `decode()` were not re-entrant in reusing instances ([#257](https://github.com/msgpack/msgpack-javascript/pull/257)) + +## 3.0.0-beta4 2025-02-04 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta3...v3.0.0-beta4 + +* Added Deno test to CI +* Added Bun tests to CI +* Allowed the data alignment to support zero-copy decoding ([#248](https://github.com/msgpack/msgpack-javascript/pull/248), thanks to @EddiG) + +## 3.0.0-beta3 2025-01-26 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta2...v3.0.0-beta3 + +* Added an option `rawStrings: boolean` to decoders ([#235](https://github.com/msgpack/msgpack-javascript/pull/235), thanks to @jasonpaulos) +* Optimized GC load by reusing stack states ([#228](https://github.com/msgpack/msgpack-javascript/pull/228), thanks to @sergeyzenchenko) +* Drop support for Node.js v16 +* Type compatibility with ES2024 / SharedArrayBuffer + +## 3.0.0-beta2 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta1...v3.0.0-beta2 + +* Upgrade TypeScript compiler to v5.0 + +## 3.0.0-beta1 + +https://github.com/msgpack/msgpack-javascript/compare/v2.8.0...v3.0.0-beta1 + +* Added an option `useBigInt64` to map JavaScript's BigInt to MessagePack's int64 and uint64 ([#223](https://github.com/msgpack/msgpack-javascript/pull/223)) +* Drop IE11 support ([#221](https://github.com/msgpack/msgpack-javascript/pull/221)) + * It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://github.com/msgpack/msgpack-javascript/issues/219) +* Change the interfaces of `Encoder` and `Decoder`, and describe the interfaces in README.md ([#224](https://github.com/msgpack/msgpack-javascript/pull/224)): + * `new Encoder(options: EncoderOptions)`: it takes the same named-options as `encode()` + * `new Decoder(options: DecoderOptions)`: it takes the same named-options as `decode()` + +## 2.8.0 2022-09-02 + +https://github.com/msgpack/msgpack-javascript/compare/v2.7.2...v2.8.0 + +* Let `Encoder#encode()` return a copy of the internal buffer, instead of the reference of the buffer (fix #212). + * Introducing `Encoder#encodeSharedRef()` to return the shared reference to the internal buffer. + ## 2.7.2 2022/02/08 https://github.com/msgpack/msgpack-javascript/compare/v2.7.1...v2.7.2 diff --git a/Makefile b/Makefile index dcfc8df7..88c663a8 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ test: test-all: npm ci - npm publish --dry-run + npm publish --dry-run --tag "$(shell node --experimental-strip-types tools/get-release-tag.mjs)" publish: validate-git-status - npm publish + npm publish --tag "$(shell node --experimental-strip-types tools/get-release-tag.mjs)" git push origin main git push origin --tags @@ -31,7 +31,7 @@ profile-decode: node --prof-process --preprocess -j isolate-*.log | npx flamebearer benchmark: - npx ts-node benchmark/benchmark-from-msgpack-lite.ts + npx node -r ts-node/register benchmark/benchmark-from-msgpack-lite.ts @echo node benchmark/msgpack-benchmark.js diff --git a/README.md b/README.md index b8560805..c0f4dc06 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,16 @@ -# MessagePack for JavaScript/ECMA-262 +# MessagePack for ECMA-262/JavaScript/TypeScript [![npm version](https://img.shields.io/npm/v/@msgpack/msgpack.svg)](https://www.npmjs.com/package/@msgpack/msgpack) ![CI](https://github.com/msgpack/msgpack-javascript/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/msgpack/msgpack-javascript/branch/master/graphs/badge.svg)](https://codecov.io/gh/msgpack/msgpack-javascript) [![minzip](https://badgen.net/bundlephobia/minzip/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) [![tree-shaking](https://badgen.net/bundlephobia/tree-shaking/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) -This is a JavaScript/ECMA-262 implementation of **MessagePack**, an efficient binary serilization format: +This library is an implementation of **MessagePack** for TypeScript and JavaScript, providing a compact and efficient binary serialization format. Learn more about MessagePack at: https://msgpack.org/ -This library is a universal JavaScript, meaning it is compatible with all the major browsers and NodeJS. In addition, because it is implemented in [TypeScript](https://www.typescriptlang.org/), type definition files (`d.ts`) are bundled in the distribution. +This library serves as a comprehensive reference implementation of MessagePack for JavaScript with a focus on accuracy, compatibility, interoperability, and performance. -*Note that this is the second version of MessagePack for JavaScript. The first version, which was implemented in ES5 and was never released to npmjs.com, is tagged as [classic](https://github.com/msgpack/msgpack-javascript/tree/classic).* +Additionally, this is also a universal JavaScript library. It is compatible not only with browsers, but with Node.js or other JavaScript engines that implement ES2015+ standards. As it is written in [TypeScript](https://www.typescriptlang.org/), this library bundles up-to-date type definition files (`d.ts`). + +*Note that this is the second edition of "MessagePack for JavaScript". The first edition, which was implemented in ES5 and never released to npmjs.com, is tagged as [`classic`](https://github.com/msgpack/msgpack-javascript/tree/classic). ## Synopsis @@ -38,19 +40,20 @@ deepStrictEqual(decode(encoded), object); - [Table of Contents](#table-of-contents) - [Install](#install) - [API](#api) - - [`encode(data: unknown, options?: EncodeOptions): Uint8Array`](#encodedata-unknown-options-encodeoptions-uint8array) - - [`EncodeOptions`](#encodeoptions) - - [`decode(buffer: ArrayLike | BufferSource, options?: DecodeOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decodeoptions-unknown) - - [`DecodeOptions`](#decodeoptions) - - [`decodeMulti(buffer: ArrayLike | BufferSource, options?: DecodeOptions): Generator`](#decodemultibuffer-arraylikenumber--buffersource-options-decodeoptions-generatorunknown-void-unknown) - - [`decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): Promise`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-promiseunknown) - - [`decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown) - - [`decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable`](#decodemultistreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown) + - [`encode(data: unknown, options?: EncoderOptions): Uint8Array`](#encodedata-unknown-options-encoderoptions-uint8array) + - [`EncoderOptions`](#encoderoptions) + - [`decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decoderoptions-unknown) + - [`DecoderOptions`](#decoderoptions) + - [`decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator`](#decodemultibuffer-arraylikenumber--buffersource-options-decoderoptions-generatorunknown-void-unknown) + - [`decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): Promise`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-promiseunknown) + - [`decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) + - [`decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodemultistreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) - [Reusing Encoder and Decoder instances](#reusing-encoder-and-decoder-instances) - [Extension Types](#extension-types) - - [ExtensionCodec context](#extensioncodec-context) - - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) - - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) + - [ExtensionCodec context](#extensioncodec-context) + - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) + - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) +- [Faster way to decode a large array of floating point numbers](#faster-way-to-decode-a-large-array-of-floating-point-numbers) - [Decoding a Blob](#decoding-a-blob) - [MessagePack Specification](#messagepack-specification) - [MessagePack Mapping Table](#messagepack-mapping-table) @@ -63,6 +66,7 @@ deepStrictEqual(decode(encoded), object); - [NPM / npmjs.com](#npm--npmjscom) - [CDN / unpkg.com](#cdn--unpkgcom) - [Deno Support](#deno-support) +- [Bun Support](#bun-support) - [Maintenance](#maintenance) - [Testing](#testing) - [Continuous Integration](#continuous-integration) @@ -80,9 +84,9 @@ npm install @msgpack/msgpack ## API -### `encode(data: unknown, options?: EncodeOptions): Uint8Array` +### `encode(data: unknown, options?: EncoderOptions): Uint8Array` -It encodes `data` into a single MessagePack-encoded object, and returns a byte array as `Uint8Array`, throwing errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`. +It encodes `data` into a single MessagePack-encoded object, and returns a byte array as `Uint8Array`. It throws errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`. for example: @@ -105,26 +109,27 @@ const buffer: Buffer = Buffer.from(encoded.buffer, encoded.byteOffset, encoded.b console.log(buffer); ``` -#### `EncodeOptions` +#### `EncoderOptions` -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` -maxDepth | number | `100` -initialBufferSize | number | `2048` -sortKeys | boolean | false -forceFloat32 | boolean | false -forceIntegerToFloat | boolean | false -ignoreUndefined | boolean | false -context | user-defined | - +| Name | Type | Default | +| ------------------- | -------------- | ----------------------------- | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| useBigInt64 | boolean | false | +| maxDepth | number | `100` | +| initialBufferSize | number | `2048` | +| sortKeys | boolean | false | +| forceFloat32 | boolean | false | +| forceIntegerToFloat | boolean | false | +| ignoreUndefined | boolean | false | -### `decode(buffer: ArrayLike | BufferSource, options?: DecodeOptions): unknown` +### `decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown` It decodes `buffer` that includes a MessagePack-encoded object, and returns the decoded object typed `unknown`. `buffer` must be an array of bytes, which is typically `Uint8Array` or `ArrayBuffer`. `BufferSource` is defined as `ArrayBuffer | ArrayBufferView`. -In addition, `buffer` can include a single encoded object. If the `buffer` includes extra bytes after an object, it will throw `RangeError`. To decode `buffer` that includes multiple encoded objects, use `decodeMulti()` or `decodeMultiStream()` (recommended) instead. +The `buffer` must include a single encoded object. If the `buffer` includes extra bytes after an object or the `buffer` is empty, it throws `RangeError`. To decode `buffer` that includes multiple encoded objects, use `decodeMulti()` or `decodeMultiStream()` (recommended) instead. for example: @@ -138,25 +143,32 @@ console.log(object); NodeJS `Buffer` is also acceptable because it is a subclass of `Uint8Array`. -#### `DecodeOptions` +#### `DecoderOptions` + +| Name | Type | Default | +| --------------- | ------------------- | ---------------------------------------------- | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| useBigInt64 | boolean | false | +| rawStrings | boolean | false | +| maxStrLength | number | `4_294_967_295` (UINT32_MAX) | +| maxBinLength | number | `4_294_967_295` (UINT32_MAX) | +| maxArrayLength | number | `4_294_967_295` (UINT32_MAX) | +| maxMapLength | number | `4_294_967_295` (UINT32_MAX) | +| maxExtLength | number | `4_294_967_295` (UINT32_MAX) | +| mapKeyConverter | MapKeyConverterType | throw exception if key is not string or number | + +`MapKeyConverterType` is defined as `(key: unknown) => string | number`. -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` -maxStrLength | number | `4_294_967_295` (UINT32_MAX) -maxBinLength | number | `4_294_967_295` (UINT32_MAX) -maxArrayLength | number | `4_294_967_295` (UINT32_MAX) -maxMapLength | number | `4_294_967_295` (UINT32_MAX) -maxExtLength | number | `4_294_967_295` (UINT32_MAX) -context | user-defined | - +To skip UTF-8 decoding of strings, `rawStrings` can be set to `true`. In this case, strings are decoded into `Uint8Array`. You can use `max${Type}Length` to limit the length of each type decoded. -### `decodeMulti(buffer: ArrayLike | BufferSource, options?: DecodeOptions): Generator` +### `decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator` -It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. That is, this is a synchronous variant for `decodeMultiStream()`. +It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. See also `decodeMultiStream()`, which is an asynchronous variant of this function. -This function is not recommended to decode a MessagePack binary via I/O stream including sockets because it's synchronous. Instead, `decodeMultiStream()` decodes it asynchronously, typically spending less time and memory. +This function is not recommended to decode a MessagePack binary via I/O stream including sockets because it's synchronous. Instead, `decodeMultiStream()` decodes a binary stream asynchronously, typically spending less CPU and memory. for example: @@ -170,11 +182,11 @@ for (const object of decodeMulti(encoded)) { } ``` -### `decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): Promise` +### `decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): Promise` -It decodes `stream`, where `ReadableStreamLike` is defined as `ReadableStream | AsyncIterable`, in an async iterable of byte arrays, and returns decoded object as `unknown` type, wrapped in `Promise`. This function works asynchronously. This is an async variant for `decode()`. +It decodes `stream`, where `ReadableStreamLike` is defined as `ReadableStream | AsyncIterable`, in an async iterable of byte arrays, and returns decoded object as `unknown` type, wrapped in `Promise`. -`DecodeAsyncOptions` is the same as `DecodeOptions` for `decode()`. +This function works asynchronously, and might CPU resources more efficiently compared with synchronous `decode()`, because it doesn't wait for the completion of downloading. This function is designed to work with whatwg `fetch()` like this: @@ -191,7 +203,7 @@ if (contentType && contentType.startsWith(MSGPACK_TYPE) && response.body != null } else { /* handle errors */ } ``` -### `decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable` +### `decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable` It is alike to `decodeAsync()`, but only accepts a `stream` that includes an array of items, and emits a decoded item one by one. @@ -208,7 +220,7 @@ for await (const item of decodeArrayStream(stream)) { } ``` -### `decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable` +### `decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable` It is alike to `decodeAsync()` and `decodeArrayStream()`, but the input `stream` must consist of multiple MessagePack-encoded items. This is an asynchronous variant for `decodeMulti()`. @@ -231,7 +243,7 @@ This function is available since v2.4.0; previously it was called as `decodeStre ### Reusing Encoder and Decoder instances -`Encoder` and `Decoder` classes is provided for better performance: +`Encoder` and `Decoder` classes are provided to have better performance by reusing instances: ```typescript import { deepStrictEqual } from "assert"; @@ -249,11 +261,13 @@ than `encode()` function, and reusing `Decoder` instance is about 2% faster than `decode()` function. Note that the result should vary in environments and data structure. +`Encoder` and `Decoder` take the same options as `encode()` and `decode()` respectively. + ## Extension Types To handle [MessagePack Extension Types](https://github.com/msgpack/msgpack/blob/master/spec.md#extension-types), this library provides `ExtensionCodec` class. -Here is an example to setup custom extension types that handles `Map` and `Set` classes in TypeScript: +This is an example to setup custom extension types that handles `Map` and `Set` classes in TypeScript: ```typescript import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; @@ -266,30 +280,30 @@ extensionCodec.register({ type: SET_EXT_TYPE, encode: (object: unknown): Uint8Array | null => { if (object instanceof Set) { - return encode([...object]); + return encode([...object], { extensionCodec }); } else { return null; } }, decode: (data: Uint8Array) => { - const array = decode(data) as Array; + const array = decode(data, { extensionCodec }) as Array; return new Set(array); }, }); -// Map +// Map const MAP_EXT_TYPE = 1; // Any in 0-127 extensionCodec.register({ type: MAP_EXT_TYPE, encode: (object: unknown): Uint8Array => { if (object instanceof Map) { - return encode([...object]); + return encode([...object], { extensionCodec }); } else { return null; } }, decode: (data: Uint8Array) => { - const array = decode(data) as Array<[unknown, unknown]>; + const array = decode(data, { extensionCodec }) as Array<[unknown, unknown]>; return new Map(array); }, }); @@ -298,11 +312,13 @@ const encoded = encode([new Set(), new Map()], { extensionCodec } const decoded = decode(encoded, { extensionCodec }); ``` -Not that extension types for custom objects must be `[0, 127]`, while `[-1, -128]` is reserved for MessagePack itself. +Ensure you include your extensionCodec in any recursive encode and decode statements! + +Note that extension types for custom objects must be `[0, 127]`, while `[-1, -128]` is reserved for MessagePack itself. -#### ExtensionCodec context +### ExtensionCodec context -When using an extension codec, it may be necessary to keep encoding/decoding state, to keep track of which objects got encoded/re-created. To do this, pass a `context` to the `EncodeOptions` and `DecodeOptions` (and if using typescript, type the `ExtensionCodec` too). Don't forget to pass the `{extensionCodec, context}` along recursive encoding/decoding: +When you use an extension codec, it might be necessary to have encoding/decoding state to keep track of which objects got encoded/re-created. To do this, pass a `context` to the `EncoderOptions` and `DecoderOptions`: ```typescript import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; @@ -321,7 +337,7 @@ extensionCodec.register({ type: MYTYPE_EXT_TYPE, encode: (object, context) => { if (object instanceof MyType) { - context.track(object); // <-- like this + context.track(object); return encode(object.toJSON(), { extensionCodec, context }); } else { return null; @@ -330,7 +346,7 @@ extensionCodec.register({ decode: (data, extType, context) => { const decoded = decode(data, { extensionCodec, context }); const my = new MyType(decoded); - context.track(my); // <-- and like this + context.track(my); return my; }, }); @@ -340,44 +356,60 @@ import { encode, decode } from "@msgpack/msgpack"; const context = new MyContext(); -const encoded = = encode({myType: new MyType()}, { extensionCodec, context }); +const encoded = encode({ myType: new MyType() }, { extensionCodec, context }); const decoded = decode(encoded, { extensionCodec, context }); ``` -#### Handling BigInt with ExtensionCodec +### Handling BigInt with ExtensionCodec -This library does not handle BigInt by default, but you can handle it with `ExtensionCodec` like this: +This library does not handle BigInt by default, but you have two options to handle it: + +* Set `useBigInt64: true` to map bigint to MessagePack's int64/uint64 +* Define a custom `ExtensionCodec` to map bigint to a MessagePack's extension type + +`useBigInt64: true` is the simplest way to handle bigint, but it has limitations: + +* A bigint is encoded in 8 byte binaries even if it's a small integer +* A bigint must be smaller than the max value of the uint64 and larger than the min value of the int64. Otherwise the behavior is undefined. + +So you might want to define a custom codec to handle bigint like this: ```typescript import { deepStrictEqual } from "assert"; -import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; +import { encode, decode, ExtensionCodec, DecodeError } from "@msgpack/msgpack"; +// to define a custom codec: const BIGINT_EXT_TYPE = 0; // Any in 0-127 const extensionCodec = new ExtensionCodec(); extensionCodec.register({ - type: BIGINT_EXT_TYPE, - encode: (input: unknown) => { - if (typeof input === "bigint") { - if (input <= Number.MAX_SAFE_INTEGER && input >= Number.MIN_SAFE_INTEGER) { - return encode(parseInt(input.toString(), 10)); - } else { - return encode(input.toString()); - } - } else { - return null; - } - }, - decode: (data: Uint8Array) => { - return BigInt(decode(data)); - }, + type: BIGINT_EXT_TYPE, + encode(input: unknown): Uint8Array | null { + if (typeof input === "bigint") { + if (input <= Number.MAX_SAFE_INTEGER && input >= Number.MIN_SAFE_INTEGER) { + return encode(Number(input)); + } else { + return encode(String(input)); + } + } else { + return null; + } + }, + decode(data: Uint8Array): bigint { + const val = decode(data); + if (!(typeof val === "string" || typeof val === "number")) { + throw new DecodeError(`unexpected BigInt source: ${val} (${typeof val})`); + } + return BigInt(val); + }, }); +// to use it: const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); -const encoded: = encode(value, { extensionCodec }); +const encoded = encode(value, { extensionCodec }); deepStrictEqual(decode(encoded, { extensionCodec }), value); ``` -#### The temporal module as timestamp extensions +### The temporal module as timestamp extensions There is a proposal for a new date/time representations in JavaScript: @@ -397,10 +429,11 @@ import { decodeTimestampToTimeSpec, } from "@msgpack/msgpack"; +// to define a custom codec const extensionCodec = new ExtensionCodec(); extensionCodec.register({ type: EXT_TIMESTAMP, // override the default behavior! - encode: (input: any) => { + encode(input: unknown): Uint8Array | null { if (input instanceof Instant) { const sec = input.seconds; const nsec = Number(input.nanoseconds - BigInt(sec) * BigInt(1e9)); @@ -409,7 +442,7 @@ extensionCodec.register({ return null; } }, - decode: (data: Uint8Array) => { + decode(data: Uint8Array): Instant { const timeSpec = decodeTimestampToTimeSpec(data); const sec = BigInt(timeSpec.sec); const nsec = BigInt(timeSpec.nsec); @@ -417,17 +450,60 @@ extensionCodec.register({ }, }); +// to use it const instant = Instant.fromEpochMilliseconds(Date.now()); const encoded = encode(instant, { extensionCodec }); const decoded = decode(encoded, { extensionCodec }); deepStrictEqual(decoded, instant); ``` -This will be default once the temporal module is standardizied, which is not a near-future, though. +This will become default in this library with major-version increment, if the temporal module is standardized. + +## Faster way to decode a large array of floating point numbers + +If there are large arrays of floating point numbers in your payload, there +is a way to decode it faster: define a custom extension type for `Float#Array` +with alignment. + +An extension type's `encode` method can return a function that takes a parameter +`pos: number`. This parameter can be used to make alignment of the buffer, +resulting decoding it much more performant. + +See an example implementation for `Float32Array`: + +```typescript +const extensionCodec = new ExtensionCodec(); + +const EXT_TYPE_FLOAT32ARRAY = 0; // Any in 0-127 +extensionCodec.register({ + type: EXT_TYPE_FLOAT32ARRAY, + encode: (object: unknown) => { + if (object instanceof Float32Array) { + return (pos: number) => { + const bpe = Float32Array.BYTES_PER_ELEMENT; + const padding = 1 + ((bpe - ((pos + 1) % bpe)) % bpe); + const data = new Uint8Array(object.buffer); + const result = new Uint8Array(padding + data.length); + result[0] = padding; + result.set(data, padding); + return result; + }; + } + return null; + }, + decode: (data: Uint8Array) => { + const padding = data[0]!; + const bpe = Float32Array.BYTES_PER_ELEMENT; + const offset = data.byteOffset + padding; + const length = data.byteLength - padding; + return new Float32Array(data.buffer, offset, length / bpe); + }, +}); +``` ## Decoding a Blob -[`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) is a binary data container provided by browsers. To read its contents, you can use `Blob#arrayBuffer()` or `Blob#stream()`. `Blob#stream()` +[`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) is a binary data container provided by browsers. To read its contents when it contains a MessagePack binary, you can use `Blob#arrayBuffer()` or `Blob#stream()`. `Blob#stream()` is recommended if your target platform support it. This is because streaming decode should be faster for large objects. In both ways, you need to use asynchronous API. @@ -452,7 +528,7 @@ This library is compatible with the "August 2017" revision of MessagePack specif * [x] extension types, added at August 2013 * [x] timestamp ext type, added at August 2017 -The livinng specification is here: +The living specification is here: https://github.com/msgpack/msgpack @@ -462,22 +538,47 @@ Note that as of June 2019 there're no official "version" on the MessagePack spec The following table shows how JavaScript values are mapped to [MessagePack formats](https://github.com/msgpack/msgpack/blob/master/spec.md) and vice versa. -Source Value|MessagePack Format|Value Decoded -----|----|---- -null, undefined|nil|null (*1) -boolean (true, false)|bool family|boolean (true, false) -number (53-bit int)|int family|number (53-bit int) -number (64-bit float)|float family|number (64-bit float) -string|str family|string -ArrayBufferView |bin family|Uint8Array (*2) -Array|array family|Array -Object|map family|Object (*3) -Date|timestamp ext family|Date (*4) +The mapping of integers varies on the setting of `useBigInt64`. + +The default, `useBigInt64: false` is: + +| Source Value | MessagePack Format | Value Decoded | +| --------------------- | -------------------- | --------------------- | +| null, undefined | nil | null (*1) | +| boolean (true, false) | bool family | boolean (true, false) | +| number (53-bit int) | int family | number | +| number (64-bit float) | float family | number | +| string | str family | string (*2) | +| ArrayBufferView | bin family | Uint8Array (*3) | +| Array | array family | Array | +| Object | map family | Object (*4) | +| Date | timestamp ext family | Date (*5) | +| bigint | N/A | N/A (*6) | * *1 Both `null` and `undefined` are mapped to `nil` (`0xC0`) type, and are decoded into `null` -* *2 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` -* *3 In handling `Object`, it is regarded as `Record` in terms of TypeScript -* *4 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. +* *2 If you'd like to skip UTF-8 decoding of strings, set `rawStrings: true`. In this case, strings are decoded into `Uint8Array`. +* *3 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` +* *4 In handling `Object`, it is regarded as `Record` in terms of TypeScript +* *5 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. +* *6 bigint is not supported in `useBigInt64: false` mode, but you can define an extension codec for it. + +If you set `useBigInt64: true`, the following mapping is used: + +| Source Value | MessagePack Format | Value Decoded | +| --------------------------------- | -------------------- | --------------------- | +| null, undefined | nil | null | +| boolean (true, false) | bool family | boolean (true, false) | +| **number (32-bit int)** | int family | number | +| **number (except for the above)** | float family | number | +| **bigint** | int64 / uint64 | bigint (*7) | +| string | str family | string | +| ArrayBufferView | bin family | Uint8Array | +| Array | array family | Array | +| Object | map family | Object | +| Date | timestamp ext family | Date | + + +* *7 If the bigint is larger than the max value of uint64 or smaller than the min value of int64, then the behavior is undefined. ## Prerequisites @@ -485,25 +586,24 @@ This is a universal JavaScript library that supports major browsers and NodeJS. ### ECMA-262 -* ES5 language features -* ES2018 standard library, including: +* ES2015 language features +* ES2024 standard library, including: * Typed arrays (ES2015) * Async iterations (ES2018) - * Features added in ES2015-ES2018 + * Features added in ES2015-ES2022 +* whatwg encodings (`TextEncoder` and `TextDecoder`) -ES2018 standard library used in this library can be polyfilled with [core-js](https://github.com/zloirock/core-js). +ES2022 standard library used in this library can be polyfilled with [core-js](https://github.com/zloirock/core-js). -If you support IE11, import `core-js` in your application entrypoints, as this library does in testing for browsers. +IE11 is no longer supported. If you'd like to use this library in IE11, use v2.x versions. ### NodeJS -NodeJS v10 is required, but NodeJS v12 or later is recommended because it includes the V8 feature of [Improving DataView performance in V8](https://v8.dev/blog/dataview). - -NodeJS before v10 will work by importing `@msgpack/msgpack/dist.es5+umd/msgpack`. +NodeJS v18 is required. ### TypeScript Compiler / Type Definitions -This module requires type definitions of `AsyncIterator`, `SourceBuffer`, whatwg streams, and so on. They are provided by `"lib": ["ES2021", "DOM"]` in `tsconfig.json`. +This module requires type definitions of `AsyncIterator`, `ArrayBufferLike`, whatwg streams, and so on. They are provided by `"lib": ["ES2024", "DOM"]` in `tsconfig.json`. Regarding the TypeScript compiler version, only the latest TypeScript is tested in development. @@ -513,23 +613,22 @@ Run-time performance is not the only reason to use MessagePack, but it's importa V8's built-in JSON has been improved for years, esp. `JSON.parse()` is [significantly improved in V8/7.6](https://v8.dev/blog/v8-release-76), it is the fastest deserializer as of 2019, as the benchmark result bellow suggests. -However, MessagePack can handles binary data effectively, actual performance depends on situations. You'd better take benchmark on your own use-case if performance matters. +However, MessagePack can handles binary data effectively, actual performance depends on situations. Esp. streaming-decoding may be significantly faster than non-streaming decoding if it's effective. You'd better take benchmark on your own use-case if performance matters. -Benchmark on NodeJS/v12.18.3 (V8/7.8) +Benchmark on NodeJS/v22.13.1 (V8/12.4) -operation | op | ms | op/s ------------------------------------------------------------------ | ------: | ----: | ------: -buf = Buffer.from(JSON.stringify(obj)); | 840700 | 5000 | 168140 -buf = JSON.stringify(obj); | 1249800 | 5000 | 249960 -obj = JSON.parse(buf); | 1648000 | 5000 | 329600 -buf = require("msgpack-lite").encode(obj); | 603500 | 5000 | 120700 -obj = require("msgpack-lite").decode(buf); | 315900 | 5000 | 63180 -buf = require("@msgpack/msgpack").encode(obj); | 945400 | 5000 | 189080 -obj = require("@msgpack/msgpack").decode(buf); | 770200 | 5000 | 154040 -buf = /* @msgpack/msgpack */ encoder.encode(obj); | 1162600 | 5000 | 232520 -obj = /* @msgpack/msgpack */ decoder.decode(buf); | 787800 | 5000 | 157560 +| operation | op | ms | op/s | +| ------------------------------------------------- | ------: | ---: | -----: | +| buf = Buffer.from(JSON.stringify(obj)); | 1348700 | 5000 | 269740 | +| obj = JSON.parse(buf.toString("utf-8")); | 1700300 | 5000 | 340060 | +| buf = require("msgpack-lite").encode(obj); | 591300 | 5000 | 118260 | +| obj = require("msgpack-lite").decode(buf); | 539500 | 5000 | 107900 | +| buf = require("@msgpack/msgpack").encode(obj); | 1238700 | 5000 | 247740 | +| obj = require("@msgpack/msgpack").decode(buf); | 1402000 | 5000 | 280400 | +| buf = /* @msgpack/msgpack */ encoder.encode(obj); | 1379800 | 5000 | 275960 | +| obj = /* @msgpack/msgpack */ decoder.decode(buf); | 1406100 | 5000 | 281220 | -Note that `Buffer.from()` for `JSON.stringify()` is necessary to emulate I/O where a JavaScript string must be converted into a byte array encoded in UTF-8, whereas MessagePack's `encode()` returns a byte array. +Note that `JSON` cases use `Buffer` to emulate I/O where a JavaScript string must be converted into a byte array encoded in UTF-8, whereas MessagePack modules deal with byte arrays. ## Distribution @@ -537,11 +636,11 @@ Note that `Buffer.from()` for `JSON.stringify()` is necessary to emulate I/O whe The NPM package distributed in npmjs.com includes both ES2015+ and ES5 files: -* `dist/` is compiled into ES2019 with CommomJS, provided for NodeJS v10 -* `dist.es5+umd/` is compiled into ES5 with UMD - * `dist.es5+umd/msgpack.min.js` - the minified file - * `dist.es5+umd/msgpack.js` - the non-minified file -* `dist.es5+esm/` is compiled into ES5 with ES modules, provided for webpack-like bundlers and NodeJS's ESM-mode +* `dist/` is compiled into ES2020 with CommomJS, provided for NodeJS v10 +* `dist.umd/` is compiled into ES5 with UMD + * `dist.umd/msgpack.min.js` - the minified file + * `dist.umd/msgpack.js` - the non-minified file +* `dist.esm/` is compiled into ES2020 with ES modules, provided for webpack-like bundlers and NodeJS's ESM-mode If you use NodeJS and/or webpack, their module resolvers use the suitable one automatically. @@ -562,7 +661,11 @@ You can use this module on Deno. See `example/deno-*.ts` for examples. -`deno.land/x` is not supported yet. +`deno.land/x` is not supported. + +## Bun Support + +You can use this module on Bun. ## Maintenance @@ -576,16 +679,16 @@ npm run test ### Continuous Integration -This library uses Travis CI. - -test matrix: +This library uses GitHub Actions. -* TypeScript targets - * `target=es2019` / `target=es5` -* JavaScript engines - * NodeJS, browsers (Chrome, Firefox, Safari, IE11, and so on) +Test matrix: -See [test:* in package.json](./package.json) and [.travis.yml](./.travis.yml) for details. +* NodeJS + * v18 / v20 / v22 +* Browsers: + * Chrome, Firefox +* Deno +* Bun ### Release Engineering diff --git a/benchmark/benchmark-from-msgpack-lite.ts b/benchmark/benchmark-from-msgpack-lite.ts index 5cfacc60..c03cd0ad 100644 --- a/benchmark/benchmark-from-msgpack-lite.ts +++ b/benchmark/benchmark-from-msgpack-lite.ts @@ -42,8 +42,7 @@ var buf, obj; if (JSON) { buf = bench('buf = Buffer.from(JSON.stringify(obj));', JSON_stringify, data); - buf = bench('buf = JSON.stringify(obj);', JSON.stringify, data); - obj = bench('obj = JSON.parse(buf);', JSON.parse, buf); + obj = bench('obj = JSON.parse(buf.toString("utf-8"));', JSON_parse, buf); runTest(obj); } @@ -100,10 +99,14 @@ if (notepack) { runTest(obj); } -function JSON_stringify(src: any) { +function JSON_stringify(src: any): Buffer { return Buffer.from(JSON.stringify(src)); } +function JSON_parse(json: Buffer): any { + return JSON.parse(json.toString("utf-8")); +} + function bench(name: string, func: (...args: any[]) => any, src: any) { if (argv.length) { var match = argv.filter(function(grep) { diff --git a/benchmark/decode-string.ts b/benchmark/decode-string.ts index 7e57d043..a6ea1467 100644 --- a/benchmark/decode-string.ts +++ b/benchmark/decode-string.ts @@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/util import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 100, 500, 1_000].map((n) => { return baseStr.repeat(n); }); @@ -14,7 +14,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const bytes = new Uint8Array(new ArrayBuffer(byteLength)); utf8EncodeJs(str, bytes, 0); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/benchmark/encode-string.ts b/benchmark/encode-string.ts index 47662dd1..3f6aac6c 100644 --- a/benchmark/encode-string.ts +++ b/benchmark/encode-string.ts @@ -5,7 +5,7 @@ import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8"; import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 30, 50, 100].map((n) => { return baseStr.repeat(n); }); @@ -13,7 +13,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const byteLength = utf8Count(str); const buffer = new Uint8Array(byteLength); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/benchmark/msgpack-benchmark.js b/benchmark/msgpack-benchmark.js index 60635fdc..2fd90a57 100644 --- a/benchmark/msgpack-benchmark.js +++ b/benchmark/msgpack-benchmark.js @@ -3,13 +3,65 @@ "use strict"; require("ts-node/register"); const Benchmark = require("benchmark"); -const fs = require("fs"); -const msgpack = require("../src"); + +const msgpackEncode = require("..").encode; +const msgpackDecode = require("..").decode; +const ExtensionCodec = require("..").ExtensionCodec; + +const float32ArrayExtensionCodec = new ExtensionCodec(); +float32ArrayExtensionCodec.register({ + type: 0x01, + encode: (object) => { + if (object instanceof Float32Array) { + return new Uint8Array(object.buffer, object.byteOffset, object.byteLength); + } + return null; + }, + decode: (data) => { + const copy = new Uint8Array(data.byteLength); + copy.set(data); + return new Float32Array(copy.buffer); + }, +}); + +const float32ArrayZeroCopyExtensionCodec = new ExtensionCodec(); +float32ArrayZeroCopyExtensionCodec.register({ + type: 0x01, + encode: (object) => { + if (object instanceof Float32Array) { + return (pos) => { + const bpe = Float32Array.BYTES_PER_ELEMENT; + const padding = 1 + ((bpe - ((pos + 1) % bpe)) % bpe); + const data = new Uint8Array(object.buffer); + const result = new Uint8Array(padding + data.length); + result[0] = padding; + result.set(data, padding); + return result; + }; + } + return null; + }, + decode: (data) => { + const padding = data[0]; + const bpe = Float32Array.BYTES_PER_ELEMENT; + const offset = data.byteOffset + padding; + const length = data.byteLength - padding; + return new Float32Array(data.buffer, offset, length / bpe); + }, +}); const implementations = { "@msgpack/msgpack": { - encode: require("..").encode, - decode: require("..").decode, + encode: msgpackEncode, + decode: msgpackDecode, + }, + "@msgpack/msgpack (Float32Array extension)": { + encode: (data) => msgpackEncode(data, { extensionCodec: float32ArrayExtensionCodec }), + decode: (data) => msgpackDecode(data, { extensionCodec: float32ArrayExtensionCodec }), + }, + "@msgpack/msgpack (Float32Array with zero-copy extension)": { + encode: (data) => msgpackEncode(data, { extensionCodec: float32ArrayZeroCopyExtensionCodec }), + decode: (data) => msgpackDecode(data, { extensionCodec: float32ArrayZeroCopyExtensionCodec }), }, "msgpack-lite": { encode: require("msgpack-lite").encode, @@ -21,28 +73,52 @@ const implementations = { }, }; -// exactly the same as: -// https://raw.githubusercontent.com/endel/msgpack-benchmark/master/sample-large.json -const sampleFiles = ["./sample-large.json"]; +const samples = [ + { + // exactly the same as: + // https://raw.githubusercontent.com/endel/msgpack-benchmark/master/sample-large.json + name: "./sample-large.json", + data: require("./sample-large.json"), + }, + { + name: "Large array of numbers", + data: [ + { + position: new Array(1e3).fill(1.14), + }, + ], + }, + { + name: "Large Float32Array", + data: [ + { + position: new Float32Array(1e3).fill(1.14), + }, + ], + }, +]; function validate(name, data, encoded) { - if (JSON.stringify(data) !== JSON.stringify(implementations[name].decode(encoded))) { - throw new Error("Bad implementation: " + name); - } + return JSON.stringify(data) === JSON.stringify(implementations[name].decode(encoded)); } -for (const sampleFile of sampleFiles) { - const data = require(sampleFile); +for (const sample of samples) { + const { name: sampleName, data } = sample; const encodeSuite = new Benchmark.Suite(); const decodeSuite = new Benchmark.Suite(); console.log(""); - console.log("**" + sampleFile + ":** (" + JSON.stringify(data).length + " bytes in JSON)"); + console.log("**" + sampleName + ":** (" + JSON.stringify(data).length + " bytes in JSON)"); console.log(""); for (const name of Object.keys(implementations)) { implementations[name].toDecode = implementations[name].encode(data); - validate(name, data, implementations[name].toDecode); + if (!validate(name, data, implementations[name].toDecode)) { + console.log("```"); + console.log("Not supported by " + name); + console.log("```"); + continue; + } encodeSuite.add("(encode) " + name, () => { implementations[name].encode(data); }); @@ -60,7 +136,7 @@ for (const sampleFile of sampleFiles) { console.log(""); - decodeSuite.on("cycle", function(event) { + decodeSuite.on("cycle", (event) => { console.log(String(event.target)); }); diff --git a/benchmark/package.json b/benchmark/package.json index 5d9e0574..161581b0 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -2,10 +2,13 @@ "name": "@msgpack/msgpack-benchmark", "private": true, "version": "0.0.0", + "scripts": { + "update-dependencies": "npx rimraf node_modules/ package-lock.json ; npm install ; npm audit fix --force ; git restore package.json ; npm install" + }, "dependencies": { - "benchmark": "^2.1.4", - "msgpack-lite": "^0.1.26", - "msgpackr": "^1.2.10", - "notepack.io": "^2.2.0" + "benchmark": "latest", + "msgpack-lite": "latest", + "msgpackr": "latest", + "notepack.io": "latest" } } diff --git a/benchmark/sample-large.json b/benchmark/sample-large.json index 050bfbdd..cd393880 100644 --- a/benchmark/sample-large.json +++ b/benchmark/sample-large.json @@ -16,7 +16,9 @@ "published":true, "title":"The Ruby Rogues", "updated_at":"2015-11-15T22:50:06.565Z", - "url":"http://feeds.feedwrench.com/RubyRogues.rss" + "url":"http://feeds.feedwrench.com/RubyRogues.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"56490d6ad9275a00030000eb", @@ -35,7 +37,9 @@ "published":true, "title":"Grok Podcast", "updated_at":"2015-11-15T22:55:47.498Z", - "url":"http://www.grokpodcast.com/atom.xml" + "url":"http://www.grokpodcast.com/atom.xml", + "score1": 100, + "score2": 0.1 }, { "_id":"564a1c30b1191d0003000000", @@ -53,7 +57,9 @@ "published":true, "title":"The Web Platform Podcast", "updated_at":"2015-11-16T18:11:02.022Z", - "url":"http://thewebplatform.libsyn.com//rss" + "url":"http://thewebplatform.libsyn.com//rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a1de3b1191d0003000047", @@ -72,7 +78,9 @@ "published":true, "title":"Developer Tea", "updated_at":"2015-11-16T23:00:23.224Z", - "url":"http://feeds.feedburner.com/developertea" + "url":"http://feeds.feedburner.com/developertea", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3163e51cc0000300004c", @@ -88,7 +96,9 @@ "published":true, "title":"Remote Conferences - Audio", "updated_at":"2015-11-16T19:41:24.367Z", - "url":"http://feeds.feedwrench.com/remoteconfs-audio.rss" + "url":"http://feeds.feedwrench.com/remoteconfs-audio.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a315de51cc00003000000", @@ -108,7 +118,9 @@ "published":true, "title":"The Freelancers' Show", "updated_at":"2015-11-16T19:41:27.459Z", - "url":"http://feeds.feedwrench.com/TheFreelancersShow.rss" + "url":"http://feeds.feedwrench.com/TheFreelancersShow.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3169e51cc000030000cd", @@ -124,7 +136,9 @@ "published":true, "title":"React Native Radio", "updated_at":"2015-11-16T19:41:29.999Z", - "url":"http://feeds.feedwrench.com/react-native-radio.rss" + "url":"http://feeds.feedwrench.com/react-native-radio.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a316fe51cc000030000d4", @@ -142,7 +156,9 @@ "published":true, "title":"The iPhreaks Show", "updated_at":"2015-11-16T19:41:43.700Z", - "url":"http://feeds.feedwrench.com/iPhreaks.rss" + "url":"http://feeds.feedwrench.com/iPhreaks.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3184e51cc00003000156", @@ -161,7 +177,9 @@ "published":true, "title":"JavaScript Jabber", "updated_at":"2015-11-16T19:42:24.692Z", - "url":"http://feeds.feedwrench.com/JavaScriptJabber.rss" + "url":"http://feeds.feedwrench.com/JavaScriptJabber.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a31dee51cc00003000210", @@ -177,7 +195,9 @@ "published":true, "title":"Web Security Warriors", "updated_at":"2015-11-16T19:43:28.133Z", - "url":"http://feeds.feedwrench.com/websecwarriors.rss" + "url":"http://feeds.feedwrench.com/websecwarriors.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3ddbe51cc00003000217", @@ -194,7 +214,9 @@ "published":true, "title":"Podcast NowLoading", "updated_at":"2015-11-16T23:00:23.963Z", - "url":"http://feeds.feedburner.com/podcastnowloading" + "url":"http://feeds.feedburner.com/podcastnowloading", + "score1": 100, + "score2": 0.1 }, { "_id":"564b9cfe08602e00030000fa", @@ -210,7 +232,9 @@ "published":true, "title":"Being Boss // A Podcast for Creative Entrepreneurs", "updated_at":"2015-11-17T21:32:50.672Z", - "url":"http://www.lovebeingboss.com/RSSRetrieve.aspx?ID=18365\u0026Type=RSS20" + "url":"http://www.lovebeingboss.com/RSSRetrieve.aspx?ID=18365\u0026Type=RSS20", + "score1": 100, + "score2": 0.1 }, { "_id":"564c5c8008602e0003000128", @@ -226,6 +250,8 @@ "published":true, "title":"Nerdcast", "updated_at":"2015-11-18T11:11:20.034Z", - "url":"http://jovemnerd.com.br/categoria/nerdcast/feed/" + "url":"http://jovemnerd.com.br/categoria/nerdcast/feed/", + "score1": 100, + "score2": 0.1 } ] diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000..0af8fa63 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,151 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { fixupConfigRules, fixupPluginRules } from "@eslint/compat"; +import typescriptEslintEslintPlugin from "@typescript-eslint/eslint-plugin"; +import tsdoc from "eslint-plugin-tsdoc"; +import tsParser from "@typescript-eslint/parser"; +import js from "@eslint/js"; +import { FlatCompat } from "@eslint/eslintrc"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all, +}); + +export default [ + { + ignores: ["**/*.js", "test/deno*", "test/bun*"], + }, + ...fixupConfigRules( + compat.extends( + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:import/recommended", + "plugin:import/typescript", + "prettier", + ), + ), + { + plugins: { + "@typescript-eslint": fixupPluginRules(typescriptEslintEslintPlugin), + tsdoc, + }, + + languageOptions: { + parser: tsParser, + ecmaVersion: 5, + sourceType: "script", + + parserOptions: { + project: "./tsconfig.json", + }, + }, + + settings: {}, + + rules: { + "no-constant-condition": [ + "warn", + { + checkLoops: false, + }, + ], + + "no-useless-escape": "warn", + "no-console": "warn", + "no-var": "warn", + "no-return-await": "warn", + "prefer-const": "warn", + "guard-for-in": "warn", + curly: "warn", + "no-param-reassign": "warn", + "prefer-spread": "warn", + "import/no-unresolved": "off", + "import/no-cycle": "error", + "import/no-default-export": "warn", + "tsdoc/syntax": "warn", + "@typescript-eslint/await-thenable": "warn", + + "@typescript-eslint/array-type": [ + "warn", + { + default: "generic", + }, + ], + + "@typescript-eslint/naming-convention": [ + "warn", + { + selector: "default", + format: ["camelCase", "UPPER_CASE", "PascalCase"], + leadingUnderscore: "allow", + }, + { + selector: "typeLike", + format: ["PascalCase"], + leadingUnderscore: "allow", + }, + ], + + "@typescript-eslint/restrict-plus-operands": "warn", + //"@typescript-eslint/no-throw-literal": "warn", + "@typescript-eslint/unbound-method": "warn", + "@typescript-eslint/explicit-module-boundary-types": "warn", + //"@typescript-eslint/no-extra-semi": "warn", + "@typescript-eslint/no-extra-non-null-assertion": "warn", + + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + }, + ], + + "@typescript-eslint/no-use-before-define": "warn", + "@typescript-eslint/no-for-in-array": "warn", + "@typescript-eslint/no-unsafe-argument": "warn", + "@typescript-eslint/no-unsafe-call": "warn", + + "@typescript-eslint/no-unnecessary-condition": [ + "warn", + { + allowConstantLoopConditions: true, + }, + ], + + "@typescript-eslint/no-unnecessary-type-constraint": "warn", + "@typescript-eslint/no-implied-eval": "warn", + "@typescript-eslint/no-non-null-asserted-optional-chain": "warn", + "@typescript-eslint/no-invalid-void-type": "warn", + "@typescript-eslint/no-loss-of-precision": "warn", + "@typescript-eslint/no-confusing-void-expression": "warn", + "@typescript-eslint/no-redundant-type-constituents": "warn", + "@typescript-eslint/prefer-for-of": "warn", + "@typescript-eslint/prefer-includes": "warn", + "@typescript-eslint/prefer-string-starts-ends-with": "warn", + "@typescript-eslint/prefer-readonly": "warn", + "@typescript-eslint/prefer-regexp-exec": "warn", + "@typescript-eslint/prefer-nullish-coalescing": "warn", + "@typescript-eslint/prefer-optional-chain": "warn", + "@typescript-eslint/prefer-ts-expect-error": "warn", + "@typescript-eslint/consistent-type-imports": [ + "error", + { + prefer: "type-imports", + disallowTypeAnnotations: false, + }, + ], + "@typescript-eslint/indent": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-empty-interface": "off", + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-var-requires": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/ban-ts-comment": "off", + }, + }, +]; diff --git a/example/deno-with-jsdeliver.ts b/example/deno-with-jsdeliver.ts new file mode 100755 index 00000000..af72b397 --- /dev/null +++ b/example/deno-with-jsdeliver.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "https://cdn.jsdelivr.net/npm/@msgpack/msgpack/mod.ts"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/deno-with-npm.ts b/example/deno-with-npm.ts new file mode 100755 index 00000000..98fdf92b --- /dev/null +++ b/example/deno-with-npm.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "npm:@msgpack/msgpack"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/fetch-example.html b/example/fetch-example.html index 58f66def..d59c2503 100644 --- a/example/fetch-example.html +++ b/example/fetch-example.html @@ -3,7 +3,7 @@ - +