diff --git a/.eslintrc.js b/.eslintrc.js deleted file mode 100644 index 33f31d35..00000000 --- a/.eslintrc.js +++ /dev/null @@ -1,75 +0,0 @@ -module.exports = { - extends: [ - // https://eslint.org/docs/rules/ - "eslint:recommended", - // https://github.com/typescript-eslint/typescript-eslint/blob/master/packages/eslint-plugin/src/configs/recommended.json - "plugin:@typescript-eslint/recommended", - // https://prettier.io/docs/en/eslint.html - "plugin:prettier/recommended", - - // https://github.com/benmosher/eslint-plugin-import - "plugin:import/recommended", - "plugin:import/typescript", - ], - plugins: [], - parser: "@typescript-eslint/parser", - parserOptions: { - project: "./tsconfig.json", - }, - settings: {}, - rules: { - "no-undef": "off", // useless in TypeScript - "no-constant-condition": ["warn", { checkLoops: false }], - "no-useless-escape": "warn", - "no-console": "warn", - "no-var": "warn", - "valid-typeof": "warn", // "bigint" is not yet supported - "no-return-await": "warn", - "prefer-const": "warn", - "guard-for-in": "warn", - "curly": "warn", - "no-param-reassign": "warn", - "prefer-spread": "warn", - - "import/no-unresolved": "off", // cannot handle `paths` in tsconfig - "import/no-cycle": "error", - "import/no-default-export": "error", - - "@typescript-eslint/await-thenable": "warn", - "@typescript-eslint/array-type": ["warn", { default: "generic" }], - "@typescript-eslint/camelcase": "warn", - "@typescript-eslint/class-name-casing": "warn", // to allow the initial underscore - "@typescript-eslint/restrict-plus-operands": ["warn", { "checkCompoundAssignments": true }], - "@typescript-eslint/no-non-null-assertion": "warn", // NOTE: pay attention to it because it may cause unexpected behavior - "@typescript-eslint/no-throw-literal": "warn", - "@typescript-eslint/no-extra-semi": "warn", - "@typescript-eslint/no-extra-non-null-assertion": "warn", - "@typescript-eslint/no-unused-vars": "warn", - "@typescript-eslint/no-use-before-define": "warn", - "@typescript-eslint/no-for-in-array": "warn", - "@typescript-eslint/no-unnecessary-condition": ["warn", { "allowConstantLoopConditions": true }], - "@typescript-eslint/no-implied-eval": "warn", - "@typescript-eslint/no-non-null-asserted-optional-chain": "warn", - "@typescript-eslint/prefer-for-of": "warn", - "@typescript-eslint/prefer-includes": "warn", - "@typescript-eslint/prefer-string-starts-ends-with": "warn", - "@typescript-eslint/prefer-readonly": "warn", - "@typescript-eslint/prefer-regexp-exec": "warn", - "@typescript-eslint/prefer-nullish-coalescing": "warn", - "@typescript-eslint/prefer-optional-chain": "warn", - - "@typescript-eslint/indent": "off", - "@typescript-eslint/explicit-function-return-type": "off", - "@typescript-eslint/explicit-member-accessibility": "off", - "@typescript-eslint/no-explicit-any": "off", - "@typescript-eslint/no-object-literal-type-assertion": "off", - "@typescript-eslint/no-empty-interface": "off", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-parameter-properties": "off", - "@typescript-eslint/no-var-requires": "off", // enforces `import x = require("x")`, which is TypeScript-specific - "@typescript-eslint/prefer-interface": "off", - "@typescript-eslint/ban-ts-ignore": "off", - - "prettier/prettier": "warn", - }, -}; diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..eecf70db --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.vscode/*.json linguist-language=jsonc diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..0f7e6df0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,108 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + nodejs: + runs-on: ubuntu-latest + strategy: + matrix: + node-version: + - '18' + - '20' + - '22' + - '24' + steps: + - uses: actions/checkout@v6 + - name: Setup Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v6 + with: + cache: npm + node-version: ${{ matrix.node-version }} + - run: npm install -g nyc + - run: npm ci + - run: npm run test:cover + - uses: codecov/codecov-action@v5 + with: + files: coverage/coverage-final.json + token: ${{ secrets.CODECOV_TOKEN }} + + browser: + runs-on: ubuntu-latest + strategy: + matrix: + browser: [ChromeHeadless, FirefoxHeadless] + steps: + - uses: actions/checkout@v6 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache: npm + node-version: '22' + - run: npm install -g npm + - run: npm ci + - run: npm run test:browser -- --browsers ${{ matrix.browser }} + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache: npm + node-version: '22' + - run: npm ci + - run: npx tsgo + - run: npm run lint + + deno: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: "v2.x" + - run: npm ci + - run: npm run test:deno + + bun: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Bun + uses: oven-sh/setup-bun@v2 + - run: bun install + - run: npm run test:bun + + node_with_strip_types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache: npm + node-version: '24' + - run: npm ci + - run: npm run test:node_with_strip_types + + timeline: + runs-on: ubuntu-latest + permissions: + actions: read + needs: + - nodejs + - browser + - lint + - deno + - bun + steps: + - uses: Kesin11/actions-timeline@v2 + diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..685134e5 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,30 @@ +name: "CodeQL" + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + schedule: + - cron: '44 6 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + permissions: + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: typescript + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 00000000..1d4dd374 --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,28 @@ +# https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/jsfuzz + +name: Fuzz + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + fuzzing: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + cache: npm + node-version: "20" + + # npm@9 may fail with https://github.com/npm/cli/issues/6723 + # npm@10 may fail with "GitFetcher requires an Arborist constructor to pack a tarball" + - run: npm install -g npm@8 + - run: npm ci + - run: npm run test:fuzz diff --git a/.gitignore b/.gitignore index 92fd1349..7a381633 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ node_modules/ dist/ -dist.es5/ +dist.*/ build/ .nyc_output/ coverage/ @@ -9,5 +9,14 @@ benchmark/sandbox.ts # v8 profiler logs isolate-*.log +# tsimp +.tsimp/ + +# deno +deno.lock + # flamebearer flamegraph.html + +# jsfuzz +corpus/ diff --git a/.mocharc.js b/.mocharc.js index 35d19d66..cc57238c 100644 --- a/.mocharc.js +++ b/.mocharc.js @@ -1,7 +1,6 @@ 'use strict'; require("ts-node/register"); -require("tsconfig-paths/register"); module.exports = { diff: true, diff --git a/.nycrc.json b/.nycrc.json index 69f58e8d..89492926 100644 --- a/.nycrc.json +++ b/.nycrc.json @@ -1,6 +1,6 @@ { - "include": ["src/**/*.ts"], - "extension": [".ts"], + "include": ["src/**/*.ts", "src/**/*.mts"], + "extension": [".ts", ".mtx"], "reporter": [], "sourceMap": true, "instrument": true diff --git a/.travis.yml b/.travis.yml index 35350c4e..219e8145 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ matrix: include: - node_js: 10 - node_js: 12 + - node_js: 14 - node_js: lts/* env: BROWSER=FirefoxHeadless - node_js: lts/* @@ -40,6 +41,7 @@ matrix: - env: BROWSER=slAndroid cache: npm install: | + npm install -g npm if [ "${BROWSER}" = "" ] then npm install -g nyc codecov fi diff --git a/.vscode/extensions.json b/.vscode/extensions.json index a479e37c..2035884f 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,7 +1,8 @@ { // List of extensions which should be recommended for users of this workspace. "recommendations": [ - "dbaeumer.vscode-eslint" + "dbaeumer.vscode-eslint", + "yzhang.markdown-all-in-one" ], // List of extensions recommended by VS Code that should not be recommended for users of this workspace. "unwantedRecommendations": [ diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..fc2d6922 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,35 @@ +// For configurations: +// https://code.visualstudio.com/Docs/editor/debugging +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run the current Mocha test file", + "type": "node", + "sourceMaps": true, + "request": "launch", + "internalConsoleOptions": "openOnSessionStart", + "runtimeExecutable": "npx", + "program": "mocha", + "args": [ + "--colors", + "${relativeFile}" + ], + "cwd": "${workspaceFolder}" + }, + { + "name": "Run the current TypeScript file", + "type": "node", + "sourceMaps": true, + "request": "launch", + "internalConsoleOptions": "openOnSessionStart", + "args": [ + "--nolazy", + "-r", + "ts-node/register", + "${relativeFile}" + ], + "cwd": "${workspaceFolder}" + }, + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index 9a09c382..de744665 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,13 @@ "files.eol": "\n", "editor.tabSize": 2, "editor.codeActionsOnSave": { - "source.fixAll.eslint": true - } + "source.fixAll.eslint": "explicit" + }, + "cSpell.words": [ + "instanceof", + "tsdoc", + "typeof", + "whatwg" + ], + "makefile.configureOnOpen": false } diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c919630..b459252b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,241 @@ -This is the revision history of @msgpack/msgpack +# This is the revision history of @msgpack/msgpack + +## 3.1.3 2025-12-26 + +https://github.com/msgpack/msgpack-javascript/compare/v3.1.2...v3.1.3 + +* More specific data types - ArrayBuffer instead of ArrayBufferLike (ts 5.9 compatibility issue) by @joshkel ([#279](https://github.com/msgpack/msgpack-javascript/pull/279)) + +## 3.1.2 2025-05-25 + +https://github.com/msgpack/msgpack-javascript/compare/v3.1.1...v3.1.2 + +* Make sure this library works with `node --experimental-strip-types` + +## 3.1.1 2025-03-12 + +https://github.com/msgpack/msgpack-javascript/compare/v3.1.0...v3.1.1 + +* Stop using `Symbol.dispose`, which is not yet supported in some environments ([#268](https://github.com/msgpack/msgpack-javascript/pull/268) by @rijenkii) + + +## 3.1.0 2025-02-21 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.1...v3.1.0 + +* Added support for nonstandard map keys in the decoder ([#266](https://github.com/msgpack/msgpack-javascript/pull/266) by @PejmanNik) + +## 3.0.1 2025-02-11 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0...v3.0.1 + +* Implement a tiny polyfill to Symbol.dispose ([#261](https://github.com/msgpack/msgpack-javascript/pull/261) to fix #260) + + +## 3.0.0 2025-02-07 + +https://github.com/msgpack/msgpack-javascript/compare/v2.8.0...v3.0.0 + +* Set the compile target to ES2020, dropping support for the dists with the ES5 target +* Fixed a bug that `encode()` and `decode()` were not re-entrant in reusing instances ([#257](https://github.com/msgpack/msgpack-javascript/pull/257)) +* Allowed the data alignment to support zero-copy decoding ([#248](https://github.com/msgpack/msgpack-javascript/pull/248), thanks to @EddiG) +* Added an option `rawStrings: boolean` to decoders ([#235](https://github.com/msgpack/msgpack-javascript/pull/235), thanks to @jasonpaulos) +* Optimized GC load by reusing stack states ([#228](https://github.com/msgpack/msgpack-javascript/pull/228), thanks to @sergeyzenchenko) +* Added an option `useBigInt64` to map JavaScript's BigInt to MessagePack's int64 and uint64 ([#223](https://github.com/msgpack/msgpack-javascript/pull/223)) +* Drop IE11 support ([#221](https://github.com/msgpack/msgpack-javascript/pull/221)) + * It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://github.com/msgpack/msgpack-javascript/issues/219) +* Change the interfaces of `Encoder` and `Decoder`, and describe the interfaces in README.md ([#224](https://github.com/msgpack/msgpack-javascript/pull/224)): + * `new Encoder(options: EncoderOptions)`: it takes the same named-options as `encode()` + * `new Decoder(options: DecoderOptions)`: it takes the same named-options as `decode()` + +## 3.0.0-beta6 2025-02-07 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta5...v3.0.0-beta6 + +* Set the compile target to ES2020, dropping support for the dists with the ES5 target + +## 3.0.0-beta5 2025-02-06 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta4...v3.0.0-beta5 + +* Fixed a bug that `encode()` and `decode()` were not re-entrant in reusing instances ([#257](https://github.com/msgpack/msgpack-javascript/pull/257)) + +## 3.0.0-beta4 2025-02-04 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta3...v3.0.0-beta4 + +* Added Deno test to CI +* Added Bun tests to CI +* Allowed the data alignment to support zero-copy decoding ([#248](https://github.com/msgpack/msgpack-javascript/pull/248), thanks to @EddiG) + +## 3.0.0-beta3 2025-01-26 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta2...v3.0.0-beta3 + +* Added an option `rawStrings: boolean` to decoders ([#235](https://github.com/msgpack/msgpack-javascript/pull/235), thanks to @jasonpaulos) +* Optimized GC load by reusing stack states ([#228](https://github.com/msgpack/msgpack-javascript/pull/228), thanks to @sergeyzenchenko) +* Drop support for Node.js v16 +* Type compatibility with ES2024 / SharedArrayBuffer + +## 3.0.0-beta2 + +https://github.com/msgpack/msgpack-javascript/compare/v3.0.0-beta1...v3.0.0-beta2 + +* Upgrade TypeScript compiler to v5.0 + +## 3.0.0-beta1 + +https://github.com/msgpack/msgpack-javascript/compare/v2.8.0...v3.0.0-beta1 + +* Added an option `useBigInt64` to map JavaScript's BigInt to MessagePack's int64 and uint64 ([#223](https://github.com/msgpack/msgpack-javascript/pull/223)) +* Drop IE11 support ([#221](https://github.com/msgpack/msgpack-javascript/pull/221)) + * It also fixes [feature request: option to disable TEXT_ENCODING env check #219](https://github.com/msgpack/msgpack-javascript/issues/219) +* Change the interfaces of `Encoder` and `Decoder`, and describe the interfaces in README.md ([#224](https://github.com/msgpack/msgpack-javascript/pull/224)): + * `new Encoder(options: EncoderOptions)`: it takes the same named-options as `encode()` + * `new Decoder(options: DecoderOptions)`: it takes the same named-options as `decode()` + +## 2.8.0 2022-09-02 + +https://github.com/msgpack/msgpack-javascript/compare/v2.7.2...v2.8.0 + +* Let `Encoder#encode()` return a copy of the internal buffer, instead of the reference of the buffer (fix #212). + * Introducing `Encoder#encodeSharedRef()` to return the shared reference to the internal buffer. + +## 2.7.2 2022/02/08 + +https://github.com/msgpack/msgpack-javascript/compare/v2.7.1...v2.7.2 + +* Fix a build problem in Nuxt3 projects [#200](https://github.com/msgpack/msgpack-javascript/pull/200) reported by (reported as #199 in @masaha03) + +## 2.7.1 2021/09/01 + +https://github.com/msgpack/msgpack-javascript/compare/v2.7.0...v2.7.1 + +* No code changes +* Build with TypeScript 4.4 + +## 2.7.0 2021/05/20 + +https://github.com/msgpack/msgpack-javascript/compare/v2.6.3...v2.7.0 + +* Made sure timestamp decoder to raise DecodeError in errors + * This was found by fuzzing tests using [jsfuzz](https://gitlab.com/gitlab-org/security-products/analyzers/fuzzers/jsfuzz) +* Tiny optimizations and refactoring + +## 2.6.3 2021/05/04 + +https://github.com/msgpack/msgpack-javascript/compare/v2.6.2...v2.6.3 + +* Added `mod.ts` for Deno support + +## 2.6.2 2021/05/04 + +https://github.com/msgpack/msgpack-javascript/compare/v2.6.1...v2.6.2 + +* Improve Deno support (see example/deno-*.ts for details) + +## 2.6.1 2021/05/04 + +https://github.com/msgpack/msgpack-javascript/compare/v2.6.0...v2.6.1 + +* Recover Decoder instance states after `DecodeError` (mitigating [#160](https://github.com/msgpack/msgpack-javascript/issues/160)) + +## 2.6.0 2021/04/21 + +https://github.com/msgpack/msgpack-javascript/compare/v2.5.1...v2.6.0 + +* Revert use of `tslib` (added in 2.5.0) to fix [#169](https://github.com/msgpack/msgpack-javascript/issues/169) + +## v2.5.1 2021/03/21 + +https://github.com/msgpack/msgpack-javascript/compare/v2.5.0...v2.5.1 + +* Fixed the ESM package's dependencies +## v2.5.0 2021/03/21 + +https://github.com/msgpack/msgpack-javascript/compare/v2.4.1...v2.5.0 + +* Throws `DecodeError` in decoding errors +* Rejects `__proto__` as a map key, throwing `DecodeError` + * Thank you to Ninevra Leanne Walden for reporting this issue +* Added `tslib` as a dependency + +## v2.4.1 2021/03/01 + +https://github.com/msgpack/msgpack-javascript/compare/v2.4.0...v2.4.1 + +* Fixed a performance regression that `TextEncoder` and `TextDecoder` were never used even if available ([reported as #157 by @ChALkeR](https://github.com/msgpack/msgpack-javascript/issues/157)) + +## v2.4.0 2021/02/15 + +https://github.com/msgpack/msgpack-javascript/compare/v2.3.1...v2.4.0 + +* Renamed `decodeStream()` to `decodeMultiStream()` + * `decodeStream()` is kept as a deprecated function but will be removed in a future +* Added `decodeMulti()`, a synchronous variant for `decodeMultiStream()` (thanks to @Bilge for the request in [#152](https://github.com/msgpack/msgpack-javascript/issues/152)) +* Improved `decodeAsync()` and its family to accept `BufferSource` (thanks to @rajaybasu for the suggestion in [#152-issuecomment-778712021)](https://github.com/msgpack/msgpack-javascript/issues/152#issuecomment-778712021)) + +## v2.3.1 2021/02/13 + +https://github.com/msgpack/msgpack-javascript/compare/v2.3.0...v2.3.1 + +* Fixed a lot of typos +* Update dev environment: + * Migration to GitHub Actions + * Upgrade Webpack from v4 to v5 + * Enable `noImplicitReturns` and `noUncheckedIndexedAccess` in tsconfig + +## v2.3.0 2020/10/17 + +https://github.com/msgpack/msgpack-javascript/compare/v2.2.1...v2.3.0 + +* Change the extension of ESM files from `.js` to `.mjs` [#144](https://github.com/msgpack/msgpack-javascript/pull/144) +* Make the package work with `strictNullChecks: false` [#139](https://github.com/msgpack/msgpack-javascript/pull/139) by @bananaumai + +## v2.2.1 2020/10/11 + +https://github.com/msgpack/msgpack-javascript/compare/v2.2.0...v2.2.1 + +* Fix `package.json` for webpack to use `module` field + +## v2.2.0 2020/10/04 + +https://github.com/msgpack/msgpack-javascript/compare/v2.1.1...v2.2.0 + +* Now `package.json` has a `module` field to support ES modules + +## v2.1.1 2020/10/04 + +https://github.com/msgpack/msgpack-javascript/compare/v2.1.0...v2.1.1 + +* Fixed typos +* Refactored the codebase + +## v2.1.0 2020/09/21 + +https://github.com/msgpack/msgpack-javascript/compare/v2.0.0...v2.1.0 + +* Added `forceIntegerToFloat` option to `EncodeOptions` by @carbotaniuman ([#123](https://github.com/msgpack/msgpack-javascript/pull/123)) + +## v2.0.0 2020/09/06 + +https://github.com/msgpack/msgpack-javascript/compare/v1.12.2...v2.0.0 + +* Officially introduce direct use of `Encoder` and `Decoder` for better performance + * The major version was bumped because it changed the interface to `Encoder` and `Decoder` +* Build with TypeScript 4.0 + +## v1.12.2 2020/05/14 + +https://github.com/msgpack/msgpack-javascript/compare/v1.12.1...v1.12.2 + +* Build with TypeScript 3.９ + +## v1.12.1 2020/04/08 + +https://github.com/msgpack/msgpack-javascript/compare/v1.12.0...v1.12.1 + +* Build with TypeScript 3.8 ## v1.12.0 2020/03/03 diff --git a/Makefile b/Makefile index f6acfc22..88c663a8 100644 --- a/Makefile +++ b/Makefile @@ -4,16 +4,16 @@ test: test-all: npm ci - npm publish --dry-run + npm publish --dry-run --tag "$(shell node --experimental-strip-types tools/get-release-tag.mjs)" publish: validate-git-status - npm publish - git push origin master - git push origin master --tags + npm publish --tag "$(shell node --experimental-strip-types tools/get-release-tag.mjs)" + git push origin main + git push origin --tags validate-git-status: - @ if [ "`git symbolic-ref --short HEAD`" != "master" ] ; \ - then echo "Not on the master branch!\n" ; exit 1 ; \ + @ if [ "`git symbolic-ref --short HEAD`" != "main" ] ; \ + then echo "Not on the main branch!\n" ; exit 1 ; \ fi @ if ! git diff --exit-code --quiet ; \ then echo "Local differences!\n" ; git status ; exit 1 ; \ @@ -31,7 +31,8 @@ profile-decode: node --prof-process --preprocess -j isolate-*.log | npx flamebearer benchmark: - npx ts-node benchmark/benchmark-from-msgpack-lite.ts + npx node -r ts-node/register benchmark/benchmark-from-msgpack-lite.ts + @echo node benchmark/msgpack-benchmark.js .PHONY: test dist validate-branch benchmark diff --git a/README.md b/README.md index 12e1b9df..c0f4dc06 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ -# MessagePack for JavaScript/ECMA-262 +# MessagePack for ECMA-262/JavaScript/TypeScript -[![npm version](https://img.shields.io/npm/v/@msgpack/msgpack.svg)](https://www.npmjs.com/package/@msgpack/msgpack) [![Build Status](https://travis-ci.org/msgpack/msgpack-javascript.svg?branch=master)](https://travis-ci.org/msgpack/msgpack-javascript) [![codecov](https://codecov.io/gh/msgpack/msgpack-javascript/branch/master/graphs/badge.svg)](https://codecov.io/gh/msgpack/msgpack-javascript) [![bundlephobia](https://badgen.net/bundlephobia/minzip/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) +[![npm version](https://img.shields.io/npm/v/@msgpack/msgpack.svg)](https://www.npmjs.com/package/@msgpack/msgpack) ![CI](https://github.com/msgpack/msgpack-javascript/workflows/CI/badge.svg) [![codecov](https://codecov.io/gh/msgpack/msgpack-javascript/branch/master/graphs/badge.svg)](https://codecov.io/gh/msgpack/msgpack-javascript) [![minzip](https://badgen.net/bundlephobia/minzip/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) [![tree-shaking](https://badgen.net/bundlephobia/tree-shaking/@msgpack/msgpack)](https://bundlephobia.com/result?p=@msgpack/msgpack) -[![Browser Matrix powered by Sauce Labs](https://saucelabs.com/browser-matrix/gfx2019.svg)](https://saucelabs.com) - -This is a JavaScript/ECMA-262 implementation of **MessagePack**, an efficient binary serilization format: +This library is an implementation of **MessagePack** for TypeScript and JavaScript, providing a compact and efficient binary serialization format. Learn more about MessagePack at: https://msgpack.org/ -This library is a universal JavaScript, meaning it is compatible with all the major browsers and NodeJS. In addition, because it is implemented in [TypeScript](https://www.typescriptlang.org/), type definition files (`d.ts`) are bundled in the distribution. +This library serves as a comprehensive reference implementation of MessagePack for JavaScript with a focus on accuracy, compatibility, interoperability, and performance. + +Additionally, this is also a universal JavaScript library. It is compatible not only with browsers, but with Node.js or other JavaScript engines that implement ES2015+ standards. As it is written in [TypeScript](https://www.typescriptlang.org/), this library bundles up-to-date type definition files (`d.ts`). -*Note that this is the second version of MessagePack for JavaScript. The first version, which was implemented in ES5 and was never released to npmjs.com, is tagged as [classic](https://github.com/msgpack/msgpack-javascript/tree/classic).* +*Note that this is the second edition of "MessagePack for JavaScript". The first edition, which was implemented in ES5 and never released to npmjs.com, is tagged as [`classic`](https://github.com/msgpack/msgpack-javascript/tree/classic). ## Synopsis @@ -36,42 +36,44 @@ deepStrictEqual(decode(encoded), object); ## Table of Contents - - - [Synopsis](#synopsis) - [Table of Contents](#table-of-contents) - [Install](#install) - [API](#api) - - [`encode(data: unknown, options?: EncodeOptions): Uint8Array`](#encodedata-unknown-options-encodeoptions-uint8array) - - [`EncodeOptions`](#encodeoptions) - - [`decode(buffer: ArrayLike | ArrayBuffer, options?: DecodeOptions): unknown`](#decodebuffer-arraylikenumber--arraybuffer-options-decodeoptions-unknown) - - [`DecodeOptions`](#decodeoptions) - - [`decodeAsync(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): Promise`](#decodeasyncstream-asynciterablearraylikenumber--readablestreamarraylikenumber-options-decodeasyncoptions-promiseunknown) - - [`decodeArrayStream(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): AsyncIterable`](#decodearraystreamstream-asynciterablearraylikenumber--readablestreamarraylikenumber-options-decodeasyncoptions-asynciterableunknown) - - [`decodeStream(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): AsyncIterable`](#decodestreamstream-asynciterablearraylikenumber--readablestreamarraylikenumber-options-decodeasyncoptions-asynciterableunknown) - - [Extension Types](#extension-types) - - [Codec context](#codec-context) - - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) - - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) + - [`encode(data: unknown, options?: EncoderOptions): Uint8Array`](#encodedata-unknown-options-encoderoptions-uint8array) + - [`EncoderOptions`](#encoderoptions) + - [`decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decoderoptions-unknown) + - [`DecoderOptions`](#decoderoptions) + - [`decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator`](#decodemultibuffer-arraylikenumber--buffersource-options-decoderoptions-generatorunknown-void-unknown) + - [`decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): Promise`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-promiseunknown) + - [`decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) + - [`decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable`](#decodemultistreamstream-readablestreamlikearraylikenumber--buffersource-options-decoderoptions-asynciterableunknown) + - [Reusing Encoder and Decoder instances](#reusing-encoder-and-decoder-instances) +- [Extension Types](#extension-types) + - [ExtensionCodec context](#extensioncodec-context) + - [Handling BigInt with ExtensionCodec](#handling-bigint-with-extensioncodec) + - [The temporal module as timestamp extensions](#the-temporal-module-as-timestamp-extensions) +- [Faster way to decode a large array of floating point numbers](#faster-way-to-decode-a-large-array-of-floating-point-numbers) +- [Decoding a Blob](#decoding-a-blob) - [MessagePack Specification](#messagepack-specification) - [MessagePack Mapping Table](#messagepack-mapping-table) -- [Prerequsites](#prerequsites) +- [Prerequisites](#prerequisites) - [ECMA-262](#ecma-262) - [NodeJS](#nodejs) + - [TypeScript Compiler / Type Definitions](#typescript-compiler--type-definitions) - [Benchmark](#benchmark) - [Distribution](#distribution) - [NPM / npmjs.com](#npm--npmjscom) - [CDN / unpkg.com](#cdn--unpkgcom) +- [Deno Support](#deno-support) +- [Bun Support](#bun-support) - [Maintenance](#maintenance) - [Testing](#testing) - [Continuous Integration](#continuous-integration) - [Release Engineering](#release-engineering) - [Updating Dependencies](#updating-dependencies) -- [Big Thanks](#big-thanks) - [License](#license) - - ## Install This library is published to `npmjs.com` as [@msgpack/msgpack](https://www.npmjs.com/package/@msgpack/msgpack). @@ -82,9 +84,9 @@ npm install @msgpack/msgpack ## API -### `encode(data: unknown, options?: EncodeOptions): Uint8Array` +### `encode(data: unknown, options?: EncoderOptions): Uint8Array` -It encodes `data` and returns a byte array as `Uint8Array`, throwing errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`. +It encodes `data` into a single MessagePack-encoded object, and returns a byte array as `Uint8Array`. It throws errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`. for example: @@ -95,7 +97,7 @@ const encoded: Uint8Array = encode({ foo: "bar" }); console.log(encoded); ``` -If you'd like to convert the uint8array to a NodeJS `Buffer`, use `Buffer.from(arrayBuffer, offset, length)` in order not to copy the underlying `ArrayBuffer`, while `Buffer.from(uint8array)` copies it: +If you'd like to convert an `uint8array` to a NodeJS `Buffer`, use `Buffer.from(arrayBuffer, offset, length)` in order not to copy the underlying `ArrayBuffer`, while `Buffer.from(uint8array)` copies it: ```typescript import { encode } from "@msgpack/msgpack"; @@ -107,28 +109,32 @@ const buffer: Buffer = Buffer.from(encoded.buffer, encoded.byteOffset, encoded.b console.log(buffer); ``` -#### `EncodeOptions` +#### `EncoderOptions` + +| Name | Type | Default | +| ------------------- | -------------- | ----------------------------- | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| useBigInt64 | boolean | false | +| maxDepth | number | `100` | +| initialBufferSize | number | `2048` | +| sortKeys | boolean | false | +| forceFloat32 | boolean | false | +| forceIntegerToFloat | boolean | false | +| ignoreUndefined | boolean | false | -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensinCodec.defaultCodec` -maxDepth | number | `100` -initialBufferSize | number | `2048` -sortKeys | boolean | false -forceFloat32 | boolean | false -ignoreUndefined | boolean | false -context | user-defined | - +### `decode(buffer: ArrayLike | BufferSource, options?: DecoderOptions): unknown` -### `decode(buffer: ArrayLike | ArrayBuffer, options?: DecodeOptions): unknown` +It decodes `buffer` that includes a MessagePack-encoded object, and returns the decoded object typed `unknown`. -It decodes `buffer` encoded in MessagePack, and returns a decoded object as `uknown`. +`buffer` must be an array of bytes, which is typically `Uint8Array` or `ArrayBuffer`. `BufferSource` is defined as `ArrayBuffer | ArrayBufferView`. -`buffer` must be an array of bytes, which is typically `Uint8Array`, or `ArrayBuffer`. +The `buffer` must include a single encoded object. If the `buffer` includes extra bytes after an object or the `buffer` is empty, it throws `RangeError`. To decode `buffer` that includes multiple encoded objects, use `decodeMulti()` or `decodeMultiStream()` (recommended) instead. for example: ```typescript -import { encode } from "@msgpack/msgpack"; +import { decode } from "@msgpack/msgpack"; const encoded: Uint8Array; const object = decode(encoded); @@ -137,25 +143,50 @@ console.log(object); NodeJS `Buffer` is also acceptable because it is a subclass of `Uint8Array`. -#### `DecodeOptions` +#### `DecoderOptions` -Name|Type|Default -----|----|---- -extensionCodec | ExtensionCodec | `ExtensinCodec.defaultCodec` -maxStrLength | number | `4_294_967_295` (UINT32_MAX) -maxBinLength | number | `4_294_967_295` (UINT32_MAX) -maxArrayLength | number | `4_294_967_295` (UINT32_MAX) -maxMapLength | number | `4_294_967_295` (UINT32_MAX) -maxExtLength | number | `4_294_967_295` (UINT32_MAX) -context | user-defined | - +| Name | Type | Default | +| --------------- | ------------------- | ---------------------------------------------- | +| extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec` | +| context | user-defined | - | +| useBigInt64 | boolean | false | +| rawStrings | boolean | false | +| maxStrLength | number | `4_294_967_295` (UINT32_MAX) | +| maxBinLength | number | `4_294_967_295` (UINT32_MAX) | +| maxArrayLength | number | `4_294_967_295` (UINT32_MAX) | +| maxMapLength | number | `4_294_967_295` (UINT32_MAX) | +| maxExtLength | number | `4_294_967_295` (UINT32_MAX) | +| mapKeyConverter | MapKeyConverterType | throw exception if key is not string or number | + +`MapKeyConverterType` is defined as `(key: unknown) => string | number`. + +To skip UTF-8 decoding of strings, `rawStrings` can be set to `true`. In this case, strings are decoded into `Uint8Array`. You can use `max${Type}Length` to limit the length of each type decoded. -### `decodeAsync(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): Promise` +### `decodeMulti(buffer: ArrayLike | BufferSource, options?: DecoderOptions): Generator` + +It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. See also `decodeMultiStream()`, which is an asynchronous variant of this function. -It decodes `stream` in an async iterable of byte arrays, and returns decoded object as `uknown` type, wrapped in `Promise`. This function works asyncronously. +This function is not recommended to decode a MessagePack binary via I/O stream including sockets because it's synchronous. Instead, `decodeMultiStream()` decodes a binary stream asynchronously, typically spending less CPU and memory. -`DecodeAsyncOptions` is the same as `DecodeOptions` for `decode()`. +for example: + +```typescript +import { decode } from "@msgpack/msgpack"; + +const encoded: Uint8Array; + +for (const object of decodeMulti(encoded)) { + console.log(object); +} +``` + +### `decodeAsync(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): Promise` + +It decodes `stream`, where `ReadableStreamLike` is defined as `ReadableStream | AsyncIterable`, in an async iterable of byte arrays, and returns decoded object as `unknown` type, wrapped in `Promise`. + +This function works asynchronously, and might CPU resources more efficiently compared with synchronous `decode()`, because it doesn't wait for the completion of downloading. This function is designed to work with whatwg `fetch()` like this: @@ -172,16 +203,14 @@ if (contentType && contentType.startsWith(MSGPACK_TYPE) && response.body != null } else { /* handle errors */ } ``` -### `decodeArrayStream(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): AsyncIterable` - -It is alike to `decodeAsync()`, but only accepts an array of items as the input `stream`, and emits the decoded item one by one. +### `decodeArrayStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable` -It throws errors when the input is not an array-family. +It is alike to `decodeAsync()`, but only accepts a `stream` that includes an array of items, and emits a decoded item one by one. for example: ```typescript -import { encode } from "@msgpack/msgpack"; +import { decodeArrayStream } from "@msgpack/msgpack"; const stream: AsyncIterator; @@ -191,31 +220,54 @@ for await (const item of decodeArrayStream(stream)) { } ``` +### `decodeMultiStream(stream: ReadableStreamLike | BufferSource>, options?: DecoderOptions): AsyncIterable` -### `decodeStream(stream: AsyncIterable> | ReadableStream>, options?: DecodeAsyncOptions): AsyncIterable` - -It is alike to `decodeAsync()` and `decodeArrayStream()`, but the input `stream` consists of independent MessagePack items. +It is alike to `decodeAsync()` and `decodeArrayStream()`, but the input `stream` must consist of multiple MessagePack-encoded items. This is an asynchronous variant for `decodeMulti()`. -In other words, it decodes an unlimited stream and emits an item one by one. +In other words, it could decode an unlimited stream and emits a decoded item one by one. for example: ```typescript -import { encode } from "@msgpack/msgpack"; +import { decodeMultiStream } from "@msgpack/msgpack"; const stream: AsyncIterator; // in an async function: -for await (const item of decodeStream(stream)) { +for await (const item of decodeMultiStream(stream)) { console.log(item); } ``` -### Extension Types +This function is available since v2.4.0; previously it was called as `decodeStream()`. + +### Reusing Encoder and Decoder instances + +`Encoder` and `Decoder` classes are provided to have better performance by reusing instances: + +```typescript +import { deepStrictEqual } from "assert"; +import { Encoder, Decoder } from "@msgpack/msgpack"; + +const encoder = new Encoder(); +const decoder = new Decoder(); + +const encoded: Uint8Array = encoder.encode(object); +deepStrictEqual(decoder.decode(encoded), object); +``` + +According to our benchmark, reusing `Encoder` instance is about 20% faster +than `encode()` function, and reusing `Decoder` instance is about 2% faster +than `decode()` function. Note that the result should vary in environments +and data structure. + +`Encoder` and `Decoder` take the same options as `encode()` and `decode()` respectively. + +## Extension Types To handle [MessagePack Extension Types](https://github.com/msgpack/msgpack/blob/master/spec.md#extension-types), this library provides `ExtensionCodec` class. -Here is an example to setup custom extension types that handles `Map` and `Set` classes in TypeScript: +This is an example to setup custom extension types that handles `Map` and `Set` classes in TypeScript: ```typescript import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; @@ -228,46 +280,45 @@ extensionCodec.register({ type: SET_EXT_TYPE, encode: (object: unknown): Uint8Array | null => { if (object instanceof Set) { - return encode([...object]); + return encode([...object], { extensionCodec }); } else { return null; } }, decode: (data: Uint8Array) => { - const array = decode(data) as Array; + const array = decode(data, { extensionCodec }) as Array; return new Set(array); }, }); -// Map +// Map const MAP_EXT_TYPE = 1; // Any in 0-127 extensionCodec.register({ type: MAP_EXT_TYPE, encode: (object: unknown): Uint8Array => { if (object instanceof Map) { - return encode([...object]); + return encode([...object], { extensionCodec }); } else { return null; } }, decode: (data: Uint8Array) => { - const array = decode(data) as Array<[unknown, unknown]>; + const array = decode(data, { extensionCodec }) as Array<[unknown, unknown]>; return new Map(array); }, }); -// and later -import { encode, decode } from "@msgpack/msgpack"; - -const encoded = = encode([new Set(), new Map()], { extensionCodec }); +const encoded = encode([new Set(), new Map()], { extensionCodec }); const decoded = decode(encoded, { extensionCodec }); ``` -Not that extension types for custom objects must be `[0, 127]`, while `[-1, -128]` is reserved for MessagePack itself. +Ensure you include your extensionCodec in any recursive encode and decode statements! -#### Codec context +Note that extension types for custom objects must be `[0, 127]`, while `[-1, -128]` is reserved for MessagePack itself. -When using an extension codec, it may be necessary to keep encoding/decoding state, to keep track of which objects got encoded/re-created. To do this, pass a `context` to the `EncodeOptions` and `DecodeOptions` (and if using typescript, type the `ExtensionCodec` too). Don't forget to pass the `{extensionCodec, context}` along recursive encoding/decoding: +### ExtensionCodec context + +When you use an extension codec, it might be necessary to have encoding/decoding state to keep track of which objects got encoded/re-created. To do this, pass a `context` to the `EncoderOptions` and `DecoderOptions`: ```typescript import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; @@ -286,7 +337,7 @@ extensionCodec.register({ type: MYTYPE_EXT_TYPE, encode: (object, context) => { if (object instanceof MyType) { - context.track(object); // <-- like this + context.track(object); return encode(object.toJSON(), { extensionCodec, context }); } else { return null; @@ -295,7 +346,7 @@ extensionCodec.register({ decode: (data, extType, context) => { const decoded = decode(data, { extensionCodec, context }); const my = new MyType(decoded); - context.track(my); // <-- and like this + context.track(my); return my; }, }); @@ -305,40 +356,60 @@ import { encode, decode } from "@msgpack/msgpack"; const context = new MyContext(); -const encoded = = encode({myType: new MyType()}, { extensionCodec, context }); +const encoded = encode({ myType: new MyType() }, { extensionCodec, context }); const decoded = decode(encoded, { extensionCodec, context }); ``` -#### Handling BigInt with ExtensionCodec +### Handling BigInt with ExtensionCodec + +This library does not handle BigInt by default, but you have two options to handle it: + +* Set `useBigInt64: true` to map bigint to MessagePack's int64/uint64 +* Define a custom `ExtensionCodec` to map bigint to a MessagePack's extension type -This library does not handle BigInt by default, but you can handle it with `ExtensionCodec` like this: +`useBigInt64: true` is the simplest way to handle bigint, but it has limitations: + +* A bigint is encoded in 8 byte binaries even if it's a small integer +* A bigint must be smaller than the max value of the uint64 and larger than the min value of the int64. Otherwise the behavior is undefined. + +So you might want to define a custom codec to handle bigint like this: ```typescript import { deepStrictEqual } from "assert"; -import { encode, decode, ExtensionCodec } from "@msgpack/msgpack"; +import { encode, decode, ExtensionCodec, DecodeError } from "@msgpack/msgpack"; +// to define a custom codec: const BIGINT_EXT_TYPE = 0; // Any in 0-127 const extensionCodec = new ExtensionCodec(); extensionCodec.register({ type: BIGINT_EXT_TYPE, - encode: (input: unknown) => { + encode(input: unknown): Uint8Array | null { if (typeof input === "bigint") { - return encode(input.toString()); + if (input <= Number.MAX_SAFE_INTEGER && input >= Number.MIN_SAFE_INTEGER) { + return encode(Number(input)); + } else { + return encode(String(input)); + } } else { return null; } }, - decode: (data: Uint8Array) => { - return BigInt(decode(data)); + decode(data: Uint8Array): bigint { + const val = decode(data); + if (!(typeof val === "string" || typeof val === "number")) { + throw new DecodeError(`unexpected BigInt source: ${val} (${typeof val})`); + } + return BigInt(val); }, }); +// to use it: const value = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); -const encoded: = encode(value, { extensionCodec }); +const encoded = encode(value, { extensionCodec }); deepStrictEqual(decode(encoded, { extensionCodec }), value); ``` -#### The temporal module as timestamp extensions +### The temporal module as timestamp extensions There is a proposal for a new date/time representations in JavaScript: @@ -358,10 +429,11 @@ import { decodeTimestampToTimeSpec, } from "@msgpack/msgpack"; +// to define a custom codec const extensionCodec = new ExtensionCodec(); extensionCodec.register({ type: EXT_TIMESTAMP, // override the default behavior! - encode: (input: any) => { + encode(input: unknown): Uint8Array | null { if (input instanceof Instant) { const sec = input.seconds; const nsec = Number(input.nanoseconds - BigInt(sec) * BigInt(1e9)); @@ -370,7 +442,7 @@ extensionCodec.register({ return null; } }, - decode: (data: Uint8Array) => { + decode(data: Uint8Array): Instant { const timeSpec = decodeTimestampToTimeSpec(data); const sec = BigInt(timeSpec.sec); const nsec = BigInt(timeSpec.nsec); @@ -378,13 +450,75 @@ extensionCodec.register({ }, }); +// to use it const instant = Instant.fromEpochMilliseconds(Date.now()); const encoded = encode(instant, { extensionCodec }); const decoded = decode(encoded, { extensionCodec }); deepStrictEqual(decoded, instant); ``` -This will be default once the temporal module is standardizied, which is not a near-future, though. +This will become default in this library with major-version increment, if the temporal module is standardized. + +## Faster way to decode a large array of floating point numbers + +If there are large arrays of floating point numbers in your payload, there +is a way to decode it faster: define a custom extension type for `Float#Array` +with alignment. + +An extension type's `encode` method can return a function that takes a parameter +`pos: number`. This parameter can be used to make alignment of the buffer, +resulting decoding it much more performant. + +See an example implementation for `Float32Array`: + +```typescript +const extensionCodec = new ExtensionCodec(); + +const EXT_TYPE_FLOAT32ARRAY = 0; // Any in 0-127 +extensionCodec.register({ + type: EXT_TYPE_FLOAT32ARRAY, + encode: (object: unknown) => { + if (object instanceof Float32Array) { + return (pos: number) => { + const bpe = Float32Array.BYTES_PER_ELEMENT; + const padding = 1 + ((bpe - ((pos + 1) % bpe)) % bpe); + const data = new Uint8Array(object.buffer); + const result = new Uint8Array(padding + data.length); + result[0] = padding; + result.set(data, padding); + return result; + }; + } + return null; + }, + decode: (data: Uint8Array) => { + const padding = data[0]!; + const bpe = Float32Array.BYTES_PER_ELEMENT; + const offset = data.byteOffset + padding; + const length = data.byteLength - padding; + return new Float32Array(data.buffer, offset, length / bpe); + }, +}); +``` + +## Decoding a Blob + +[`Blob`](https://developer.mozilla.org/en-US/docs/Web/API/Blob) is a binary data container provided by browsers. To read its contents when it contains a MessagePack binary, you can use `Blob#arrayBuffer()` or `Blob#stream()`. `Blob#stream()` +is recommended if your target platform support it. This is because streaming +decode should be faster for large objects. In both ways, you need to use +asynchronous API. + +```typescript +async function decodeFromBlob(blob: Blob): unknown { + if (blob.stream) { + // Blob#stream(): ReadableStream (recommended) + return await decodeAsync(blob.stream()); + } else { + // Blob#arrayBuffer(): Promise (if stream() is not available) + return decode(await blob.arrayBuffer()); + } +} +``` ## MessagePack Specification @@ -394,7 +528,7 @@ This library is compatible with the "August 2017" revision of MessagePack specif * [x] extension types, added at August 2013 * [x] timestamp ext type, added at August 2017 -The livinng specification is here: +The living specification is here: https://github.com/msgpack/msgpack @@ -404,59 +538,97 @@ Note that as of June 2019 there're no official "version" on the MessagePack spec The following table shows how JavaScript values are mapped to [MessagePack formats](https://github.com/msgpack/msgpack/blob/master/spec.md) and vice versa. -Source Value|MessagePack Format|Value Decoded -----|----|---- -null, undefined|nil|null (*1) -boolean (true, false)|bool family|boolean (true, false) -number (53-bit int)|int family|number (53-bit int) -number (64-bit float)|float family|number (64-bit float) -string|str family|string -ArrayBufferView |bin family|Uint8Array (*2) -Array|array family|Array -Object|map family|Object (*3) -Date|timestamp ext family|Date (*4) +The mapping of integers varies on the setting of `useBigInt64`. + +The default, `useBigInt64: false` is: + +| Source Value | MessagePack Format | Value Decoded | +| --------------------- | -------------------- | --------------------- | +| null, undefined | nil | null (*1) | +| boolean (true, false) | bool family | boolean (true, false) | +| number (53-bit int) | int family | number | +| number (64-bit float) | float family | number | +| string | str family | string (*2) | +| ArrayBufferView | bin family | Uint8Array (*3) | +| Array | array family | Array | +| Object | map family | Object (*4) | +| Date | timestamp ext family | Date (*5) | +| bigint | N/A | N/A (*6) | * *1 Both `null` and `undefined` are mapped to `nil` (`0xC0`) type, and are decoded into `null` -* *2 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` -* *3 In handling `Object`, it is regarded as `Record` in terms of TypeScript -* *4 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. +* *2 If you'd like to skip UTF-8 decoding of strings, set `rawStrings: true`. In this case, strings are decoded into `Uint8Array`. +* *3 Any `ArrayBufferView`s including NodeJS's `Buffer` are mapped to `bin` family, and are decoded into `Uint8Array` +* *4 In handling `Object`, it is regarded as `Record` in terms of TypeScript +* *5 MessagePack timestamps may have nanoseconds, which will lost when it is decoded into JavaScript `Date`. This behavior can be overridden by registering `-1` for the extension codec. +* *6 bigint is not supported in `useBigInt64: false` mode, but you can define an extension codec for it. + +If you set `useBigInt64: true`, the following mapping is used: + +| Source Value | MessagePack Format | Value Decoded | +| --------------------------------- | -------------------- | --------------------- | +| null, undefined | nil | null | +| boolean (true, false) | bool family | boolean (true, false) | +| **number (32-bit int)** | int family | number | +| **number (except for the above)** | float family | number | +| **bigint** | int64 / uint64 | bigint (*7) | +| string | str family | string | +| ArrayBufferView | bin family | Uint8Array | +| Array | array family | Array | +| Object | map family | Object | +| Date | timestamp ext family | Date | -## Prerequsites + +* *7 If the bigint is larger than the max value of uint64 or smaller than the min value of int64, then the behavior is undefined. + +## Prerequisites This is a universal JavaScript library that supports major browsers and NodeJS. ### ECMA-262 -* ES5 language features -* ES2018 standard library, including: +* ES2015 language features +* ES2024 standard library, including: * Typed arrays (ES2015) * Async iterations (ES2018) - * Features added in ES2015-ES2018 + * Features added in ES2015-ES2022 +* whatwg encodings (`TextEncoder` and `TextDecoder`) + +ES2022 standard library used in this library can be polyfilled with [core-js](https://github.com/zloirock/core-js). -ES2018 standard library used in this library can be polyfilled with [core-js](https://github.com/zloirock/core-js). +IE11 is no longer supported. If you'd like to use this library in IE11, use v2.x versions. -If you support IE11, import `core-js` in your application entrypoints, as this library does in testing for browsers . ### NodeJS -NodeJS v10 is required, but NodeJS v12 or later is recommended because it includes the V8 feature of [Improving DataView performance in V8](https://v8.dev/blog/dataview). +NodeJS v18 is required. -NodeJS before v10 will work by importing `@msgpack/msgpack/dist.es5/msgpack`. +### TypeScript Compiler / Type Definitions + +This module requires type definitions of `AsyncIterator`, `ArrayBufferLike`, whatwg streams, and so on. They are provided by `"lib": ["ES2024", "DOM"]` in `tsconfig.json`. + +Regarding the TypeScript compiler version, only the latest TypeScript is tested in development. ## Benchmark -Benchmark on NodeJS/v12.14.1 +Run-time performance is not the only reason to use MessagePack, but it's important to choose MessagePack libraries, so a benchmark suite is provided to monitor the performance of this library. + +V8's built-in JSON has been improved for years, esp. `JSON.parse()` is [significantly improved in V8/7.6](https://v8.dev/blog/v8-release-76), it is the fastest deserializer as of 2019, as the benchmark result bellow suggests. + +However, MessagePack can handles binary data effectively, actual performance depends on situations. Esp. streaming-decoding may be significantly faster than non-streaming decoding if it's effective. You'd better take benchmark on your own use-case if performance matters. + +Benchmark on NodeJS/v22.13.1 (V8/12.4) -operation | op | ms | op/s ------------------------------------------------------------------ | ------: | ----: | ------: -buf = Buffer.from(JSON.stringify(obj)); | 627500 | 5000 | 125500 -buf = JSON.stringify(obj); | 1095600 | 5000 | 219120 -obj = JSON.parse(buf); | 1469100 | 5000 | 293820 -buf = require("msgpack-lite").encode(obj); | 541800 | 5000 | 108360 -obj = require("msgpack-lite").decode(buf); | 315200 | 5000 | 63040 -buf = require("@msgpack/msgpack").encode(obj); | 815200 | 5000 | 163040 -obj = require("@msgpack/msgpack").decode(buf); | 764600 | 5001 | 152889 +| operation | op | ms | op/s | +| ------------------------------------------------- | ------: | ---: | -----: | +| buf = Buffer.from(JSON.stringify(obj)); | 1348700 | 5000 | 269740 | +| obj = JSON.parse(buf.toString("utf-8")); | 1700300 | 5000 | 340060 | +| buf = require("msgpack-lite").encode(obj); | 591300 | 5000 | 118260 | +| obj = require("msgpack-lite").decode(buf); | 539500 | 5000 | 107900 | +| buf = require("@msgpack/msgpack").encode(obj); | 1238700 | 5000 | 247740 | +| obj = require("@msgpack/msgpack").decode(buf); | 1402000 | 5000 | 280400 | +| buf = /* @msgpack/msgpack */ encoder.encode(obj); | 1379800 | 5000 | 275960 | +| obj = /* @msgpack/msgpack */ decoder.decode(buf); | 1406100 | 5000 | 281220 | -Note that `Buffer.from()` for `JSON.stringify()` is added to emulate I/O where a JavaScript string must be converted into a byte array encoded in UTF-8, whereas MessagePack's `encode()` returns a byte array. +Note that `JSON` cases use `Buffer` to emulate I/O where a JavaScript string must be converted into a byte array encoded in UTF-8, whereas MessagePack modules deal with byte arrays. ## Distribution @@ -464,16 +636,17 @@ Note that `Buffer.from()` for `JSON.stringify()` is added to emulate I/O where a The NPM package distributed in npmjs.com includes both ES2015+ and ES5 files: -* `dist/` is compiled into ES2015+ -* `dist.es5/` is compiled into ES5 and bundled to singile file - * `dist.es5/msgpack.min.js` - the default, minified file (UMD) - * `dist.es5/msgpack.js` - an optional, non-minified file (UMD) +* `dist/` is compiled into ES2020 with CommomJS, provided for NodeJS v10 +* `dist.umd/` is compiled into ES5 with UMD + * `dist.umd/msgpack.min.js` - the minified file + * `dist.umd/msgpack.js` - the non-minified file +* `dist.esm/` is compiled into ES2020 with ES modules, provided for webpack-like bundlers and NodeJS's ESM-mode If you use NodeJS and/or webpack, their module resolvers use the suitable one automatically. ### CDN / unpkg.com -This library is availble via CDN: +This library is available via CDN: ```html @@ -481,6 +654,19 @@ This library is availble via CDN: It loads `MessagePack` module to the global object. + +## Deno Support + +You can use this module on Deno. + +See `example/deno-*.ts` for examples. + +`deno.land/x` is not supported. + +## Bun Support + +You can use this module on Bun. + ## Maintenance ### Testing @@ -493,16 +679,16 @@ npm run test ### Continuous Integration -This library uses Travis CI. - -test matrix: +This library uses GitHub Actions. -* TypeScript targets - * `target=es2019` / `target=es5` -* JavaScript engines - * NodeJS, borwsers (Chrome, Firefox, Safari, IE11, and so on) +Test matrix: -See [test:* in package.json](./package.json) and [.travis.yml](./.travis.yml) for details. +* NodeJS + * v18 / v20 / v22 +* Browsers: + * Chrome, Firefox +* Deno +* Bun ### Release Engineering @@ -526,12 +712,6 @@ make publish npm run update-dependencies ``` -## Big Thanks - -Cross-browser Testing Platform and Open Source <3 Provided by Sauce Labs. - -Sauce Labs - ## License Copyright 2019 The MessagePack community. diff --git a/assets/SauceLabs.svg b/assets/SauceLabs.svg deleted file mode 100644 index 4dd11641..00000000 --- a/assets/SauceLabs.svg +++ /dev/null @@ -1 +0,0 @@ -Powered by Sauce Labs badges whiteTESTING POWERED BY \ No newline at end of file diff --git a/benchmark/benchmark-from-msgpack-lite.ts b/benchmark/benchmark-from-msgpack-lite.ts index 2bd9463e..40ac6f9b 100644 --- a/benchmark/benchmark-from-msgpack-lite.ts +++ b/benchmark/benchmark-from-msgpack-lite.ts @@ -1,14 +1,13 @@ /* eslint-disable */ // original: https://raw.githubusercontent.com/kawanet/msgpack-lite/master/lib/benchmark.js -var msgpack_msgpack = require("../src"); +var msgpack_msgpack = require("../src/index.ts"); var msgpack_node = try_require("msgpack"); var msgpack_lite = try_require("msgpack-lite"); var msgpack_js = try_require("msgpack-js"); -var msgpack_js_v5 = try_require("msgpack-js-v5"); +var msgpackr = try_require("msgpackr"); var msgpack5 = try_require("msgpack5"); -var msgpack_unpack = try_require("msgpack-unpack"); var notepack = try_require("notepack"); msgpack5 = msgpack5 && msgpack5(); @@ -34,7 +33,8 @@ var COL2 = 7; var COL3 = 5; var COL4 = 7; -console.log(`Benchmark on NodeJS/${process.version}\n`) +const v8version = process.versions.v8.split(/\./, 2).join('.'); +console.log(`Benchmark on NodeJS/${process.version} (V8/${v8version})\n`) console.log(rpad("operation", COL1), "|", " op ", "|", " ms ", "|", " op/s "); console.log(rpad("", COL1, "-"), "|", lpad(":", COL2, "-"), "|", lpad(":", COL3, "-"), "|", lpad(":", COL4, "-")); @@ -42,8 +42,7 @@ var buf, obj; if (JSON) { buf = bench('buf = Buffer.from(JSON.stringify(obj));', JSON_stringify, data); - buf = bench('buf = JSON.stringify(obj);', JSON.stringify, data); - obj = bench('obj = JSON.parse(buf);', JSON.parse, buf); + obj = bench('obj = JSON.parse(buf.toString("utf-8"));', JSON_parse, buf); runTest(obj); } @@ -63,11 +62,22 @@ if (msgpack_msgpack) { buf = bench('buf = require("@msgpack/msgpack").encode(obj);', msgpack_msgpack.encode, data); obj = bench('obj = require("@msgpack/msgpack").decode(buf);', msgpack_msgpack.decode, buf); runTest(obj); + + const encoder = new msgpack_msgpack.Encoder(); + const decoder = new msgpack_msgpack.Decoder(); + buf = bench('buf = /* @msgpack/msgpack */ encoder.encode(obj);', (data) => encoder.encode(data), data); + obj = bench('obj = /* @msgpack/msgpack */ decoder.decode(buf);', (buf) => decoder.decode(buf), buf); + runTest(obj); + + if (process.env["CACHE_HIT_RATE"]) { + const {hit, miss} = decoder.keyDecoder; + console.log(`CACHE_HIT_RATE: cache hit rate in CachedKeyDecoder: hit=${hit}, miss=${miss}, hit rate=${hit / (hit + miss)}`); + } } -if (msgpack_js_v5) { - buf = bench('buf = require("msgpack-js-v5").encode(obj);', msgpack_js_v5.encode, data); - obj = bench('obj = require("msgpack-js-v5").decode(buf);', msgpack_js_v5.decode, buf); +if (msgpackr) { + buf = bench('buf = require("msgpackr").pack(obj);', msgpackr.pack, data); + obj = bench('obj = require("msgpackr").unpack(buf);', msgpackr.unpack, buf); runTest(obj); } @@ -89,13 +99,12 @@ if (notepack) { runTest(obj); } -if (msgpack_unpack) { - obj = bench('obj = require("msgpack-unpack").decode(buf);', msgpack_unpack, packed); - runTest(obj); +function JSON_stringify(src: any): Buffer { + return Buffer.from(JSON.stringify(src)); } -function JSON_stringify(src: any) { - return Buffer.from(JSON.stringify(src)); +function JSON_parse(json: Buffer): any { + return JSON.parse(json.toString("utf-8")); } function bench(name: string, func: (...args: any[]) => any, src: any) { diff --git a/benchmark/decode-string.ts b/benchmark/decode-string.ts index 7e57d043..3e6bfbb1 100644 --- a/benchmark/decode-string.ts +++ b/benchmark/decode-string.ts @@ -1,11 +1,11 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8DecodeJs, utf8DecodeTD } from "../src/utils/utf8.ts"; // @ts-ignore import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 100, 500, 1_000].map((n) => { return baseStr.repeat(n); }); @@ -14,7 +14,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const bytes = new Uint8Array(new ArrayBuffer(byteLength)); utf8EncodeJs(str, bytes, 0); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/benchmark/encode-string.ts b/benchmark/encode-string.ts index 47662dd1..df1b2835 100644 --- a/benchmark/encode-string.ts +++ b/benchmark/encode-string.ts @@ -1,11 +1,11 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8EncodeTE } from "../src/utils/utf8.ts"; // @ts-ignore import Benchmark from "benchmark"; for (const baseStr of ["A", "あ", "🌏"]) { - const dataSet = [10, 100, 200, 1_000, 10_000, 100_000].map((n) => { + const dataSet = [10, 30, 50, 100].map((n) => { return baseStr.repeat(n); }); @@ -13,7 +13,7 @@ for (const baseStr of ["A", "あ", "🌏"]) { const byteLength = utf8Count(str); const buffer = new Uint8Array(byteLength); - console.log(`\n## string "${baseStr}" x ${str.length} (byteLength=${byteLength})\n`); + console.log(`\n## string "${baseStr}" (strLength=${str.length}, byteLength=${byteLength})\n`); const suite = new Benchmark.Suite(); diff --git a/benchmark/key-decoder.ts b/benchmark/key-decoder.ts index 594bbab2..dec38f30 100644 --- a/benchmark/key-decoder.ts +++ b/benchmark/key-decoder.ts @@ -1,9 +1,11 @@ /* eslint-disable no-console */ -import { utf8EncodeJs, utf8Count, utf8DecodeJs } from "../src/utils/utf8"; +import { utf8EncodeJs, utf8Count, utf8DecodeJs } from "../src/utils/utf8.ts"; +import { CachedKeyDecoder } from "../src/CachedKeyDecoder.ts"; + +import data from "./benchmark-from-msgpack-lite-data.json" with { type: "json" }; // @ts-ignore import Benchmark from "benchmark"; -import { CachedKeyDecoder } from "../src/CachedKeyDecoder"; type InputType = { bytes: Uint8Array; @@ -11,7 +13,7 @@ type InputType = { str: string; }; -const keys: Array = Object.keys(require("./benchmark-from-msgpack-lite-data.json")).map((str) => { +const keys: Array = Object.keys(data).map((str) => { const byteLength = utf8Count(str); const bytes = new Uint8Array(new ArrayBuffer(byteLength)); utf8EncodeJs(str, bytes, 0); diff --git a/benchmark/msgpack-benchmark.js b/benchmark/msgpack-benchmark.js index 60635fdc..2fd90a57 100644 --- a/benchmark/msgpack-benchmark.js +++ b/benchmark/msgpack-benchmark.js @@ -3,13 +3,65 @@ "use strict"; require("ts-node/register"); const Benchmark = require("benchmark"); -const fs = require("fs"); -const msgpack = require("../src"); + +const msgpackEncode = require("..").encode; +const msgpackDecode = require("..").decode; +const ExtensionCodec = require("..").ExtensionCodec; + +const float32ArrayExtensionCodec = new ExtensionCodec(); +float32ArrayExtensionCodec.register({ + type: 0x01, + encode: (object) => { + if (object instanceof Float32Array) { + return new Uint8Array(object.buffer, object.byteOffset, object.byteLength); + } + return null; + }, + decode: (data) => { + const copy = new Uint8Array(data.byteLength); + copy.set(data); + return new Float32Array(copy.buffer); + }, +}); + +const float32ArrayZeroCopyExtensionCodec = new ExtensionCodec(); +float32ArrayZeroCopyExtensionCodec.register({ + type: 0x01, + encode: (object) => { + if (object instanceof Float32Array) { + return (pos) => { + const bpe = Float32Array.BYTES_PER_ELEMENT; + const padding = 1 + ((bpe - ((pos + 1) % bpe)) % bpe); + const data = new Uint8Array(object.buffer); + const result = new Uint8Array(padding + data.length); + result[0] = padding; + result.set(data, padding); + return result; + }; + } + return null; + }, + decode: (data) => { + const padding = data[0]; + const bpe = Float32Array.BYTES_PER_ELEMENT; + const offset = data.byteOffset + padding; + const length = data.byteLength - padding; + return new Float32Array(data.buffer, offset, length / bpe); + }, +}); const implementations = { "@msgpack/msgpack": { - encode: require("..").encode, - decode: require("..").decode, + encode: msgpackEncode, + decode: msgpackDecode, + }, + "@msgpack/msgpack (Float32Array extension)": { + encode: (data) => msgpackEncode(data, { extensionCodec: float32ArrayExtensionCodec }), + decode: (data) => msgpackDecode(data, { extensionCodec: float32ArrayExtensionCodec }), + }, + "@msgpack/msgpack (Float32Array with zero-copy extension)": { + encode: (data) => msgpackEncode(data, { extensionCodec: float32ArrayZeroCopyExtensionCodec }), + decode: (data) => msgpackDecode(data, { extensionCodec: float32ArrayZeroCopyExtensionCodec }), }, "msgpack-lite": { encode: require("msgpack-lite").encode, @@ -21,28 +73,52 @@ const implementations = { }, }; -// exactly the same as: -// https://raw.githubusercontent.com/endel/msgpack-benchmark/master/sample-large.json -const sampleFiles = ["./sample-large.json"]; +const samples = [ + { + // exactly the same as: + // https://raw.githubusercontent.com/endel/msgpack-benchmark/master/sample-large.json + name: "./sample-large.json", + data: require("./sample-large.json"), + }, + { + name: "Large array of numbers", + data: [ + { + position: new Array(1e3).fill(1.14), + }, + ], + }, + { + name: "Large Float32Array", + data: [ + { + position: new Float32Array(1e3).fill(1.14), + }, + ], + }, +]; function validate(name, data, encoded) { - if (JSON.stringify(data) !== JSON.stringify(implementations[name].decode(encoded))) { - throw new Error("Bad implementation: " + name); - } + return JSON.stringify(data) === JSON.stringify(implementations[name].decode(encoded)); } -for (const sampleFile of sampleFiles) { - const data = require(sampleFile); +for (const sample of samples) { + const { name: sampleName, data } = sample; const encodeSuite = new Benchmark.Suite(); const decodeSuite = new Benchmark.Suite(); console.log(""); - console.log("**" + sampleFile + ":** (" + JSON.stringify(data).length + " bytes in JSON)"); + console.log("**" + sampleName + ":** (" + JSON.stringify(data).length + " bytes in JSON)"); console.log(""); for (const name of Object.keys(implementations)) { implementations[name].toDecode = implementations[name].encode(data); - validate(name, data, implementations[name].toDecode); + if (!validate(name, data, implementations[name].toDecode)) { + console.log("```"); + console.log("Not supported by " + name); + console.log("```"); + continue; + } encodeSuite.add("(encode) " + name, () => { implementations[name].encode(data); }); @@ -60,7 +136,7 @@ for (const sampleFile of sampleFiles) { console.log(""); - decodeSuite.on("cycle", function(event) { + decodeSuite.on("cycle", (event) => { console.log(String(event.target)); }); diff --git a/benchmark/package.json b/benchmark/package.json index 692d58cb..ddc30b8c 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -2,9 +2,14 @@ "name": "@msgpack/msgpack-benchmark", "private": true, "version": "0.0.0", + "type": "module", + "scripts": { + "update-dependencies": "npx rimraf node_modules/ package-lock.json ; npm install ; npm audit fix --force ; git restore package.json ; npm install" + }, "dependencies": { - "benchmark": "^2.1.4", - "msgpack-lite": "^0.1.26", - "notepack.io": "^2.2.0" + "benchmark": "latest", + "msgpack-lite": "latest", + "msgpackr": "latest", + "notepack.io": "latest" } } diff --git a/benchmark/profile-decode.ts b/benchmark/profile-decode.ts index 081ddab2..512d87e0 100644 --- a/benchmark/profile-decode.ts +++ b/benchmark/profile-decode.ts @@ -1,4 +1,4 @@ -import { encode, decode, decodeAsync } from "../src"; +import { encode, decode, decodeAsync } from "../src/index.ts"; // @ts-ignore import _ from "lodash"; const data = require("./benchmark-from-msgpack-lite-data.json"); diff --git a/benchmark/profile-encode.ts b/benchmark/profile-encode.ts index 6efcf017..245e71b5 100644 --- a/benchmark/profile-encode.ts +++ b/benchmark/profile-encode.ts @@ -1,4 +1,4 @@ -import { encode } from "../src"; +import { encode } from "../src/index.ts"; // @ts-ignore import _ from "lodash"; diff --git a/benchmark/sample-large.json b/benchmark/sample-large.json index 050bfbdd..cd393880 100644 --- a/benchmark/sample-large.json +++ b/benchmark/sample-large.json @@ -16,7 +16,9 @@ "published":true, "title":"The Ruby Rogues", "updated_at":"2015-11-15T22:50:06.565Z", - "url":"http://feeds.feedwrench.com/RubyRogues.rss" + "url":"http://feeds.feedwrench.com/RubyRogues.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"56490d6ad9275a00030000eb", @@ -35,7 +37,9 @@ "published":true, "title":"Grok Podcast", "updated_at":"2015-11-15T22:55:47.498Z", - "url":"http://www.grokpodcast.com/atom.xml" + "url":"http://www.grokpodcast.com/atom.xml", + "score1": 100, + "score2": 0.1 }, { "_id":"564a1c30b1191d0003000000", @@ -53,7 +57,9 @@ "published":true, "title":"The Web Platform Podcast", "updated_at":"2015-11-16T18:11:02.022Z", - "url":"http://thewebplatform.libsyn.com//rss" + "url":"http://thewebplatform.libsyn.com//rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a1de3b1191d0003000047", @@ -72,7 +78,9 @@ "published":true, "title":"Developer Tea", "updated_at":"2015-11-16T23:00:23.224Z", - "url":"http://feeds.feedburner.com/developertea" + "url":"http://feeds.feedburner.com/developertea", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3163e51cc0000300004c", @@ -88,7 +96,9 @@ "published":true, "title":"Remote Conferences - Audio", "updated_at":"2015-11-16T19:41:24.367Z", - "url":"http://feeds.feedwrench.com/remoteconfs-audio.rss" + "url":"http://feeds.feedwrench.com/remoteconfs-audio.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a315de51cc00003000000", @@ -108,7 +118,9 @@ "published":true, "title":"The Freelancers' Show", "updated_at":"2015-11-16T19:41:27.459Z", - "url":"http://feeds.feedwrench.com/TheFreelancersShow.rss" + "url":"http://feeds.feedwrench.com/TheFreelancersShow.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3169e51cc000030000cd", @@ -124,7 +136,9 @@ "published":true, "title":"React Native Radio", "updated_at":"2015-11-16T19:41:29.999Z", - "url":"http://feeds.feedwrench.com/react-native-radio.rss" + "url":"http://feeds.feedwrench.com/react-native-radio.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a316fe51cc000030000d4", @@ -142,7 +156,9 @@ "published":true, "title":"The iPhreaks Show", "updated_at":"2015-11-16T19:41:43.700Z", - "url":"http://feeds.feedwrench.com/iPhreaks.rss" + "url":"http://feeds.feedwrench.com/iPhreaks.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3184e51cc00003000156", @@ -161,7 +177,9 @@ "published":true, "title":"JavaScript Jabber", "updated_at":"2015-11-16T19:42:24.692Z", - "url":"http://feeds.feedwrench.com/JavaScriptJabber.rss" + "url":"http://feeds.feedwrench.com/JavaScriptJabber.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a31dee51cc00003000210", @@ -177,7 +195,9 @@ "published":true, "title":"Web Security Warriors", "updated_at":"2015-11-16T19:43:28.133Z", - "url":"http://feeds.feedwrench.com/websecwarriors.rss" + "url":"http://feeds.feedwrench.com/websecwarriors.rss", + "score1": 100, + "score2": 0.1 }, { "_id":"564a3ddbe51cc00003000217", @@ -194,7 +214,9 @@ "published":true, "title":"Podcast NowLoading", "updated_at":"2015-11-16T23:00:23.963Z", - "url":"http://feeds.feedburner.com/podcastnowloading" + "url":"http://feeds.feedburner.com/podcastnowloading", + "score1": 100, + "score2": 0.1 }, { "_id":"564b9cfe08602e00030000fa", @@ -210,7 +232,9 @@ "published":true, "title":"Being Boss // A Podcast for Creative Entrepreneurs", "updated_at":"2015-11-17T21:32:50.672Z", - "url":"http://www.lovebeingboss.com/RSSRetrieve.aspx?ID=18365\u0026Type=RSS20" + "url":"http://www.lovebeingboss.com/RSSRetrieve.aspx?ID=18365\u0026Type=RSS20", + "score1": 100, + "score2": 0.1 }, { "_id":"564c5c8008602e0003000128", @@ -226,6 +250,8 @@ "published":true, "title":"Nerdcast", "updated_at":"2015-11-18T11:11:20.034Z", - "url":"http://jovemnerd.com.br/categoria/nerdcast/feed/" + "url":"http://jovemnerd.com.br/categoria/nerdcast/feed/", + "score1": 100, + "score2": 0.1 } ] diff --git a/benchmark/string.ts b/benchmark/string.ts index b801394b..4f11f307 100644 --- a/benchmark/string.ts +++ b/benchmark/string.ts @@ -1,5 +1,5 @@ /* eslint-disable no-console */ -import { encode, decode } from "../src"; +import { encode, decode } from "../src/index.ts"; const ascii = "A".repeat(40000); const emoji = "🌏".repeat(20000); diff --git a/benchmark/sync-vs-async.ts b/benchmark/sync-vs-async.ts index 69819321..620438ef 100644 --- a/benchmark/sync-vs-async.ts +++ b/benchmark/sync-vs-async.ts @@ -1,12 +1,14 @@ -#!ts-node +#!/usr/bin/env node /* eslint-disable no-console */ -import { encode, decode, decodeAsync, decodeArrayStream } from "../src"; -import { writeFileSync, unlinkSync, readFileSync, createReadStream } from "fs"; -import { deepStrictEqual } from "assert"; +import { encode, decode, decodeAsync, decodeArrayStream } from "../src/index.ts"; +import { writeFileSync, unlinkSync, readFileSync, createReadStream } from "node:fs"; +import { deepStrictEqual } from "node:assert"; + +type Data = { id: number; score: number; title: string; content: string; createdAt: Date }; (async () => { - const data = []; + const data: Data[] = []; for (let i = 0; i < 1000; i++) { const id = i + 1; data.push({ diff --git a/benchmark/timestamp-ext.ts b/benchmark/timestamp-ext.ts index ba96a222..318b4451 100644 --- a/benchmark/timestamp-ext.ts +++ b/benchmark/timestamp-ext.ts @@ -1,4 +1,4 @@ -import { encode, decode } from "../src"; +import { encode, decode } from "../src/index.ts"; const data = new Array(100).fill(new Date()); diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000..0af8fa63 --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,151 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { fixupConfigRules, fixupPluginRules } from "@eslint/compat"; +import typescriptEslintEslintPlugin from "@typescript-eslint/eslint-plugin"; +import tsdoc from "eslint-plugin-tsdoc"; +import tsParser from "@typescript-eslint/parser"; +import js from "@eslint/js"; +import { FlatCompat } from "@eslint/eslintrc"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all, +}); + +export default [ + { + ignores: ["**/*.js", "test/deno*", "test/bun*"], + }, + ...fixupConfigRules( + compat.extends( + "eslint:recommended", + "plugin:@typescript-eslint/recommended", + "plugin:import/recommended", + "plugin:import/typescript", + "prettier", + ), + ), + { + plugins: { + "@typescript-eslint": fixupPluginRules(typescriptEslintEslintPlugin), + tsdoc, + }, + + languageOptions: { + parser: tsParser, + ecmaVersion: 5, + sourceType: "script", + + parserOptions: { + project: "./tsconfig.json", + }, + }, + + settings: {}, + + rules: { + "no-constant-condition": [ + "warn", + { + checkLoops: false, + }, + ], + + "no-useless-escape": "warn", + "no-console": "warn", + "no-var": "warn", + "no-return-await": "warn", + "prefer-const": "warn", + "guard-for-in": "warn", + curly: "warn", + "no-param-reassign": "warn", + "prefer-spread": "warn", + "import/no-unresolved": "off", + "import/no-cycle": "error", + "import/no-default-export": "warn", + "tsdoc/syntax": "warn", + "@typescript-eslint/await-thenable": "warn", + + "@typescript-eslint/array-type": [ + "warn", + { + default: "generic", + }, + ], + + "@typescript-eslint/naming-convention": [ + "warn", + { + selector: "default", + format: ["camelCase", "UPPER_CASE", "PascalCase"], + leadingUnderscore: "allow", + }, + { + selector: "typeLike", + format: ["PascalCase"], + leadingUnderscore: "allow", + }, + ], + + "@typescript-eslint/restrict-plus-operands": "warn", + //"@typescript-eslint/no-throw-literal": "warn", + "@typescript-eslint/unbound-method": "warn", + "@typescript-eslint/explicit-module-boundary-types": "warn", + //"@typescript-eslint/no-extra-semi": "warn", + "@typescript-eslint/no-extra-non-null-assertion": "warn", + + "@typescript-eslint/no-unused-vars": [ + "warn", + { + argsIgnorePattern: "^_", + }, + ], + + "@typescript-eslint/no-use-before-define": "warn", + "@typescript-eslint/no-for-in-array": "warn", + "@typescript-eslint/no-unsafe-argument": "warn", + "@typescript-eslint/no-unsafe-call": "warn", + + "@typescript-eslint/no-unnecessary-condition": [ + "warn", + { + allowConstantLoopConditions: true, + }, + ], + + "@typescript-eslint/no-unnecessary-type-constraint": "warn", + "@typescript-eslint/no-implied-eval": "warn", + "@typescript-eslint/no-non-null-asserted-optional-chain": "warn", + "@typescript-eslint/no-invalid-void-type": "warn", + "@typescript-eslint/no-loss-of-precision": "warn", + "@typescript-eslint/no-confusing-void-expression": "warn", + "@typescript-eslint/no-redundant-type-constituents": "warn", + "@typescript-eslint/prefer-for-of": "warn", + "@typescript-eslint/prefer-includes": "warn", + "@typescript-eslint/prefer-string-starts-ends-with": "warn", + "@typescript-eslint/prefer-readonly": "warn", + "@typescript-eslint/prefer-regexp-exec": "warn", + "@typescript-eslint/prefer-nullish-coalescing": "warn", + "@typescript-eslint/prefer-optional-chain": "warn", + "@typescript-eslint/prefer-ts-expect-error": "warn", + "@typescript-eslint/consistent-type-imports": [ + "error", + { + prefer: "type-imports", + disallowTypeAnnotations: false, + }, + ], + "@typescript-eslint/indent": "off", + "@typescript-eslint/no-explicit-any": "off", + "@typescript-eslint/no-empty-interface": "off", + "@typescript-eslint/no-empty-function": "off", + "@typescript-eslint/no-var-requires": "off", + "@typescript-eslint/no-non-null-assertion": "off", + "@typescript-eslint/ban-ts-comment": "off", + }, + }, +]; diff --git a/example/deno-with-esmsh.ts b/example/deno-with-esmsh.ts new file mode 100755 index 00000000..4911fc59 --- /dev/null +++ b/example/deno-with-esmsh.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "https://esm.sh/@msgpack/msgpack/mod.ts"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/deno-with-jsdeliver.ts b/example/deno-with-jsdeliver.ts new file mode 100755 index 00000000..af72b397 --- /dev/null +++ b/example/deno-with-jsdeliver.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "https://cdn.jsdelivr.net/npm/@msgpack/msgpack/mod.ts"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/deno-with-npm.ts b/example/deno-with-npm.ts new file mode 100755 index 00000000..98fdf92b --- /dev/null +++ b/example/deno-with-npm.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "npm:@msgpack/msgpack"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/deno-with-unpkg.ts b/example/deno-with-unpkg.ts new file mode 100755 index 00000000..84d9128b --- /dev/null +++ b/example/deno-with-unpkg.ts @@ -0,0 +1,5 @@ +#!/usr/bin/env deno run +/* eslint-disable no-console */ +import * as msgpack from "https://unpkg.com/@msgpack/msgpack/mod.ts"; + +console.log(msgpack.decode(msgpack.encode("Hello, world!"))); diff --git a/example/fetch-example-server.ts b/example/fetch-example-server.ts new file mode 100644 index 00000000..e4b5d10c --- /dev/null +++ b/example/fetch-example-server.ts @@ -0,0 +1,31 @@ +// ts-node example/fetch-example-server.ts +// open example/fetch-example.html + +import http from "http"; +import { encode } from "../src"; + +const hostname = "127.0.0.1"; +const port = 8080; + +function bufferView(b: Uint8Array) { + return Buffer.from(b.buffer, b.byteOffset, b.byteLength); +} + +const server = http.createServer((req, res) => { + console.log("accept:", req.method, req.url); + + res.statusCode = 200; + res.setHeader("content-type", "application/x-msgpack"); + res.setHeader("access-control-allow-origin", "*"); + res.end( + bufferView( + encode({ + message: "Hello, world!", + }), + ), + ); +}); + +server.listen(port, hostname, () => { + console.log(`Server running at http://${hostname}:${port}/`); +}); diff --git a/example/fetch-example.html b/example/fetch-example.html new file mode 100644 index 00000000..d59c2503 --- /dev/null +++ b/example/fetch-example.html @@ -0,0 +1,62 @@ + + + + + + + + + + +
+

Fetch API example

+

Open DevTool and see the console logs.

+ +
+ +
+ + + diff --git a/example/amd-example.html b/example/umd-example.html similarity index 67% rename from example/amd-example.html rename to example/umd-example.html index efd963e2..23ea96c3 100644 --- a/example/amd-example.html +++ b/example/umd-example.html @@ -1,7 +1,7 @@ - +