How to fix "Dynamic require of "os" is not supported"

[enricoschaaf]

How can we use dependencies that still use require while shipping esm? Is that possible?

Minimal reproduction:
https://github.com/enricoschaaf/esbuild-issue
Just run node index.mjs. This was generated from running build which executes esbuild to bundle this file.

Problem:
The problem is that node internal modules can of course not bet bundled so I expected the require syntax would get written to import syntax if the target is esm but they are still require.

Error message:
Error: Dynamic require of "os" is not supported
at file:///home/enrico/work/esbuild-issue/index.mjs:1:450
at file:///home/enrico/work/esbuild-issue/index.mjs:1:906
at file:///home/enrico/work/esbuild-issue/index.mjs:1:530
at file:///home/enrico/work/esbuild-issue/index.mjs:1:950
at ModuleJob.run (node:internal/modules/esm/module_job:195:25)
at async Promise.all (index 0)
at async ESMLoader.import (node:internal/modules/esm/loader:337:24)
at async loadESM (node:internal/process/esm_loader:88:5)
at async handleMainPromise (node:internal/modules/run_main:65:12)

[kzc]

As far as I know, esbuild cannot convert external "static" commonjs require statements to static esm imports - not even for the side effect free node platform built in modules. This would be a useful feature for esbuild to have, as it's a fairly common use case for NodeJS applications and libraries. One could write an esbuild plugin for that, but a comprehensive solution would require reparsing each JS source file and the build would be considerably slower.

[enricoschaaf]

Ok, I thought about writing one but I had the same reservations you had. But indeed that would be a super nice feature for node apps.

[hronro]

I create a new issue #1927 and I think it may be related.

[kzc]

Although this functionality would be better in esbuild itself, here's a super hacky script to transform node built-in __require()s in esbuild esm output to import statements. It only works with non-minified esbuild output.

$ cat importify-esbuild-output.js 
var fs = require("fs");
var arg = process.argv[2];
var data = !arg || arg == "-" ? fs.readFileSync(0, "utf-8") : fs.readFileSync(arg, "utf-8");;
var rx = /\b__require\("(_http_agent|_http_client|_http_common|_http_incoming|_http_outgoing|_http_server|_stream_duplex|_stream_passthrough|_stream_readable|_stream_transform|_stream_wrap|_stream_writable|_tls_common|_tls_wrap|assert|async_hooks|buffer|child_process|cluster|console|constants|crypto|dgram|diagnostics_channel|dns|domain|events|fs|http|http2|https|inspector|module|net|os|path|perf_hooks|process|punycode|querystring|readline|repl|stream|string_decoder|sys|timers|tls|trace_events|tty|url|util|v8|vm|wasi|worker_threads|zlib)"\)/gm;
var modules = new Map;
var out = data.replace(rx, function(req, mod) {
    var id = "__import_" + mod.toUpperCase();
    modules.set(mod, id);
    return id;
});
modules.forEach(function(val, key) {
    console.log("import %s from %s;", val, JSON.stringify(key));
});
console.log("\n%s", out);

Example input:

$ cat 0.js
console.log(require("path").extname("foo/bar.hello"));
var os = require("os");
console.log(require("path").extname("ok.world"));

Expected output:

$ cat 0.js | node
.hello
.world

Unmodified esbuild esm bundle output:

$ cat 0.js | esbuild --bundle --platform=node --format=esm
var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
}) : x)(function(x) {
  if (typeof require !== "undefined")
    return require.apply(this, arguments);
  throw new Error('Dynamic require of "' + x + '" is not supported');
});

// <stdin>
console.log(__require("path").extname("foo/bar.hello"));
var os = __require("os");
console.log(__require("path").extname("ok.world"));

...piped through importify-esbuild-output.js:

$ cat 0.js | esbuild --bundle --platform=node --format=esm | node importify-esbuild-output.js 
import __import_PATH from "path";
import __import_OS from "os";

var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
}) : x)(function(x) {
  if (typeof require !== "undefined")
    return require.apply(this, arguments);
  throw new Error('Dynamic require of "' + x + '" is not supported');
});

// <stdin>
console.log(__import_PATH.extname("foo/bar.hello"));
var os = __import_OS;
console.log(__import_PATH.extname("ok.world"));

... and then piped through esbuild again to perform some DCE:

$ cat 0.js | esbuild --bundle --platform=node --format=esm | node importify-esbuild-output.js | esbuild --bundle --platform=node --format=esm
// <stdin>
import __import_PATH from "path";
console.log(__import_PATH.extname("foo/bar.hello"));
console.log(__import_PATH.extname("ok.world"));

to produce:

$ cat 0.js | esbuild --bundle --platform=node --format=esm | node importify-esbuild-output.js | esbuild --bundle --platform=node --format=esm | node --input-type=module
.hello
.world

[hyrious]

In addition to the workaround above:

esbuild actually will print a warning when converting require to esm format (see it live) like:

▲ [WARNING] Converting "require" to "esm" is currently not supported

    <stdin>:1:0:
      1 │ require('a')
        ╵ ~~~~~~~

Maybe we can use that info to write a plugin: https://gist.github.com/hyrious/7120a56c593937457c0811443563e017

One note: the modification actually turns a commonjs module into mixed module, however it seems esbuild can handle this case correctly:

import xxx from 'xxx'
module.exports = xxx

[jjenzz]

I'm getting this with require("buffer/") and from what I can gather, libs use the trailing slash to bypass core module and lookup in node_modules folder. I believe this is because they provide the polyfill for the browser as part of the package.

For now, I've added @esbuild-plugins/node-globals-polyfill and @esbuild-plugins/node-modules-polyfill, and then patched the packages to use require("buffer") so that they reference the esbuild polyfill instead but wondering if esbuild should be able to handle trailing slashes here?

This is all a bit over my head tbh, so apologies in advance if I'm not being clear 😅

[kzc]

require("buffer/") is a different issue and appears to be working as intended - see b2d7329.

[jjenzz]

@kzc in my case, it seems the trailing slash is being used to lookup a buffer polyfill for the browser so i am getting this error client-side when --platform=browser.

[kzc]

@jjenzz If you're seeing a bug you should probably open a separate issue for that with a reproducible test case, as it doesn't appear to be related to this issue. This issue is concerned with --format=esm --platform=node bundles producing require() instead of import for NodeJS built-in modules. Your issue appears to be related to a NodeJS module polyfill not being inlined into your bundle for --format=esm --platform=browser.

$ rm -rf node_modules/buffer
$ echo 'console.log(require("buffer/"));' | esbuild --bundle --format=esm | node --input-type=module
✘ [ERROR] Could not resolve "buffer/"

    <stdin>:1:20:
      1 │ console.log(require("buffer/"));
        ╵                     ~~~~~~~~~

  You can mark the path "buffer/" as external to exclude it from the bundle, which will remove this
  error. You can also surround this "require" call with a try/catch block to handle this failure at
  run-time instead of bundle-time.

1 error
$ mkdir -p node_modules/buffer
$ echo "module.exports = {abc: 42};" > node_modules/buffer/index.js
$ echo 'console.log(require("buffer/"));' | esbuild --bundle --format=esm | node --input-type=module
{ abc: 42 }
$ echo 'console.log(require("buffer/"));' | esbuild --bundle --format=esm
var __getOwnPropNames = Object.getOwnPropertyNames;
var __commonJS = (cb, mod) => function __require() {
  return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};

// node_modules/buffer/index.js
var require_buffer = __commonJS({
  "node_modules/buffer/index.js"(exports, module) {
    module.exports = { abc: 42 };
  }
});

// <stdin>
console.log(require_buffer());

Notice that require("buffer/") was inlined as expected once node_modules/buffer/ was in place. node --input-type=module would have errored out had it encountered a require().

The trailing backslash did not appear to matter for the browser platform if node_modules/buffer existed - identical esbuild output was produced:

$ cat node_modules/buffer/index.js 
module.exports = {abc: 42};
$ echo 'console.log(require("buffer/"));' | esbuild --bundle --format=esm --platform=browser | shasum
a4d2d5027a1ab792dcf6bfb6f0ddc63e273a2b11  -
$ echo 'console.log(require("buffer"));' | esbuild --bundle --format=esm --platform=browser | shasum
a4d2d5027a1ab792dcf6bfb6f0ddc63e273a2b11  -
$ echo 'console.log(require("buffer"));' | esbuild --bundle --format=esm --platform=browser | node --input-type=module
{ abc: 42 }
$ echo 'console.log(require("buffer/"));' | esbuild --bundle --format=esm --platform=browser | node --input-type=module
{ abc: 42 }

[jjenzz]

@kzc thanks for all of that. I'll see if I can create a minimal repro of the issue I am experiencing and create a separate issue if so.

[eric-burel]

Related: https://stackoverflow.com/questions/68423950/when-using-esbuild-with-external-react-i-get-dynamic-require-of-react-is-not-s?rq=1 It has no solution either