esm: support https remotely and http locally under flag

Co-authored-by: Jacob Smith
  <3012099+JakobJingleheimer@users.noreply.github.com>
Co-authored-by: James M Snell <jasnell@gmail.com>
Co-authored-by: Jordan Harband <ljharb@gmail.com>
Co-authored-by: James Sumners <james@sumners.email>
PR-URL: #36328
Backport-PR-URL: #42726
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Geoffrey Booth <webadmin@geoffreybooth.com>

Author: 5 people

doc/api/cli.md

@@ -291,6 +291,16 @@ added: v9.0.0
 Specify the `module` of a custom experimental [ECMAScript module loader][].
 `module` may be any string accepted as an [`import` specifier][].
 
+### `--experimental-network-imports`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Enable experimental support for the `https:` protocol in `import` specifiers.
+
 ### `--experimental-policy`
 
 <!-- YAML
@@ -1534,6 +1544,7 @@ Node.js options that are allowed are:
 * `--experimental-json-modules`
 * `--experimental-loader`
 * `--experimental-modules`
+* `--experimental-network-imports`
 * `--experimental-policy`
 * `--experimental-specifier-resolution`
 * `--experimental-top-level-await`

doc/api/errors.md

@@ -3105,6 +3105,23 @@ removed: v10.0.0
 
 Used by the `Node-API` when `Constructor.prototype` is not an object.
 
+<a id="ERR_NETWORK_IMPORT_BAD_RESPONSE"></a>
+
+### `ERR_NETWORK_IMPORT_BAD_RESPONSE`
+
+> Stability: 1 - Experimental
+
+Response was received but was invalid when importing a module over the network.
+
+<a id="ERR_NETWORK_IMPORT_DISALLOWED"></a>
+
+### `ERR_NETWORK_IMPORT_DISALLOWED`
+
+> Stability: 1 - Experimental
+
+A network module attempted to load another module that it is not allowed to
+load. Likely this restriction is for security reasons.
+
 <a id="ERR_NO_LONGER_SUPPORTED"></a>
 
 ### `ERR_NO_LONGER_SUPPORTED`

doc/api/esm.md

@@ -596,6 +596,71 @@ spawn(execPath, [
 });
 ```
 
+## HTTPS and HTTP imports
+
+> Stability: 1 - Experimental
+
+Importing network based modules using `https:` and `http:` is supported under
+the `--experimental-network-imports` flag. This allows web browser-like imports
+to work in Node.js with a few differences due to application stability and
+security concerns that are different when running in a privileged environment
+instead of a browser sandbox.
+
+### Imports are limited to HTTP/1
+
+Automatic protocol negotiation for HTTP/2 and HTTP/3 is not yet supported.
+
+### HTTP is limited to loopback addresses
+
+`http:` is vulnerable to man-in-the-middle attacks and is not allowed to be
+used for addresses outside of the IPv4 address `127.0.0.0/8` (`127.0.0.1` to
+`127.255.255.255`) and the IPv6 address `::1`. Support for `http:` is intended
+to be used for local development.
+
+### Authentication is never sent to the destination server.
+
+`Authorization`, `Cookie`, and `Proxy-Authorization` headers are not sent to the
+server. Avoid including user info in parts of imported URLs. A security model
+for safely using these on the server is being worked on.
+
+### CORS is never checked on the destination server
+
+CORS is designed to allow a server to limit the consumers of an API to a
+specific set of hosts. This is not supported as it does not make sense for a
+server-based implementation.
+
+### Cannot load non-network dependencies
+
+These modules cannot access other modules that are not over `http:` or `https:`.
+To still access local modules while avoiding the security concern, pass in
+references to the local dependencies:
+
+```mjs
+// file.mjs
+import worker_threads from 'worker_threads';
+import { configure, resize } from 'https://example.com/imagelib.mjs';
+configure({ worker_threads });
+```
+
+```mjs
+// https://example.com/imagelib.mjs
+let worker_threads;
+export function configure(opts) {
+  worker_threads = opts.worker_threads;
+}
+export function resize(img, size) {
+  // Perform resizing in worker_thread to avoid main thread blocking
+}
+```
+
+### Network-based loading is not enabled by default
+
+For now, the `--experimental-network-imports` flag is required to enable loading
+resources over `http:` or `https:`. In the future, a different mechanism will be
+used to enforce this. Opt-in is required to prevent transitive dependencies
+inadvertently using potentially mutable state that could affect reliability
+of Node.js applications.
+
 <i id="esm_experimental_loaders"></i>
 
 ## Loaders

doc/node.1

@@ -147,6 +147,9 @@ Specify the
 .Ar module
 to use as a custom module loader.
 .
+.It Fl -experimental-network-imports
+Enable experimental support for loading modules using `import` over `https:`.
+.
 .It Fl -experimental-policy
 Use the specified file as a security policy.
 .

lib/internal/errors.js

@@ -1414,6 +1414,10 @@ E('ERR_NAPI_INVALID_TYPEDARRAY_ALIGNMENT',
   'start offset of %s should be a multiple of %s', RangeError);
 E('ERR_NAPI_INVALID_TYPEDARRAY_LENGTH',
   'Invalid typed array length', RangeError);
+E('ERR_NETWORK_IMPORT_BAD_RESPONSE',
+  "import '%s' received a bad response: %s", Error);
+E('ERR_NETWORK_IMPORT_DISALLOWED',
+  "import of '%s' by %s is not supported: %s", Error);
 E('ERR_NO_CRYPTO',
   'Node.js is not compiled with OpenSSL crypto support', Error);
 E('ERR_NO_ICU',
@@ -1575,12 +1579,13 @@ E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s', TypeError);
 E('ERR_UNKNOWN_FILE_EXTENSION',
   'Unknown file extension "%s" for %s',
   TypeError);
-E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s', RangeError);
+E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s for URL %s',
+  RangeError);
 E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s', TypeError);
 E('ERR_UNSUPPORTED_DIR_IMPORT', "Directory import '%s' is not supported " +
 'resolving ES modules imported from %s', Error);
-E('ERR_UNSUPPORTED_ESM_URL_SCHEME', (url) => {
-  let msg = 'Only file and data URLs are supported by the default ESM loader';
+E('ERR_UNSUPPORTED_ESM_URL_SCHEME', (url, supported) => {
+  let msg = `Only URLs with a scheme in: ${ArrayPrototypeJoin(supported, ', ')} are supported by the default ESM loader`;
   if (isWindows && url.protocol.length === 2) {
     msg +=
       '. On Windows, absolute paths must be valid file:// URLs';

lib/internal/main/check_syntax.js

@@ -45,16 +45,16 @@ if (process.argv[1] && process.argv[1] !== '-') {
   });
 }
 
-function checkSyntax(source, filename) {
+async function checkSyntax(source, filename) {
   const { getOptionValue } = require('internal/options');
   let isModule = false;
   if (filename === '[stdin]' || filename === '[eval]') {
     isModule = getOptionValue('--input-type') === 'module';
   } else {
     const { defaultResolve } = require('internal/modules/esm/resolve');
     const { defaultGetFormat } = require('internal/modules/esm/get_format');
-    const { url } = defaultResolve(pathToFileURL(filename).toString());
-    const format = defaultGetFormat(url);
+    const { url } = await defaultResolve(pathToFileURL(filename).toString());
+    const format = await defaultGetFormat(url);
     isModule = format === 'module';
   }
   if (isModule) {