diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 36f50eb..ea09a23 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,9 +12,8 @@ ------------------------------------------------------------------------------------------------- - + - [ ] I prefixed the PR-title with `docs: `, `fix(area): `, `feat(area): ` or `breaking(area): ` -- [ ] I updated ./CHANGELOG.md with a link to this PR or Issue - [ ] I updated the README.md - [ ] I Added unit test(s) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f9fad5..d352efc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,8 @@ jobs: test: strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-latest] - node: ["17.3"] + os: [ubuntu-latest] + node: ["16", "18", "20"] runs-on: ${{ matrix.os }} @@ -22,7 +22,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v2 with: - node-version: '17.3' + node-version: ${{ matrix.node }} - run: npm install - run: npm test - run: npm run report -- --colors diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 7f0fa59..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,98 +0,0 @@ -# Changelog - -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](http://keepachangelog.com/) and this -project adheres to [Semantic Versioning](http://semver.org/). - -## v3.1.3 -- Allow usage of iterable object in Blob constructor. [#108] -- Run test WPT test against our impl [#109] -- File name are now casted to string [#109] -- Slicing in the middle of multiple parts added more bytes than what what it should have [#109] -- Prefixed `stream/web` import with `node:` to allow easier static analysis detection of Node built-ins [#122] -- Added `node:` prefix in `from.js` as well [#114] -- Suppress warning when importing `stream/web` [#114] - -## v3.1.2 -- Improved typing -- Fixed a bug where position in iterator did not increase - -## v3.1.0 -- started to use real whatwg streams -- degraded fs/promise to fs.promise to support node v12 -- degraded optional changing to support node v12 - -## v3.0.0 -- Changed WeakMap for private field (require node 12) -- Switch to ESM -- blob.stream() return a subset of whatwg stream which is the async iterable part - (it no longer return a node stream) -- Reduced the dependency of Buffer by changing to global TextEncoder/Decoder (require node 11) -- Disabled xo since it could understand private fields (#) -- No longer transform the type to lowercase (https://github.com/w3c/FileAPI/issues/43) - This is more loose than strict, keys should be lowercased, but values should not. - It would require a more proper mime type parser - so we just made it loose. -- index.js and file.js can now be imported by browser & deno since it no longer depends on any - core node features (but why would you?) -- Implemented a File class - -## v2.1.2 -- Fixed a bug where `start` in BlobDataItem was undefined (#85) - -## v2.1.1 -- Add nullish values checking in Symbol.hasInstance (#82) -- Add generated typings for from.js file (#80) -- Updated dev dependencies - -## v2.1.0 -- Fix: .slice has an implementation bug (#54). -- Added blob backed up by filesystem (#55) - -## v2.0.1 - -- Fix: remove upper bound for node engine semver (#49). - -## v2.0.0 - -> Note: This release was previously published as `1.0.7`, but as it contains breaking changes, we renamed it to `2.0.0`. - -- **Breaking:** minimum supported Node.js version is now 10.17. -- **Breaking:** `buffer` option has been removed. -- Enhance: create TypeScript declarations from JSDoc (#45). -- Enhance: operate on blob parts (byte sequence) (#44). -- Enhance: use a `WeakMap` for private properties (#42) . -- Other: update formatting. - -## v1.0.6 - -- Enhance: use upstream Blob directly in typings (#38) -- Other: update dependencies - -## v1.0.5 - -- Other: no change to code, update dev dependency to address vulnerability reports - -## v1.0.4 - -- Other: general code rewrite to pass linting, prepare for `node-fetch` release v3 - -## v1.0.3 - -- Fix: package.json export `blob.js` properly now - -## v1.0.2 - -- Other: fix test integration - -## v1.0.1 - -- Other: readme update - -## v1.0.0 - -- Major: initial release - -[#108]: https://github.com/node-fetch/fetch-blob/pull/108 -[#109]: https://github.com/node-fetch/fetch-blob/pull/109 -[#114]: https://github.com/node-fetch/fetch-blob/pull/114 diff --git a/README.md b/README.md index fb3e198..7368a10 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ A Blob implementation in Node.js, originally from [node-fetch](https://github.com/node-fetch/node-fetch). +Use the built-in [`Blob`](https://nodejs.org/docs/latest-v18.x/api/buffer.html#class-blob) in Node.js 18 and later. + ## Installation ```sh @@ -24,7 +26,6 @@ npm install fetch-blob - CommonJS was replaced with ESM - The node stream returned by calling `blob.stream()` was replaced with whatwg streams - (Read "Differences from other blobs" for more info.) -
@@ -46,14 +47,10 @@ npm install fetch-blob ```js // Ways to import -// (PS it's dependency free ESM package so regular http-import from CDN works too) -import Blob from 'fetch-blob' -import File from 'fetch-blob/file.js' - -import {Blob} from 'fetch-blob' -import {File} from 'fetch-blob/file.js' +import { Blob } from 'fetch-blob' +import { File } from 'fetch-blob/file.js' -const {Blob} = await import('fetch-blob') +const { Blob } = await import('fetch-blob') // Ways to read the blob: @@ -73,7 +70,6 @@ It will not read the content into memory. It will only stat the file for last mo ```js // The default export is sync and use fs.stat to retrieve size & last modified as a blob -import blobFromSync from 'fetch-blob/from.js' import {File, Blob, blobFrom, blobFromSync, fileFrom, fileFromSync} from 'fetch-blob/from.js' const fsFile = fileFromSync('./2-GiB-file.bin', 'application/octet-stream') @@ -87,9 +83,38 @@ console.log(blob.size) // ~4 GiB `blobFrom|blobFromSync|fileFrom|fileFromSync(path, [mimetype])` +### Creating a temporary file on the disk +(requires [FinalizationRegistry] - node v14.6) + +When using both `createTemporaryBlob` and `createTemporaryFile` +then you will write data to the temporary folder in their respective OS. +The arguments can be anything that [fsPromises.writeFile] supports. NodeJS +v14.17.0+ also supports writing (async)Iterable streams and passing in a +AbortSignal, so both NodeJS stream and whatwg streams are supported. When the +file have been written it will return a Blob/File handle with a references to +this temporary location on the disk. When you no longer have a references to +this Blob/File anymore and it have been GC then it will automatically be deleted. + +This files are also unlinked upon exiting the process. +```js +import { createTemporaryBlob, createTemporaryFile } from 'fetch-blob/from.js' + +const req = new Request('https://httpbin.org/image/png') +const res = await fetch(req) +const type = res.headers.get('content-type') +const signal = req.signal +let blob = await createTemporaryBlob(res.body, { type, signal }) +// const file = createTemporaryBlob(res.body, 'img.png', { type, signal }) +blob = undefined // loosing references will delete the file from disk +``` + +- `createTemporaryBlob(data, { type, signal })` +- `createTemporaryFile(data, FileName, { type, signal, lastModified })` + ### Creating Blobs backed up by other async sources Our Blob & File class are more generic then any other polyfills in the way that it can accept any blob look-a-like item -An example of this is that our blob implementation can be constructed with parts coming from [BlobDataItem](https://github.com/node-fetch/fetch-blob/blob/8ef89adad40d255a3bbd55cf38b88597c1cd5480/from.js#L32) (aka a filepath) or from [buffer.Blob](https://nodejs.org/api/buffer.html#buffer_new_buffer_blob_sources_options), It dose not have to implement all the methods - just enough that it can be read/understood by our Blob implementation. The minium requirements is that it has `Symbol.toStringTag`, `size`, `slice()` and either a `stream()` or a `arrayBuffer()` method. If you then wrap it in our Blob or File `new Blob([blobDataItem])` then you get all of the other methods that should be implemented in a blob or file +An example of this is that our blob implementation can be constructed with parts coming from [BlobDataItem](https://github.com/node-fetch/fetch-blob/blob/8ef89adad40d255a3bbd55cf38b88597c1cd5480/from.js#L32) (aka a filepath) or from [buffer.Blob](https://nodejs.org/api/buffer.html#buffer_new_buffer_blob_sources_options), It dose not have to implement all the methods - just enough that it can be read/understood by our Blob implementation. The minium requirements is that it has `Symbol.toStringTag`, `size`, `slice()`, `stream()` methods (the stream method +can be as simple as being a sync or async iterator that yields Uint8Arrays. If you then wrap it in our Blob or File `new Blob([blobDataItem])` then you get all of the other methods that should be implemented in a blob or file (aka: text(), arrayBuffer() and type and a ReadableStream) An example of this could be to create a file or blob like item coming from a remote HTTP request. Or from a DataBase @@ -104,3 +129,5 @@ See the [MDN documentation](https://developer.mozilla.org/en-US/docs/Web/API/Blo [install-size-image]: https://flat.badgen.net/packagephobia/install/fetch-blob [install-size-url]: https://packagephobia.now.sh/result?p=fetch-blob [fs-blobs]: https://github.com/nodejs/node/issues/37340 +[fsPromises.writeFile]: https://nodejs.org/dist/latest-v18.x/docs/api/fs.html#fspromiseswritefilefile-data-options +[FinalizationRegistry]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/FinalizationRegistry diff --git a/file.js b/file.js index 7b26538..b2739c9 100644 --- a/file.js +++ b/file.js @@ -1,4 +1,4 @@ -import Blob from './index.js' +import { Blob } from './index.js' const _File = class File extends Blob { #lastModified = 0 @@ -46,4 +46,3 @@ const _File = class File extends Blob { /** @type {typeof globalThis.File} */// @ts-ignore export const File = _File -export default File diff --git a/from.js b/from.js index 33c4e7b..d54d129 100644 --- a/from.js +++ b/from.js @@ -1,11 +1,20 @@ -import { statSync, createReadStream, promises as fs } from 'node:fs' -import { basename } from 'node:path' +import { + realpathSync, + statSync, + rmdirSync, + createReadStream, + promises as fs +} from 'node:fs' +import { basename, sep, join } from 'node:path' +import { tmpdir } from 'node:os' +import process from 'node:process' import DOMException from 'node-domexception' -import File from './file.js' -import Blob from './index.js' +import { File } from './file.js' +import { Blob } from './index.js' -const { stat } = fs +const { stat, mkdtemp } = fs +let i = 0, tempDir, registry /** * @param {string} path filepath on the disk @@ -49,6 +58,42 @@ const fromFile = (stat, path, type = '') => new File([new BlobDataItem({ start: 0 })], basename(path), { type, lastModified: stat.mtimeMs }) +/** + * Creates a temporary blob backed by the filesystem. + * NOTE: requires node.js v14 or higher to use FinalizationRegistry + * + * @param {*} data Same as fs.writeFile data + * @param {BlobPropertyBag & {signal?: AbortSignal}} options + * @param {AbortSignal} [signal] in case you wish to cancel the write operation + * @returns {Promise} + */ +const createTemporaryBlob = async (data, {signal, type} = {}) => { + registry = registry || new FinalizationRegistry(fs.unlink) + tempDir = tempDir || await mkdtemp(realpathSync(tmpdir()) + sep) + const id = `${i++}` + const destination = join(tempDir, id) + if (data instanceof ArrayBuffer) data = new Uint8Array(data) + await fs.writeFile(destination, data, { signal }) + const blob = await blobFrom(destination, type) + registry.register(blob, destination) + return blob +} + +/** + * Creates a temporary File backed by the filesystem. + * Pretty much the same as constructing a new File(data, name, options) + * + * NOTE: requires node.js v14 or higher to use FinalizationRegistry + * @param {*} data + * @param {string} name + * @param {FilePropertyBag & {signal?: AbortSignal}} opts + * @returns {Promise} + */ +const createTemporaryFile = async (data, name, opts) => { + const blob = await createTemporaryBlob(data) + return new File([blob], name, opts) +} + /** * This is a blob backed up by a file on the disk * with minium requirement. Its wrapped around a Blob as a blobPart @@ -102,5 +147,18 @@ class BlobDataItem { } } +process.once('exit', () => { + tempDir && rmdirSync(tempDir, { recursive: true }) +}) + export default blobFromSync -export { File, Blob, blobFrom, blobFromSync, fileFrom, fileFromSync } +export { + Blob, + blobFrom, + blobFromSync, + createTemporaryBlob, + File, + fileFrom, + fileFromSync, + createTemporaryFile +} \ No newline at end of file diff --git a/index.js b/index.js index 8a3809c..0683fa5 100644 --- a/index.js +++ b/index.js @@ -1,19 +1,32 @@ /*! fetch-blob. MIT License. Jimmy Wärting */ -// TODO (jimmywarting): in the feature use conditional loading with top level await (requires 14.x) -// Node has recently added whatwg stream into core - -import './streams.cjs' +if (!globalThis.ReadableStream) { + try { + const process = await import('node:process').then(m => m.default) + const { emitWarning } = process + try { + process.emitWarning = () => {} + const streams = await import('node:stream/web').then(m => m.default) + Object.assign(globalThis, streams) + process.emitWarning = emitWarning + } catch (error) { + process.emitWarning = emitWarning + throw error + } + } catch (error) {} +} // 64 KiB (same size chrome slice theirs blob into Uint8array's) const POOL_SIZE = 65536 -/** @param {(Blob | Uint8Array)[]} parts */ -async function * toIterator (parts, clone = true) { +/** + * @param {(Blob | Uint8Array)[]} parts + * @param {boolean} clone + * @returns {AsyncIterableIterator} + */ +async function * toIterator (parts, clone) { for (const part of parts) { - if ('stream' in part) { - yield * (/** @type {AsyncIterableIterator} */ (part.stream())) - } else if (ArrayBuffer.isView(part)) { + if (ArrayBuffer.isView(part)) { if (clone) { let position = part.byteOffset const end = part.byteOffset + part.byteLength @@ -26,16 +39,9 @@ async function * toIterator (parts, clone = true) { } else { yield part } - /* c8 ignore next 10 */ } else { - // For blobs that have arrayBuffer but no stream method (nodes buffer.Blob) - let position = 0, b = (/** @type {Blob} */ (part)) - while (position !== b.size) { - const chunk = b.slice(position, Math.min(b.size, position + POOL_SIZE)) - const buffer = await chunk.arrayBuffer() - position += buffer.byteLength - yield new Uint8Array(buffer) - } + // @ts-ignore TS Think blob.stream() returns a node:stream + yield * part.stream() } } } @@ -139,11 +145,6 @@ const _Blob = class Blob { * @return {Promise} */ async arrayBuffer () { - // Easier way... Just a unnecessary overhead - // const view = new Uint8Array(this.size); - // await this.stream().getReader({mode: 'byob'}).read(view); - // return view.buffer; - const data = new Uint8Array(this.size) let offset = 0 for await (const chunk of toIterator(this.#parts, false)) { @@ -218,7 +219,7 @@ const _Blob = class Blob { } } - const blob = new Blob([], { type: String(type).toLowerCase() }) + const blob = new Blob([], { type: `${type}` }) blob.#size = span blob.#parts = blobParts @@ -251,4 +252,3 @@ Object.defineProperties(_Blob.prototype, { /** @type {typeof globalThis.Blob} */ export const Blob = _Blob -export default Blob diff --git a/package.json b/package.json index cc91aa3..2a8fbd5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "fetch-blob", - "version": "3.1.5", + "version": "4.0.0", "description": "Blob & File implementation in Node.js, originally from node-fetch.", "main": "index.js", "type": "module", @@ -10,8 +10,7 @@ "file.d.ts", "index.js", "index.d.ts", - "from.d.ts", - "streams.cjs" + "from.d.ts" ], "scripts": { "test": "node --experimental-loader ./test/http-loader.js ./test/test-wpt-in-node.js", @@ -26,7 +25,7 @@ "node-fetch" ], "engines": { - "node": "^12.20 || >= 14.13" + "node": ">=16.7" }, "author": "Jimmy Wärting (https://jimmy.warting.se)", "license": "MIT", @@ -35,9 +34,9 @@ }, "homepage": "https://github.com/node-fetch/fetch-blob#readme", "devDependencies": { - "@types/node": "^17.0.9", - "c8": "^7.11.0", - "typescript": "^4.5.4" + "@types/node": "^16.5.0", + "c8": "^7.13.0", + "typescript": "^5.0.4" }, "funding": [ { @@ -50,7 +49,6 @@ } ], "dependencies": { - "node-domexception": "^1.0.0", - "web-streams-polyfill": "^3.0.3" + "node-domexception": "^1.0.0" } } diff --git a/streams.cjs b/streams.cjs deleted file mode 100644 index f760959..0000000 --- a/streams.cjs +++ /dev/null @@ -1,51 +0,0 @@ -/* c8 ignore start */ -// 64 KiB (same size chrome slice theirs blob into Uint8array's) -const POOL_SIZE = 65536 - -if (!globalThis.ReadableStream) { - // `node:stream/web` got introduced in v16.5.0 as experimental - // and it's preferred over the polyfilled version. So we also - // suppress the warning that gets emitted by NodeJS for using it. - try { - const process = require('node:process') - const { emitWarning } = process - try { - process.emitWarning = () => {} - Object.assign(globalThis, require('node:stream/web')) - process.emitWarning = emitWarning - } catch (error) { - process.emitWarning = emitWarning - throw error - } - } catch (error) { - // fallback to polyfill implementation - Object.assign(globalThis, require('web-streams-polyfill/dist/ponyfill.es2018.js')) - } -} - -try { - // Don't use node: prefix for this, require+node: is not supported until node v14.14 - // Only `import()` can use prefix in 12.20 and later - const { Blob } = require('buffer') - if (Blob && !Blob.prototype.stream) { - Blob.prototype.stream = function name (params) { - let position = 0 - const blob = this - - return new ReadableStream({ - type: 'bytes', - async pull (ctrl) { - const chunk = blob.slice(position, Math.min(blob.size, position + POOL_SIZE)) - const buffer = await chunk.arrayBuffer() - position += buffer.byteLength - ctrl.enqueue(new Uint8Array(buffer)) - - if (position === blob.size) { - ctrl.close() - } - } - }) - } - } -} catch (error) {} -/* c8 ignore end */ diff --git a/test/http-loader.js b/test/http-loader.js index cbbdbf0..5910a7a 100644 --- a/test/http-loader.js +++ b/test/http-loader.js @@ -5,36 +5,29 @@ import { get } from 'node:https' const fetch = url => new Promise(rs => get(url, rs)) const cache = new URL('./.cache/', import.meta.url) -/** - * @param {string} specifier - * @param {{ - * conditions: !Array, - * parentURL: !(string | undefined), - * }} context - * @param {Function} defaultResolve - * @returns {Promise<{ url: string }>} - */ -export async function resolve (specifier, context, defaultResolve) { - const { parentURL = null } = context +export function resolve(specifier, context, nextResolve) { + const { parentURL = null } = context; // Normally Node.js would error on specifiers starting with 'https://', so // this hook intercepts them and converts them into absolute URLs to be // passed along to the later hooks below. if (specifier.startsWith('https://')) { return { - url: specifier - } + shortCircuit: true, + url: specifier, + }; } else if (parentURL && parentURL.startsWith('https://')) { return { - url: new URL(specifier, parentURL).href - } + shortCircuit: true, + url: new URL(specifier, parentURL).href, + }; } // Let Node.js handle all other specifiers. - return defaultResolve(specifier, context, defaultResolve) + return nextResolve(specifier); } -export async function load (url, context, defaultLoad) { +export async function load(url, context, nextLoad) { // For JavaScript to be loaded over the network, we need to fetch and // return it. if (url.startsWith('https://')) { @@ -52,14 +45,15 @@ export async function load (url, context, defaultLoad) { fs.writeFileSync(cachedFile, data) } + // This example assumes all network-provided JavaScript is ES module + // code. return { - // This example assumes all network-provided JavaScript is ES module - // code. format: 'module', - source: data + shortCircuit: true, + source: data, } } // Let Node.js handle all other URLs. - return defaultLoad(url, context, defaultLoad) -} + return nextLoad(url); +} \ No newline at end of file diff --git a/test/own-misc-test.js b/test/own-misc-test.js index abb6fe2..b1336c9 100644 --- a/test/own-misc-test.js +++ b/test/own-misc-test.js @@ -3,7 +3,14 @@ import fs from 'node:fs' import buffer from 'node:buffer' -import syncBlob, { blobFromSync, blobFrom, fileFromSync, fileFrom } from '../from.js' +import syncBlob, { + blobFromSync, + blobFrom, + fileFromSync, + fileFrom, + createTemporaryBlob, + createTemporaryFile +} from '../from.js' const license = fs.readFileSync('./LICENSE') @@ -56,11 +63,6 @@ test(() => { assert_equals(blobFromSync, syncBlob) }, 'default export is named exported blobFromSync') -promise_test(async () => { - const { Blob, default: def } = await import('../index.js') - assert_equals(Blob, def) -}, 'Can use named import - as well as default') - // This was necessary to avoid large ArrayBuffer clones (slice) promise_test(async t => { const buf = new Uint8Array(65590) @@ -189,6 +191,38 @@ promise_test(async () => { assert_equals(await (await fileFrom('./LICENSE')).text(), license.toString()) }, 'blob part backed up by filesystem slice correctly') +promise_test(async () => { + let blob + // Can construct a temporary blob from a string + blob = await createTemporaryBlob(license.toString()) + assert_equals(await blob.text(), license.toString()) + + // Can construct a temporary blob from a async iterator + blob = await createTemporaryBlob(blob.stream()) + assert_equals(await blob.text(), license.toString()) + + // Can construct a temporary file from a arrayBuffer + blob = await createTemporaryBlob(await blob.arrayBuffer()) + assert_equals(await blob.text(), license.toString()) + + // Can construct a temporary file from a arrayBufferView + blob = await createTemporaryBlob(await blob.arrayBuffer().then(ab => new Uint8Array(ab))) + assert_equals(await blob.text(), license.toString()) + + // Can specify a mime type + blob = await createTemporaryBlob('abc', { type: 'text/plain' }) + assert_equals(blob.type, 'text/plain') + + // Can create files too + let file = await createTemporaryFile('abc', 'abc.txt', { + type: 'text/plain', + lastModified: 123 + }) + assert_equals(file.name, 'abc.txt') + assert_equals(file.size, 3) + assert_equals(file.lastModified, 123) +}, 'creating temporary blob/file backed up by filesystem') + promise_test(async () => { fs.writeFileSync('temp', '') await blobFromSync('./temp').text() diff --git a/test/test-wpt-in-node.js b/test/test-wpt-in-node.js index e414298..96f06f5 100644 --- a/test/test-wpt-in-node.js +++ b/test/test-wpt-in-node.js @@ -22,7 +22,7 @@ function test_blob (fn, expectations) { const blob = fn() assert_true(blob instanceof Blob) assert_false(blob instanceof File) - assert_equals(blob.type.toLowerCase(), type) + assert_equals(blob.type.toLowerCase(), type.toLowerCase()) assert_equals(await blob.text(), expected) t.done() }) @@ -140,4 +140,7 @@ import('https://wpt.live/FileAPI/blob/Blob-stream.any.js') import('https://wpt.live/FileAPI/blob/Blob-text.any.js') import('./own-misc-test.js') -hasFailed && process.exit(1) \ No newline at end of file +if (hasFailed) { + console.log('Tests failed') + process.exit(1) +} \ No newline at end of file