From 033ea2c58fe1bd7e50119e56ce00898641b5a348 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Fri, 23 Aug 2024 15:50:55 -0700 Subject: [PATCH] Remove LazyFile and make it a dep --- package.json | 5 +- pnpm-lock.yaml | 9 +++ src/file-storage.ts | 2 - src/lib/byte-range.spec.ts | 40 ---------- src/lib/byte-range.ts | 36 --------- src/lib/lazy-file.spec.ts | 141 ---------------------------------- src/lib/lazy-file.ts | 116 ---------------------------- src/lib/local-file-storage.ts | 19 ++--- 8 files changed, 23 insertions(+), 345 deletions(-) delete mode 100644 src/lib/byte-range.spec.ts delete mode 100644 src/lib/byte-range.ts delete mode 100644 src/lib/lazy-file.spec.ts delete mode 100644 src/lib/lazy-file.ts diff --git a/package.json b/package.json index 7079d52..cae2828 100644 --- a/package.json +++ b/package.json @@ -37,5 +37,8 @@ "storage", "stream", "fs" - ] + ], + "dependencies": { + "@mjackson/lazy-file": "^1.1.0" + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5cacec1..6eb3284 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,6 +7,10 @@ settings: importers: .: + dependencies: + '@mjackson/lazy-file': + specifier: ^1.1.0 + version: 1.1.0 devDependencies: '@types/node': specifier: ^20.14.10 @@ -33,6 +37,9 @@ packages: resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} + '@mjackson/lazy-file@1.1.0': + resolution: {integrity: sha512-QBbeLhBc14V+MIG6qNiX0iKNB6Cbfogmr3Kc4uhEkOnzxeyr5ImfzMulM4Od+IG7OYtHKGpoFJiaA59kmGErJQ==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -223,6 +230,8 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + '@mjackson/lazy-file@1.1.0': {} + '@pkgjs/parseargs@0.11.0': optional: true diff --git a/src/file-storage.ts b/src/file-storage.ts index 9194215..7cb0707 100644 --- a/src/file-storage.ts +++ b/src/file-storage.ts @@ -1,5 +1,3 @@ -export { type ByteRange, getByteLength, getIndexes } from "./lib/byte-range.js"; export { type FileStorage } from "./lib/file-storage.js"; -export { LazyFile } from "./lib/lazy-file.js"; export { LocalFileStorage } from "./lib/local-file-storage.js"; export { MemoryFileStorage } from "./lib/memory-file-storage.js"; diff --git a/src/lib/byte-range.spec.ts b/src/lib/byte-range.spec.ts deleted file mode 100644 index c313eb5..0000000 --- a/src/lib/byte-range.spec.ts +++ /dev/null @@ -1,40 +0,0 @@ -import * as assert from "node:assert/strict"; -import { describe, it } from "node:test"; - -import { ByteRange, getByteLength, getIndexes } from "./byte-range.js"; - -describe("getByteLength", () => { - it("returns the correct length", () => { - let size = 100; - - let range: ByteRange = { start: 10, end: 20 }; - assert.strictEqual(getByteLength(range, size), 10); - - range = { start: 10, end: -10 }; - assert.strictEqual(getByteLength(range, size), 80); - - range = { start: -10, end: -10 }; - assert.strictEqual(getByteLength(range, size), 0); - - range = { start: -10, end: 20 }; - assert.strictEqual(getByteLength(range, size), 0); - }); -}); - -describe("getIndexes", () => { - it("returns the correct indexes", () => { - let size = 100; - - let range: ByteRange = { start: 10, end: 20 }; - assert.deepStrictEqual(getIndexes(range, size), [10, 20]); - - range = { start: 10, end: -10 }; - assert.deepStrictEqual(getIndexes(range, size), [10, 90]); - - range = { start: -10, end: -10 }; - assert.deepStrictEqual(getIndexes(range, size), [90, 90]); - - range = { start: -10, end: 20 }; - assert.deepStrictEqual(getIndexes(range, size), [90, 90]); - }); -}); diff --git a/src/lib/byte-range.ts b/src/lib/byte-range.ts deleted file mode 100644 index af36fa0..0000000 --- a/src/lib/byte-range.ts +++ /dev/null @@ -1,36 +0,0 @@ -export interface ByteRange { - /** - * The start index of the range (inclusive). If this number is negative, it represents an offset - * from the end of the buffer. - */ - start: number; - /** - * The end index of the range (exclusive). If this number is negative, it represents an offset - * from the end of the buffer. `Infinity` represents the end of the buffer. - */ - end: number; -} - -/** - * Returns the length of the byte range in a buffer of the given `size`. - */ -export function getByteLength(range: ByteRange, size: number): number { - let [start, end] = getIndexes(range, size); - return end - start; -} - -/** - * Resolves a byte range to absolute indexes in a buffer of the given `size`. - */ -export function getIndexes(range: ByteRange, size: number): [number, number] { - let start = Math.min( - Math.max(0, range.start < 0 ? size + range.start : range.start), - size - ); - let end = Math.min( - Math.max(start, range.end < 0 ? size + range.end : range.end), - size - ); - - return [start, end]; -} diff --git a/src/lib/lazy-file.spec.ts b/src/lib/lazy-file.spec.ts deleted file mode 100644 index 4543af0..0000000 --- a/src/lib/lazy-file.spec.ts +++ /dev/null @@ -1,141 +0,0 @@ -import * as assert from "node:assert/strict"; -import { describe, it } from "node:test"; - -import { LazyFileContent, LazyFile } from "./lazy-file.js"; - -function createContent(value = ""): LazyFileContent { - let buffer = new TextEncoder().encode(value); - return { - byteLength: buffer.byteLength, - read() { - return new ReadableStream({ - start(controller) { - controller.enqueue(buffer); - controller.close(); - } - }); - } - }; -} - -describe("LazyFile", () => { - it("has the correct name, size, type, and lastModified timestamp", () => { - let now = Date.now(); - let file = new LazyFile(createContent("X".repeat(100)), "example.txt", { - type: "text/plain", - lastModified: now - }); - - assert.equal(file.name, "example.txt"); - assert.equal(file.size, 100); - assert.equal(file.type, "text/plain"); - assert.equal(file.lastModified, now); - }); - - it("returns the file's contents as a stream", async () => { - let content = createContent("hello world"); - let file = new LazyFile(content, "hello.txt", { - type: "text/plain" - }); - - let decoder = new TextDecoder(); - let result = ""; - for await (let chunk of file.stream()) { - result += decoder.decode(chunk, { stream: true }); - } - result += decoder.decode(); - - assert.equal(result, "hello world"); - }); - - it("returns the file's contents as a string", async () => { - let content = createContent("hello world"); - let file = new LazyFile(content, "hello.txt", { - type: "text/plain" - }); - - assert.equal(await file.text(), "hello world"); - }); - - describe("slice()", () => { - it("returns a file with the same name, type, and lastModified timestamp when slicing a file", () => { - let file = new LazyFile(createContent(), "hello.txt", { - type: "text/plain", - lastModified: Date.now() - }); - let slice = file.slice(0, 5, file.type); - assert.equal(slice.name, file.name); - assert.equal(slice.type, file.type); - assert.equal(slice.lastModified, file.lastModified); - }); - - it("returns a file with the same size as the original when slicing from 0 to the end", () => { - let file = new LazyFile(createContent("hello world"), "hello.txt", { - type: "text/plain" - }); - let slice = file.slice(0); - assert.equal(slice.size, file.size); - }); - - it('returns a file with size 0 when the "start" index is greater than the content length', () => { - let file = new LazyFile(createContent("hello world"), "hello.txt", { - type: "text/plain" - }); - let slice = file.slice(100); - assert.equal(slice.size, 0); - }); - - it('returns a file with size 0 when the "start" index is greater than the "end" index', () => { - let file = new LazyFile(createContent("hello world"), "hello.txt", { - type: "text/plain" - }); - let slice = file.slice(5, 0); - assert.equal(slice.size, 0); - }); - - it("calls content.read() with the correct range", t => { - let content = createContent("X".repeat(100)); - let read = t.mock.method(content, "read"); - let file = new LazyFile(content, "example.txt", { type: "text/plain" }); - file.slice(10, 20).stream(); - assert.equal(read.mock.calls.length, 1); - assert.deepEqual(read.mock.calls[0].arguments, [10, 20]); - }); - - it('calls content.read() with the correct range when slicing a file with a negative "start" index', t => { - let content = createContent("X".repeat(100)); - let read = t.mock.method(content, "read"); - let file = new LazyFile(content, "example.txt", { type: "text/plain" }); - file.slice(-10).stream(); - assert.equal(read.mock.calls.length, 1); - assert.deepEqual(read.mock.calls[0].arguments, [90, 100]); - }); - - it('calls content.read() with the correct range when slicing a file with a negative "end" index', t => { - let content = createContent("X".repeat(100)); - let read = t.mock.method(content, "read"); - let file = new LazyFile(content, "example.txt", { type: "text/plain" }); - file.slice(0, -10).stream(); - assert.equal(read.mock.calls.length, 1); - assert.deepEqual(read.mock.calls[0].arguments, [0, 90]); - }); - - it('calls content.read() with the correct range when slicing a file with negative "start" and "end" indexes', t => { - let content = createContent("X".repeat(100)); - let read = t.mock.method(content, "read"); - let file = new LazyFile(content, "example.txt", { type: "text/plain" }); - file.slice(-20, -10).stream(); - assert.equal(read.mock.calls.length, 1); - assert.deepEqual(read.mock.calls[0].arguments, [80, 90]); - }); - - it('calls content.read() with the correct range when slicing a file with a "start" index greater than the "end" index', t => { - let content = createContent("X".repeat(100)); - let read = t.mock.method(content, "read"); - let file = new LazyFile(content, "example.txt", { type: "text/plain" }); - file.slice(20, 10).stream(); - assert.equal(read.mock.calls.length, 1); - assert.deepEqual(read.mock.calls[0].arguments, [20, 20]); - }); - }); -}); diff --git a/src/lib/lazy-file.ts b/src/lib/lazy-file.ts deleted file mode 100644 index af6af9b..0000000 --- a/src/lib/lazy-file.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { ByteRange, getByteLength, getIndexes } from "./byte-range.js"; - -export interface LazyFileContent { - /** - * The total length of the content. - */ - byteLength: number; - /** - * Returns a stream that can be used to read the content, optionally within a given `start` - * (inclusive) and `end` (exclusive) index. - */ - read(start?: number, end?: number): ReadableStream; -} - -/** - * A `File` that is backed by a stream of data. This is useful for working with large files that - * would be impractical to load into memory all at once. - */ -export class LazyFile extends File { - #content: LazyFileContent; - #props?: FilePropertyBag; - #range?: ByteRange; - - constructor( - content: LazyFileContent, - name: string, - props?: FilePropertyBag, - range?: ByteRange - ) { - super([], name, props); - this.#content = content; - this.#props = props; - this.#range = range; - } - - /** - * The size of the file in bytes. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/size) - */ - get size(): number { - return this.#range != null - ? getByteLength(this.#range, this.#content.byteLength) - : this.#content.byteLength; - } - - /** - * Returns the file's contents as an array buffer. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/arrayBuffer) - */ - async arrayBuffer(): Promise { - return (await this.bytes()).buffer; - } - - /** - * Returns the file's contents as a byte array. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/bytes) - */ - async bytes(): Promise { - let result = new Uint8Array(this.size); - - let offset = 0; - for await (let chunk of this.stream()) { - result.set(chunk, offset); - offset += chunk.length; - } - - return result; - } - - /** - * Returns a new `File` object that contains the data in the specified range. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/slice) - */ - slice(start = 0, end = Infinity, contentType = ""): File { - let range = { start, end }; - - if (this.#range != null) { - // file.slice().slice() is additive - range = { - start: this.#range.start + start, - end: this.#range.end === Infinity ? end : this.#range.end + end - }; - } - - let props = { ...this.#props, type: contentType }; - - return new LazyFile(this.#content, this.name, props, range); - } - - /** - * Returns a stream that can be used to read the file's contents. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/stream) - */ - stream(): ReadableStream { - if (this.#range != null) { - let [start, end] = getIndexes(this.#range, this.#content.byteLength); - return this.#content.read(start, end); - } - - return this.#content.read(); - } - - /** - * Returns the file's contents as a string. - * - * [MDN Reference](https://developer.mozilla.org/en-US/docs/Web/API/Blob/text) - */ - async text(): Promise { - return new TextDecoder("utf-8").decode(await this.bytes()); - } -} diff --git a/src/lib/local-file-storage.ts b/src/lib/local-file-storage.ts index 2cff561..e924b3e 100644 --- a/src/lib/local-file-storage.ts +++ b/src/lib/local-file-storage.ts @@ -1,9 +1,9 @@ import * as fs from "node:fs"; import * as fsp from "node:fs/promises"; import * as path from "node:path"; +import { LazyFileContent, LazyFile } from "@mjackson/lazy-file"; import { FileStorage } from "./file-storage.js"; -import { LazyFileContent, LazyFile } from "./lazy-file.js"; type FileWithoutSize = Omit; @@ -16,28 +16,29 @@ type FileWithoutSize = Omit; * same storage object. */ export class LocalFileStorage implements FileStorage { - #directory: string; + #dirname: string; #metadata: FileMetadataIndex; /** * @param directory The directory where files are stored */ constructor(directory: string) { + this.#dirname = path.resolve(directory); + try { - let stat = fs.statSync(directory); + let stat = fs.statSync(this.#dirname); if (!stat.isDirectory()) { - throw new Error(`Path "${directory}" is not a directory`); + throw new Error(`Path "${this.#dirname}" is not a directory`); } } catch (error) { if (!isNoEntityError(error)) { throw error; } - fs.mkdirSync(directory, { recursive: true }); + fs.mkdirSync(this.#dirname, { recursive: true }); } - this.#directory = directory; this.#metadata = new FileMetadataIndex( path.join(directory, ".metadata.json") ); @@ -48,7 +49,7 @@ export class LocalFileStorage implements FileStorage { } async set(key: string, file: FileWithoutSize): Promise { - let { name, size } = await createFile(this.#directory, file.stream()); + let { name, size } = await createFile(this.#dirname, file.stream()); await this.#metadata.set(key, { file: name, @@ -65,7 +66,7 @@ export class LocalFileStorage implements FileStorage { return null; } - let file = path.join(this.#directory, metadata.file); + let file = path.join(this.#dirname, metadata.file); let content: LazyFileContent = { byteLength: metadata.size, read(start, end) { @@ -83,7 +84,7 @@ export class LocalFileStorage implements FileStorage { return; } - let file = path.join(this.#directory, metadata.file); + let file = path.join(this.#dirname, metadata.file); try { await fsp.unlink(file); } catch (error) {