From 51ed7d7a85ea4b45eab8128aa66a95a96fc12576 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Mon, 28 Apr 2025 00:51:02 +0800 Subject: [PATCH 1/4] chore(ci): docs: allow workflow_dispatch --- .github/workflows/docs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 9122088e0..83d1146dd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -5,6 +5,8 @@ on: push: branches: - master + workflow_dispatch: + env: nim-version: 'stable' git-url-arg: --git.url:https://github.com/${{ github.repository }} --git.commit:master From c01b02025d5b5e9774e948aa57c982ad40f90dd5 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Apr 2025 10:58:30 +0800 Subject: [PATCH 2/4] break(EXT): unexport io.raiseOsOrFileNotFoundError --- src/pylib/Lib/io.nim | 8 +------- src/pylib/io.nim | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/pylib/Lib/io.nim b/src/pylib/Lib/io.nim index a1dbf4cd3..6ca56d659 100644 --- a/src/pylib/Lib/io.nim +++ b/src/pylib/Lib/io.nim @@ -636,13 +636,7 @@ proc openNoNonInhertFlag(f: var File, filehandle: FileHandle, f = c_fdopen(filehandle, fop) result = f != nil -proc raiseOsOrFileNotFoundError*(file: int) = - let err = osLastError() - let fn = "fd: " & $file - raiseOSError(err, - "[Errno " & $err & "] " & "can't open " & fn) - -proc raiseOsOrFileNotFoundError*[T](file: PathLike[T]) = +proc raiseOsOrFileNotFoundError[T](file: PathLike[T]) = file.raiseExcWithPath() proc initBufAsPy*(nfile: var File, buf: int) = diff --git a/src/pylib/io.nim b/src/pylib/io.nim index d8d6695be..5178ca75e 100644 --- a/src/pylib/io.nim +++ b/src/pylib/io.nim @@ -23,4 +23,4 @@ but Nim's `iterator lines` does not import ./Lib/io export io.open, io.close, io.seek, io.read, io.readline, io.write, io.truncate -export io.raiseOsOrFileNotFoundError, io.initBufAsPy +export io.initBufAsPy From 153b94ab909426a0d86dbe2749f09ae5177964ff Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Apr 2025 11:06:44 +0800 Subject: [PATCH 3/4] refact(nimpatch): nim-lang/Nim#23456 to nimpatch/ --- src/pylib/Lib/io.nim | 40 +++--------------------- src/pylib/nimpatch/winOpenFileHandle.nim | 38 ++++++++++++++++++++++ 2 files changed, 42 insertions(+), 36 deletions(-) create mode 100644 src/pylib/nimpatch/winOpenFileHandle.nim diff --git a/src/pylib/Lib/io.nim b/src/pylib/Lib/io.nim index 6ca56d659..a8745f768 100644 --- a/src/pylib/Lib/io.nim +++ b/src/pylib/Lib/io.nim @@ -46,6 +46,7 @@ import ../pystring/[strimpl, strbltins] import ../pybytes/[bytesimpl, bytesbltins] import ./warnings import ./sys_impl/auditImpl as sys +import ../nimpatch/winOpenFileHandle const SEEK_SET* = 0 @@ -603,39 +604,6 @@ let # NOTE: For Win32, the behavior is the same as _IOFBF - Full Buffering IONBF {.importc: "_IONBF", nodecl.}: cint -# patch for system/io.nim or std/syncio.nim, -# see https://github.com/nim-lang/Nim/pull/23456 -const - NoInheritFlag = - # Platform specific flag for creating a File without inheritance. - when not defined(nimInheritHandles): - when defined(windows): "" - elif defined(linux) or defined(bsd): "e" - else: "" - else: "" - FormatOpen: array[FileMode, cstring] = [ - cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, - "r+b" & NoInheritFlag, "ab" & NoInheritFlag - ] -when defined(windows): - proc getOsfhandle(fd: cint): int {. - importc: "_get_osfhandle", header: "".} - proc c_fdopen(filehandle: cint, mode: cstring): File {. - importc: "_fdopen", header: "".} -else: - proc c_fdopen(filehandle: cint, mode: cstring): File {. - importc: "fdopen", header: "".} -proc openNoNonInhertFlag(f: var File, filehandle: FileHandle, - mode: FileMode = fmRead): bool {.tags: [], raises: [].} = - when not defined(nimInheritHandles) and declared(setInheritable): - let oshandle = when defined(windows): FileHandle getOsfhandle(filehandle) - else: filehandle - if not setInheritable(oshandle, false): - return false - let fop = FormatOpen[mode] - f = c_fdopen(filehandle, fop) - result = f != nil - proc raiseOsOrFileNotFoundError[T](file: PathLike[T]) = file.raiseExcWithPath() @@ -660,8 +628,8 @@ template openImpl(result: untyped; ) = bind genOpenInfo, initTextIO, FileMode, FileHandle, - openNoNonInhertFlag, `file=` - + open, `file=` + var buf = buffering var nmode: FileMode @@ -672,7 +640,7 @@ template openImpl(result: untyped; var nfile: File sys.audit("open", file, smode) ## XXX: PY-DIFF: 3rd arg shall be flags when file is int: - let succ = openNoNonInhertFlag(nfile, FileHandle file, mode=nmode) + let succ = open(nfile, FileHandle file, mode=nmode) else: let succ = open(nfile, $file, mode=nmode) # Nim/Python: diff --git a/src/pylib/nimpatch/winOpenFileHandle.nim b/src/pylib/nimpatch/winOpenFileHandle.nim new file mode 100644 index 000000000..9c0173956 --- /dev/null +++ b/src/pylib/nimpatch/winOpenFileHandle.nim @@ -0,0 +1,38 @@ +{.used.} +import ./utils +addPatch((2,1,1), defined(windows)): + ## introduced in nim-lang/Nim#23456 + # patch for system/io.nim or std/syncio.nim, + # see https://github.com/nim-lang/Nim/pull/23456 + when defined(nimPreviewSlimSystem): + import std/syncio + const + NoInheritFlag = + # Platform specific flag for creating a File without inheritance. + when not defined(nimInheritHandles): + when defined(windows): "" + elif defined(linux) or defined(bsd): "e" + else: "" + else: "" + FormatOpen: array[FileMode, cstring] = [ + cstring("rb" & NoInheritFlag), "wb" & NoInheritFlag, "w+b" & NoInheritFlag, + "r+b" & NoInheritFlag, "ab" & NoInheritFlag + ] + when defined(windows): + proc getOsfhandle(fd: cint): int {. + importc: "_get_osfhandle", header: "".} + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "_fdopen", header: "".} + else: + proc c_fdopen(filehandle: cint, mode: cstring): File {. + importc: "fdopen", header: "".} + proc open*(f: var File, filehandle: FileHandle, + mode: FileMode = fmRead): bool {.tags: [], raises: [].} = + when not defined(nimInheritHandles) and declared(setInheritable): + let oshandle = when defined(windows): FileHandle getOsfhandle(filehandle) + else: filehandle + if not setInheritable(oshandle, false): + return false + let fop = FormatOpen[mode] + f = c_fdopen(filehandle, fop) + result = f != nil From 29dbd2baf46ab6c68feb1f3c02e83185541d3f23 Mon Sep 17 00:00:00 2001 From: litlighilit Date: Tue, 29 Apr 2025 12:42:16 +0800 Subject: [PATCH 4/4] feat(js): supports io (wip) [skip ci] --- src/pylib/Lib/io.nim | 40 ++-- src/pylib/Lib/io_impl/jsio.nim | 420 +++++++++++++++++++++++++++++++++ 2 files changed, 445 insertions(+), 15 deletions(-) create mode 100644 src/pylib/Lib/io_impl/jsio.nim diff --git a/src/pylib/Lib/io.nim b/src/pylib/Lib/io.nim index a8745f768..5cb714dc7 100644 --- a/src/pylib/Lib/io.nim +++ b/src/pylib/Lib/io.nim @@ -63,6 +63,10 @@ type nlCarriageReturn nlCarriage +const InJs = defined(js) +when InJs: + import ./io_impl/jsio + type IOBase* = ref object of RootObj closed*: bool @@ -342,12 +346,15 @@ proc read*(self: TextIOWrapper, size: int): PyStr = result = self.readImpl(size) Iencode -proc write(self: IOBase, s: string): int{.discardable.} = - self.file.write s - s.len +proc writeRetI(self: IOBase, s: string): int = + when InJs: + self.file.writeRetI(s) + else: + self.file.write s + s.len proc write*(self: RawIOBase, s: PyBytes): int{.discardable.} = - write(IOBase(self), $s) + writeRetI(IOBase(self), $s) proc writeImpl(self: NoEncTextIOWrapper, s: string, cvtRet: proc(s: string): int): int{.discardable.} = proc retSubs(toNewLine: string): int = cvtRet(s.replace("\n", toNewLine)) @@ -361,7 +368,7 @@ proc writeImpl(self: NoEncTextIOWrapper, s: string, cvtRet: proc(s: string): int proc write*(self: NoEncTextIOWrapper, s: PyStr): int{.discardable.} = proc cvtRet(oriStr: string): int = - discard write(IOBase(self), s) + discard writeRetI(IOBase(self), s) s.len writeImpl(self, s, cvtRet) @@ -391,11 +398,11 @@ proc write*(self: TextIOWrapper, s: PyStr): int{.discardable.} = proc cvtRet(oriStr: string): int = let t = self.codec.encode(oriStr) - discard write(IOBase(self), t.data) + discard writeRetI(IOBase(self), t.data) t.len writeImpl(self, s, cvtRet) -proc truncate*(self: IOBase): int{.discardable.} = +proc truncate*(self: IOBase, size: int64 = self.tell()): int64{.discardable.} = runnableExamples: const fn = "tempfiletest" var f = open(fn, "w+") @@ -404,11 +411,11 @@ proc truncate*(self: IOBase): int{.discardable.} = f.truncate() assert f.read() == "" f.close() - result = self.tell().int - truncate self.fileno, result - -proc truncate*(self: IOBase, size: int64): int64{.discardable.} = truncate self.fileno, size + when InJs: + truncate self.file, size + else: + truncate self.fileno, result size # workaround, @@ -461,10 +468,13 @@ proc isatty(p: CanIOOpenT): bool = when p is int: result = p.isatty() else: - var f: File - if f.open($p, fmRead): - result = f.isatty() - f.close() + when InJs: + result = p.isatty() + else: + var f: File + if f.open($p, fmRead): + result = f.isatty() + f.close() proc norm_buffering(file: CanIOOpenT, buffering: var int): bool = ## returns line_buffering diff --git a/src/pylib/Lib/io_impl/jsio.nim b/src/pylib/Lib/io_impl/jsio.nim new file mode 100644 index 000000000..647878ef9 --- /dev/null +++ b/src/pylib/Lib/io_impl/jsio.nim @@ -0,0 +1,420 @@ +# from $nim/lib/std/syncio +{.pragma: benign, gcsafe.} +proc raiseEIO(msg: string) {.noinline, noreturn.} = + raise newException(IOError, msg) + +proc raiseEOF() {.noinline, noreturn.} = + raise newException(EOFError, "EOF reached") + +# --- + +import ../../jsutils/denoAttrs + +import std/macros +import std/jsffi +when defined(nimPreviewSlimSystem): + import std/syncio except File +type + JsNumber = cdouble + JsIntNumber = distinct int +converter toInt(n: JsIntNumber): int = int(n) + +type File*{.pure.} = JsObject + ## unbuffered byte IO: + ## + ## - deno: Deno.FileInfo with additional attribute: name + ## - node: {fd: , position: } + +template wrapStdio(ioe) = + let ioe*{.importDenoOrProcess(ioe).}: File ## ReadStream | WriteStream + ios.pos = 0.toJs # unused, however + +wrapStdio stdin +wrapStdio stdout +wrapStdio stderr + +template denoOr(deno, node): untyped = + when defined(nodejs): node + else: + if inDeno: deno + else: node + +template nodeOr(node, deno): untyped = denoOr(deno, node) + +type + TextEncoder{.importjs.} = object of JsObject + encoding: cstring + Uint8Array{.importjs.} = object of JsObject + length: JsNumber + +using buffer: Uint8Array + +proc newUint8Array(len: int|Uint8Array = 0): Uint8Array{.importjs: "(new Uint8Array(@))".} +proc subarray(buffer; n: JsNumber): Uint8Array{.importcpp.} +proc len(buffer): int = int buffer.length + +proc `[]`(buffer; i: int): uint8 = buffer[i].to uint8 +iterator items(buffer): uint8 = + for i in 0.. `f: var File` + if not f.isNull: + f.close() + f = open(fp) + + +# REUSE ``open*(filename: string,`` + +# NIMDIFF: `var f: File = nil` -> `var f = default(File)` +# readFile, writeFile, readLines + +# REUSE ``lines*(filename: string): strin`` +# REUSE ``lines*(f: File): string`` + +proc truncate*(f; len=0){. + tags: [WriteIOEffect], benign.} = + denoOr(f.truncateSync(len), f.fd.ftruncateSync(len)) + +proc flushFile*(f){.tags: [WriteIOEffect].} = + discard # node/deno's writeSync is non-buffered +# catchJsErrAndRaise: + +proc readAll*(f): string{. + tags: [ReadIOEffect], benign.} = + denoOr(f.readAll_deno, f.readAll_node) + +proc getFileSize*(f): int64{.tags: [ReadIOEffect], benign.} = + denoOr(f.getFileSize_deno, f.getFileSize_node) + +proc setFilePos*(f; pos: int64, whence: FileSeekPos = fspSet){.benign, sideEffect.} = + denoOr f.seekSync(pos, SeekMode(whence)): + f.position = case whence + of fspSet: pos + of fspCur: pos + f.position + of fspEnd: f.getFileSize + pos + + +proc getFilePos*(f): int64{.benign.} = + nodeOr f.position.to(JsNumber).int64: + f.seekSync(0, SeekMode.Current) + +template readOrWriteCharImpl(readOrWrite): JsIntNumber = + var buffer = newUint8Array(1) + nodeOr readOrWrite(f.fd, buffer) do: + let obj = f.readOrWrite(buffer) + if obj.isNull: 0 else: obj.to JsIntNumber + +proc readChar*(f): char{. + tags: [ReadIOEffect], benign.} = + case readOrWriteCharImpl readSync + of 0: raiseEOF() + of 1: return buffer[0].char + else: raiseIO("readChar") ## FIXME: better msg + +proc readChars*(f; a: var openArray[char]): int{. + tags: [ReadIOEffect], benign.} = + var buffer = newUint8Array(a.len) + + let n = denoOr readSync(f, buffer) do: + f.position += n + readSync(f.fd, buffer, + position=f.position + ) + let nn = result + n + for i in n..