diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6b62d0c..adc9502 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -11,12 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup latest deno version - uses: denoland/setup-deno@v1 - with: - deno-version: v1.x + uses: denoland/setup-deno@v2 - name: Run deno fmt run: deno fmt --check @@ -28,31 +26,26 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup latest deno version - uses: denoland/setup-deno@v1 - with: - deno-version: v1.x + uses: denoland/setup-deno@v2 - name: Run deno task check run: deno task check - test: name: test ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, ubuntu-latest, macos-latest,] + os: [windows-latest, ubuntu-latest, macos-latest] steps: - name: Checkout sources uses: actions/checkout@v2 - name: Setup latest deno version - uses: denoland/setup-deno@v1 - with: - deno-version: v1.x + uses: denoland/setup-deno@v2 - name: Setup Bun if: ${{ matrix.os != 'windows-latest' }} @@ -62,11 +55,16 @@ jobs: uses: actions/setup-python@v2 if: ${{ matrix.os == 'windows-latest' }} with: - python-version: '3.12' + python-version: "3.13" - name: Install NumPy + if: ${{ matrix.os != 'macos-latest' }} run: python3 -m pip install numpy + - name: Install NumPy on MacOs + if: ${{ matrix.os == 'macos-latest' }} + run: python3 -m pip install --user --break-system-packages numpy + - name: Run deno test run: deno task test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..24b240b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,22 @@ +name: Publish + +on: + push: + branches: + - main + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - name: Checkout sources + uses: actions/checkout@v4 + + - name: Setup latest deno version + uses: denoland/setup-deno@v2 + + - name: Publish to JSR + run: deno publish diff --git a/README.md b/README.md index b7df25d..6c583c0 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,13 @@ plt.plot(xpoints, ypoints); plt.show(); ``` -When running, you **must** specify `--allow-ffi`, `--allow-env` and `--unstable` -flags. Alternatively, you may also just specify `-A` instead of specific -permissions since enabling FFI effectively escapes the permissions sandbox. +When running, you **must** specify `--allow-ffi`, `--allow-env` and +`--unstable-ffi` flags. Alternatively, you may also just specify `-A` instead of +specific permissions since enabling FFI effectively escapes the permissions +sandbox. ```shell -deno run -A --unstable +deno run -A --unstable-ffi ``` ### Usage in Bun @@ -116,6 +117,29 @@ the Python dynamic library, which is like `python310.dll` (Windows), `libpython310.dylib` (macOS) and `libpython310.so` (Linux) depending on platform. +## Usage with docker + +Usage with docker is easiest done using the +[`denoland/deno:bin` image](https://github.com/denoland/deno_docker?tab=readme-ov-file#using-your-own-base-image) +along with the [official `python` image](https://hub.docker.com/_/python/). + +```Dockerfile +ARG DENO_VERSION=1.38.2 +ARG PYTHON_VERSION=3.12 + +FROM denoland/deno:bin-$DENO_VERSION AS deno +FROM python:$PYTHON_VERSION + +# Copy and configure deno +COPY --from=deno /deno /usr/local/bin/deno +ENTRYPOINT ["/usr/local/bin/deno"] + +# Copy your project source +COPY . . + +RUN ["run", "-A", "--unstable", "https://deno.land/x/python@0.4.2/examples/hello_python.ts"] +``` + ## Maintainers - DjDeveloper ([@DjDeveloperr](https://github.com/DjDeveloperr)) diff --git a/deno.json b/deno.json index 5093226..d8901e3 100644 --- a/deno.json +++ b/deno.json @@ -1,14 +1,20 @@ { + "name": "@denosaurs/python", + "version": "0.4.5", + "exports": { + ".": "./mod.ts", + "./ext/pip": "./ext/pip.ts" + }, "tasks": { "check": "deno task check:mod && deno task check:ext && deno task check:examples", - "check:mod": "deno check --unstable mod.ts", - "check:ext": "deno check --unstable ext/*.ts", - "check:examples": "deno check --unstable examples/*.ts", - "test": "deno test --unstable -A test/test.ts", - "example:hello_python": "deno run -A --unstable examples/hello_python.ts", - "example:matplotlib": "deno run -A --unstable examples/matplotlib.ts", - "example:pip_import": "deno run -A --unstable examples/pip_import.ts", - "example:run_code": "deno run -A --unstable examples/run_code.ts", - "example:tensorflow": "deno run -A --unstable examples/tensorflow.ts" + "check:mod": "deno check --unstable-ffi mod.ts", + "check:ext": "deno check --unstable-ffi ext/*.ts", + "check:examples": "deno check --unstable-ffi examples/*.ts", + "test": "deno test --unstable-ffi -A test/test.ts", + "example:hello_python": "deno run -A --unstable-ffi examples/hello_python.ts", + "example:matplotlib": "deno run -A --unstable-ffi examples/matplotlib.ts", + "example:pip_import": "deno run -A --unstable-ffi examples/pip_import.ts", + "example:run_code": "deno run -A --unstable-ffi examples/run_code.ts", + "example:tensorflow": "deno run -A --unstable-ffi examples/tensorflow.ts" } } diff --git a/ext/pip.ts b/ext/pip.ts index d5c4dd4..d06a2fc 100644 --- a/ext/pip.ts +++ b/ext/pip.ts @@ -1,12 +1,12 @@ import { kw, python, PythonError } from "../mod.ts"; -import { join } from "https://deno.land/std@0.203.0/path/mod.ts"; -import { ensureDir } from "https://deno.land/std@0.203.0/fs/mod.ts"; -import { green, yellow } from "https://deno.land/std@0.203.0/fmt/colors.ts"; +import { join } from "jsr:@std/path@^1/join"; +import { ensureDir } from "jsr:@std/fs@^1/ensure-dir"; +import { green, yellow } from "jsr:@std/fmt@^1/colors"; -import type { CacheLocation } from "https://deno.land/x/plug@1.0.3/types.ts"; -import { ensureCacheLocation } from "https://deno.land/x/plug@1.0.3/download.ts"; -import { hash } from "https://deno.land/x/plug@1.0.3/util.ts"; +import type { CacheLocation } from "jsr:@denosaurs/plug@^1/types"; +import { ensureCacheLocation } from "jsr:@denosaurs/plug@^1/download"; +import { hash } from "jsr:@denosaurs/plug@^1/util"; const sys = python.import("sys"); const runpy = python.import("runpy"); @@ -118,7 +118,8 @@ export class Pip { * * ``` */ - async import(module: string, entrypoint?: string) { + // deno-lint-ignore no-explicit-any + async import(module: string, entrypoint?: string): Promise { const { name } = getModuleNameAndVersion(module); await this.install(module); @@ -166,5 +167,5 @@ export class Pip { } } -export const pip = new Pip(); +export const pip: Pip = new Pip(); export default pip; diff --git a/ipy.ts b/ipy.ts index 089a16b..4fed1c3 100644 --- a/ipy.ts +++ b/ipy.ts @@ -1,5 +1,5 @@ -import py, { Python } from "./mod.ts"; -import { Pip, pip } from "./ext/pip.ts"; +import py, { type Python } from "./mod.ts"; +import { type Pip, pip } from "./ext/pip.ts"; declare global { const py: Python; diff --git a/package.json b/package.json index 291f790..a87ea3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bunpy", - "version": "0.3.3", + "version": "0.4.5", "description": "JavaScript -> Python Bridge for Deno and Bun", "main": "mod.bun.ts", "directories": { diff --git a/src/ffi.ts b/src/ffi.ts index 4f26421..67a5116 100644 --- a/src/ffi.ts +++ b/src/ffi.ts @@ -3,13 +3,17 @@ import { postSetup } from "./util.ts"; const searchPath: string[] = []; -const SUPPORTED_VERSIONS = [[3, 12], [3, 11], [3, 10], [3, 9], [3, 8]]; +const SUPPORTED_VERSIONS = [[3, 13], [3, 12], [3, 11], [3, 10], [3, 9], [3, 8]]; const DENO_PYTHON_PATH = Deno.env.get("DENO_PYTHON_PATH"); if (DENO_PYTHON_PATH) { searchPath.push(DENO_PYTHON_PATH); } else { - if (Deno.build.os === "windows" || Deno.build.os === "linux") { + if ( + Deno.build.os === "windows" || Deno.build.os === "linux" || + // @ts-ignore: users reported that `windows_nt` exists at runtime + Deno.build.os === "windows_nt" + ) { searchPath.push( ...SUPPORTED_VERSIONS.map(([major, minor]) => `${Deno.build.os === "linux" ? "lib" : ""}python${major}${ @@ -44,7 +48,7 @@ for (const path of searchPath) { } catch (err) { if (err instanceof TypeError && !("Bun" in globalThis)) { throw new Error( - "Cannot load dynamic library because --unstable flag was not set", + "Cannot load dynamic library because --unstable-ffi flag was not set", { cause: err }, ); } diff --git a/src/python.ts b/src/python.ts index f8dc2ba..8054590 100644 --- a/src/python.ts +++ b/src/python.ts @@ -61,6 +61,7 @@ export interface PythonProxy { export type PythonConvertible = | number | bigint + | void | null | undefined | boolean @@ -137,7 +138,10 @@ export function kw( * ``` */ export class Callback { - unsafe; + unsafe: Deno.UnsafeCallback<{ + parameters: ["pointer", "pointer", "pointer"]; + result: "pointer"; + }>; constructor(public callback: PythonJSCallback) { this.unsafe = new Deno.UnsafeCallback( @@ -150,13 +154,38 @@ export class Callback { args: Deno.PointerValue, kwargs: Deno.PointerValue, ) => { - return PyObject.from(callback( - kwargs === null ? {} : Object.fromEntries( - new PyObject(kwargs).asDict() - .entries(), - ), - ...(args === null ? [] : new PyObject(args).valueOf()), - )).handle; + let result: PythonConvertible; + // Prepare arguments for the JS callback + try { + // Prepare arguments for the JS callback + const jsKwargs = kwargs === null + ? {} + : Object.fromEntries(new PyObject(kwargs).asDict().entries()); + const jsArgs = args === null ? [] : new PyObject(args).valueOf(); + + // Call the actual JS function + result = callback(jsKwargs, ...jsArgs); + + // Convert the JS return value back to a Python object + return PyObject.from(result).handle; + } catch (e) { + // An error occurred in the JS callback. + // We need to set a Python exception and return NULL. + + // Prepare the error message for Python + const errorMessage = e instanceof Error + ? `${e.name}: ${e.message}` // Include JS error type and message + : String(e); // Fallback for non-Error throws + const cErrorMessage = cstr(`JS Callback Error: ${errorMessage}`); + + const errorTypeHandle = + python.builtins.RuntimeError[ProxiedPyObject].handle; + + // Set the Python exception (type and message) + py.PyErr_SetString(errorTypeHandle, cErrorMessage); + + return null; + } }, ); } @@ -189,12 +218,17 @@ export class Callback { * C PyObject. */ export class PyObject { + /** + * A Python callabale object as Uint8Array + * This is used with `PyCFunction_NewEx` in order to extend its liftime and not allow v8 to release it before its actually used + */ + #pyMethodDef?: Uint8Array; constructor(public handle: Deno.PointerValue) {} /** * Check if the object is NULL (pointer) or None type in Python. */ - get isNone() { + get isNone(): boolean { // deno-lint-ignore ban-ts-comment // @ts-expect-error return this.handle === null || this.handle === 0 || @@ -397,9 +431,17 @@ export class PyObject { /** * Performs an equals operation on the Python object. */ - equals(rhs: PythonConvertible) { + equals(rhs: PythonConvertible): boolean { const rhsObject = PyObject.from(rhs); - return py.PyObject_RichCompareBool(this.handle, rhsObject.handle, 3); + const comparison = py.PyObject_RichCompareBool( + this.handle, + rhsObject.handle, + 3, + ); + if (comparison === -1) { + maybeThrowError(); + } + return comparison === 1; } /** @@ -434,7 +476,7 @@ export class PyObject { } case "object": { - if (v === null) { + if (v === null /*or void*/) { return python.builtins.None[ProxiedPyObject]; } else if (ProxiedPyObject in v) { const proxy = v as PythonProxy; @@ -446,13 +488,15 @@ export class PyObject { } return new PyObject(list); } else if (v instanceof Callback) { - const struct = new Uint8Array(8 + 8 + 4 + 8); - const view = new DataView(struct.buffer); + // https://docs.python.org/3/c-api/structures.html#c.PyMethodDef + // there are extra 4 bytes of padding after ml_flags field + const pyMethodDef = new Uint8Array(8 + 8 + 4 + 4 + 8); + const view = new DataView(pyMethodDef.buffer); const LE = new Uint8Array(new Uint32Array([0x12345678]).buffer)[0] !== 0x7; - const nameBuf = new TextEncoder().encode( - "JSCallback:" + (v.callback.name || "anonymous") + "\0", - ); + + const name = "JSCallback:" + (v.callback.name || "anonymous"); + const nameBuf = new TextEncoder().encode(`${name}\0`); view.setBigUint64( 0, BigInt(Deno.UnsafePointer.value(Deno.UnsafePointer.of(nameBuf)!)), @@ -464,17 +508,32 @@ export class PyObject { LE, ); view.setInt32(16, 0x1 | 0x2, LE); + // https://github.com/python/cpython/blob/f27593a87c344f3774ca73644a11cbd5614007ef/Objects/typeobject.c#L688 + const SIGNATURE_END_MARKER = ")\n--\n\n"; + // We're not using the correct arguments name, but just using dummy ones (because they're not accessible in js) + const fnArgs = [...Array(v.callback.length).keys()] + .map((_, i) => String.fromCharCode(97 + i)).join(","); + const docBuf = `${name}(${fnArgs}${SIGNATURE_END_MARKER}\0`; view.setBigUint64( - 20, - BigInt(Deno.UnsafePointer.value(Deno.UnsafePointer.of(nameBuf)!)), + 24, + BigInt( + Deno.UnsafePointer.value( + Deno.UnsafePointer.of(new TextEncoder().encode(docBuf))!, + ), + ), LE, ); const fn = py.PyCFunction_NewEx( - struct, + pyMethodDef, PyObject.from(null).handle, null, ); - return new PyObject(fn); + + // NOTE: we need to extend `pyMethodDef` lifetime + // Otherwise V8 can release it before the callback is called + const pyObject = new PyObject(fn); + pyObject.#pyMethodDef = pyMethodDef; + return pyObject; } else if (v instanceof PyObject) { return v; } else if (v instanceof Set) { @@ -567,7 +626,7 @@ export class PyObject { /** * Tries to set the attribute, throws an error otherwise. */ - setAttr(name: string, v: PythonConvertible) { + setAttr(name: string, v: PythonConvertible): void { if ( py.PyObject_SetAttrString( this.handle, @@ -580,35 +639,35 @@ export class PyObject { } /** Checks if Python object has an attribute of given name. */ - hasAttr(attr: string) { + hasAttr(attr: string): boolean { return py.PyObject_HasAttrString(this.handle, cstr(attr)) !== 0; } /** * Casts a Bool Python object as JS Boolean value. */ - asBoolean() { + asBoolean(): boolean { return py.PyLong_AsLong(this.handle) === 1; } /** * Casts a Int Python object as JS Number value. */ - asLong() { + asLong(): number { return py.PyLong_AsLong(this.handle) as number; } /** * Casts a Float (Double) Python object as JS Number value. */ - asDouble() { + asDouble(): number { return py.PyFloat_AsDouble(this.handle) as number; } /** * Casts a String Python object as JS String value. */ - asString() { + asString(): string | null { const str = py.PyUnicode_AsUTF8(this.handle); return str !== null ? Deno.UnsafePointerView.getCString(str) : null; } @@ -616,7 +675,7 @@ export class PyObject { /** * Casts a List Python object as JS Array value. */ - asArray() { + asArray(): PythonConvertible[] { const array: PythonConvertible[] = []; for (const i of this) { array.push(i.valueOf()); @@ -630,7 +689,7 @@ export class PyObject { * Note: `from` supports converting both Map and Object to Python Dict. * But this only supports returning a Map. */ - asDict() { + asDict(): Map { const dict = new Map(); const keys = py.PyDict_Keys(this.handle); const length = py.PyList_Size(keys) as number; @@ -646,7 +705,7 @@ export class PyObject { return dict; } - *[Symbol.iterator]() { + *[Symbol.iterator](): Generator { const iter = py.PyObject_GetIter(this.handle); let item = py.PyIter_Next(iter); while (item !== null) { @@ -659,8 +718,8 @@ export class PyObject { /** * Casts a Set Python object as JS Set object. */ - asSet() { - const set = new Set(); + asSet(): Set { + const set = new Set(); for (const i of this) { set.add(i.valueOf()); } @@ -670,7 +729,7 @@ export class PyObject { /** * Casts a Tuple Python object as JS Array value. */ - asTuple() { + asTuple(): PythonConvertible[] { const tuple = new Array(); const length = py.PyTuple_Size(this.handle) as number; for (let i = 0; i < length; i++) { @@ -688,7 +747,7 @@ export class PyObject { * Only primitives are casted as JS value type, otherwise returns * a proxy to Python object. */ - valueOf() { + valueOf(): any { const type = py.PyObject_Type(this.handle); if (Deno.UnsafePointer.equals(type, python.None[ProxiedPyObject].handle)) { @@ -736,7 +795,7 @@ export class PyObject { call( positional: (PythonConvertible | NamedArgument)[] = [], named: Record = {}, - ) { + ): PyObject { // count named arguments const namedCount = positional.filter( (arg) => arg instanceof NamedArgument, @@ -785,23 +844,23 @@ export class PyObject { /** * Returns `str` representation of the Python object. */ - toString() { + toString(): string { return new PyObject(py.PyObject_Str(this.handle)) - .asString(); + .asString()!; } - [Symbol.for("Deno.customInspect")]() { + [Symbol.for("Deno.customInspect")](): string { return this.toString(); } - [Symbol.for("nodejs.util.inspect.custom")]() { + [Symbol.for("nodejs.util.inspect.custom")](): string { return this.toString(); } } /** Python-related error. */ export class PythonError extends Error { - name = "PythonError"; + override name = "PythonError"; constructor( public type: PyObject, @@ -905,7 +964,7 @@ export class Python { /** * Runs Python script from the given string. */ - run(code: string) { + run(code: string): void { if (py.PyRun_SimpleString(cstr(code)) !== 0) { throw new EvalError("Failed to run python code"); } @@ -915,7 +974,7 @@ export class Python { * Runs Python script as a module and returns its module object, * for using its attributes, functions, classes, etc. from JavaScript. */ - runModule(code: string, name?: string) { + runModule(code: string, name?: string): any { const module = py.PyImport_ExecCodeModule( cstr(name ?? "__main__"), PyObject.from( @@ -924,6 +983,7 @@ export class Python { .handle, ); if (module === null) { + maybeThrowError(); throw new EvalError("Failed to run python module"); } return new PyObject(module)?.proxy; @@ -932,7 +992,7 @@ export class Python { /** * Import a module as PyObject. */ - importObject(name: string) { + importObject(name: string): PyObject { const mod = py.PyImport_ImportModule(cstr(name)); if (mod === null) { maybeThrowError(); @@ -944,7 +1004,7 @@ export class Python { /** * Import a Python module as a proxy object. */ - import(name: string) { + import(name: string): any { return this.importObject(name).proxy; } @@ -952,6 +1012,34 @@ export class Python { callback(cb: PythonJSCallback): Callback { return new Callback(cb); } + + /** + * Creates a Python instance method from a JavaScript callback. + * + * @description + * This method takes a JavaScript callback function and creates a Python instance method. + * + * The method returns both the created Python instance method and the Callback object. + * The Callback object is returned to allow the user to explicitly call its `destroy` + * method when it's no longer needed, ensuring proper resource management and + * freeing of memory. + * + * @example + * const [pyMethod, callback] = instanceMethod(myJSFunction); + * // Use pyMethod as needed + * // ... + * // When done, explicitly free the callback + * callback.destroy(); + */ + instanceMethod(cb: PythonJSCallback): [PyObject, Callback] { + const pythonCb = python.callback(cb); + const method = new PyObject( + py.PyInstanceMethod_New( + PyObject.from(pythonCb).handle, + ), + ); + return [method, pythonCb]; + } } /** @@ -961,7 +1049,7 @@ export class Python { * and also make use of some common built-ins attached to * this object, such as `str`, `int`, `tuple`, etc. */ -export const python = new Python(); +export const python: Python = new Python(); /** * Returns true if the value can be converted into a Python slice or diff --git a/src/symbols.ts b/src/symbols.ts index 4e84320..c7a0abc 100644 --- a/src/symbols.ts +++ b/src/symbols.ts @@ -39,6 +39,11 @@ export const SYMBOLS = { result: "void", }, + PyErr_SetString: { + parameters: ["pointer", "buffer"], // type, message + result: "void", + }, + PyDict_New: { parameters: [], result: "pointer", @@ -249,4 +254,9 @@ export const SYMBOLS = { parameters: ["buffer", "pointer", "pointer"], result: "pointer", }, + + PyInstanceMethod_New: { + parameters: ["pointer"], + result: "pointer", + }, } as const; diff --git a/test/test.ts b/test/test.ts index 6338510..8293618 100644 --- a/test/test.ts +++ b/test/test.ts @@ -5,7 +5,7 @@ import { ProxiedPyObject, PyObject, python, - PythonProxy, + type PythonProxy, } from "../mod.ts"; const { version, executable } = python.import("sys"); @@ -298,6 +298,21 @@ def call(cb): cb.destroy(); }); +Deno.test("callback returns void", () => { + const { call } = python.runModule( + ` +def call(cb): + cb() + `, + "cb_test.py", + ); + const cb = python.callback(() => { + // return void + }); + call(cb); + cb.destroy(); +}); + Deno.test("exceptions", async (t) => { await t.step("simple exception", () => { assertThrows(() => python.runModule("1 / 0")); @@ -309,3 +324,63 @@ Deno.test("exceptions", async (t) => { assertThrows(() => array.shape = [3, 6]); }); }); + +Deno.test("instance method", () => { + const { A } = python.runModule( + ` +class A: + def b(self): + return 4 + `, + "cb_test.py", + ); + + const [m, cb] = python.instanceMethod((_args, self) => { + return self.b(); + }); + // Modifying PyObject modifes A + PyObject.from(A).setAttr("a", m); + + assertEquals(new A().a.call().valueOf(), 4); + cb.destroy(); +}); + +Deno.test("callbacks have signature", async (t) => { + const inspect = python.import("inspect"); + + await t.step("empty arguments", () => { + const fn = python.callback(() => {}); + assertEquals(inspect.signature(fn).toString(), "()"); + fn.destroy(); + }); + + await t.step("with no arguments", () => { + const fn = python.callback((_f, _b, _c) => {}); + assertEquals(inspect.signature(fn).toString(), "(a, b, c)"); + fn.destroy(); + }); +}); + +Deno.test("js exception inside python callback returns python exception", () => { + const pyCallback = python.callback(() => { + throw new Error("This is an intentional error from JS!"); + }); + + const pyModule = python.runModule( + ` +def call_the_callback(cb): + result = cb() + return result + `, + "test_module", + ); + + try { + pyModule.call_the_callback(pyCallback); + } catch (e) { + // deno-lint-ignore no-explicit-any + assertEquals((e as any).name, "PythonError"); + } finally { + pyCallback.destroy(); + } +});