diff --git a/Package.swift b/Package.swift index c1ea0aa..75d1103 100644 --- a/Package.swift +++ b/Package.swift @@ -4,6 +4,9 @@ import PackageDescription let package = Package( name: "PythonKit", + platforms: [ + .macOS(.v10_15) + ], products: [ .library( name: "PythonKit", diff --git a/PythonKit/NumpyConversion.swift b/PythonKit/NumpyConversion.swift index fc73a33..e0c81a2 100644 --- a/PythonKit/NumpyConversion.swift +++ b/PythonKit/NumpyConversion.swift @@ -110,18 +110,18 @@ where Element : NumpyScalarCompatible { guard Element.numpyScalarTypes.contains(numpyArray.dtype) else { return nil } - + // Only 1-D `ndarray` instances can be converted to `Array`. let pyShape = numpyArray.__array_interface__["shape"] guard let shape = Array(pyShape) else { return nil } guard shape.count == 1 else { return nil } - + // Make sure that the array is contiguous in memory. This does a copy if // the array is not already contiguous in memory. let contiguousNumpyArray = np.ascontiguousarray(numpyArray) - + guard let ptrVal = UInt(contiguousNumpyArray.__array_interface__["data"].tuple2.0) else { return nil @@ -137,7 +137,7 @@ where Element : NumpyScalarCompatible { self.init(repeating: dummyPointer.move(), count: scalarCount) dummyPointer.deallocate() withUnsafeMutableBufferPointer { buffPtr in - buffPtr.baseAddress!.assign(from: ptr, count: scalarCount) + buffPtr.baseAddress!.update(from: ptr, count: scalarCount) } } } diff --git a/PythonKit/Python.swift b/PythonKit/Python.swift index 841a147..6718673 100644 --- a/PythonKit/Python.swift +++ b/PythonKit/Python.swift @@ -39,7 +39,7 @@ typealias OwnedPyObjectPointer = PyObjectPointer @usableFromInline @_fixed_layout final class PyReference { private var pointer: OwnedPyObjectPointer - + // This `PyReference`, once deleted, will make no delta change to the // python object's reference count. It will however, retain the reference for // the lifespan of this object. @@ -47,21 +47,26 @@ final class PyReference { self.pointer = pointer Py_IncRef(pointer) } - + // This `PyReference` adopts the +1 reference and will decrement it in the // future. init(consuming pointer: PyObjectPointer) { self.pointer = pointer } - + deinit { - Py_DecRef(pointer) + // AIQ-230: This is causing crashing in the evals. + // We don't use PythonKit in production, so in the absence of a proper fix + // we're commenting this out. + // + // This change should be reverted if we ever use PythonKit in production. + // Py_DecRef(pointer) } - + var borrowedPyObject: PyObjectPointer { return pointer } - + var ownedPyObject: OwnedPyObjectPointer { Py_IncRef(pointer) return pointer @@ -90,26 +95,26 @@ final class PyReference { public struct PythonObject { /// The underlying `PyReference`. fileprivate var reference: PyReference - + @usableFromInline init(_ pointer: PyReference) { reference = pointer } - + /// Creates a new instance and a new reference. init(_ pointer: OwnedPyObjectPointer) { reference = PyReference(pointer) } - + /// Creates a new instance consuming the specified `PyObject` pointer. init(consuming pointer: PyObjectPointer) { reference = PyReference(consuming: pointer) } - + fileprivate var borrowedPyObject: PyObjectPointer { return reference.borrowedPyObject } - + fileprivate var ownedPyObject: OwnedPyObjectPointer { return reference.ownedPyObject } @@ -165,7 +170,7 @@ fileprivate extension PythonConvertible { var borrowedPyObject: PyObjectPointer { return pythonObject.borrowedPyObject } - + var ownedPyObject: OwnedPyObjectPointer { return pythonObject.ownedPyObject } @@ -189,7 +194,7 @@ extension PythonObject : PythonConvertible, ConvertibleFromPython { public init(_ object: PythonObject) { self.init(consuming: object.ownedPyObject) } - + public var pythonObject: PythonObject { return self } } @@ -210,7 +215,7 @@ public extension PythonObject { public enum PythonError : Error, Equatable { /// A Python runtime exception, produced by calling a Python function. case exception(PythonObject, traceback: PythonObject?) - + /// A failed call on a `PythonObject`. /// Reasons for failure include: /// - A non-callable Python object was called. @@ -218,7 +223,7 @@ public enum PythonError : Error, Equatable { /// object. /// - An invalid keyword argument was specified. case invalidCall(PythonObject) - + /// A module import error. case invalidModule(String) } @@ -248,14 +253,14 @@ extension PythonError : CustomStringConvertible { // active. private func throwPythonErrorIfPresent() throws { if PyErr_Occurred() == nil { return } - + var type: PyObjectPointer? var value: PyObjectPointer? var traceback: PyObjectPointer? - + // Fetch the exception and clear the exception state. PyErr_Fetch(&type, &value, &traceback) - + // The value for the exception may not be set but the type always should be. let resultObject = PythonObject(consuming: value ?? type!) let tracebackObject = traceback.flatMap { PythonObject(consuming: $0) } @@ -271,11 +276,11 @@ private func throwPythonErrorIfPresent() throws { /// `dynamicallyCall` until further discussion/design. public struct ThrowingPythonObject { private var base: PythonObject - + fileprivate init(_ base: PythonObject) { self.base = base } - + /// Call `self` with the specified positional arguments. /// If the call fails for some reason, `PythonError.invalidCall` is thrown. /// - Precondition: `self` must be a Python callable. @@ -285,7 +290,7 @@ public struct ThrowingPythonObject { withArguments args: PythonConvertible...) throws -> PythonObject { return try dynamicallyCall(withArguments: args) } - + /// Call `self` with the specified positional arguments. /// If the call fails for some reason, `PythonError.invalidCall` is thrown. /// - Precondition: `self` must be a Python callable. @@ -294,18 +299,18 @@ public struct ThrowingPythonObject { public func dynamicallyCall( withArguments args: [PythonConvertible] = []) throws -> PythonObject { try throwPythonErrorIfPresent() - + // Positional arguments are passed as a tuple of objects. let argTuple = pyTuple(args.map { $0.pythonObject }) defer { Py_DecRef(argTuple) } - + // Python calls always return a non-null object when successful. If the // Python function produces the equivalent of C `void`, it returns the // `None` object. A `null` result of `PyObjectCall` happens when there is an // error, like `self` not being a Python callable. let selfObject = base.ownedPyObject defer { Py_DecRef(selfObject) } - + guard let result = PyObject_CallObject(selfObject, argTuple) else { // If a Python exception was thrown, throw a corresponding Swift error. try throwPythonErrorIfPresent() @@ -313,7 +318,7 @@ public struct ThrowingPythonObject { } return PythonObject(consuming: result) } - + /// Call `self` with the specified arguments. /// If the call fails for some reason, `PythonError.invalidCall` is thrown. /// - Precondition: `self` must be a Python callable. @@ -324,7 +329,7 @@ public struct ThrowingPythonObject { KeyValuePairs = [:]) throws -> PythonObject { return try _dynamicallyCall(args) } - + /// Alias for the function above that lets the caller dynamically construct the argument list, without using a dictionary literal. /// This function must be called explicitly on a `PythonObject` because `@dynamicCallable` does not recognize it. @discardableResult @@ -333,17 +338,17 @@ public struct ThrowingPythonObject { [(key: String, value: PythonConvertible)] = []) throws -> PythonObject { return try _dynamicallyCall(args) } - + /// Implementation of `dynamicallyCall(withKeywordArguments)`. private func _dynamicallyCall(_ args: T) throws -> PythonObject where T.Element == (key: String, value: PythonConvertible) { try throwPythonErrorIfPresent() - + // An array containing positional arguments. var positionalArgs: [PythonObject] = [] // A dictionary object for storing keyword arguments, if any exist. var kwdictObject: OwnedPyObjectPointer? = nil - + for (key, value) in args { if key.isEmpty { positionalArgs.append(value.pythonObject) @@ -360,20 +365,20 @@ public struct ThrowingPythonObject { Py_DecRef(k) Py_DecRef(v) } - + defer { Py_DecRef(kwdictObject) } // Py_DecRef is `nil` safe. - + // Positional arguments are passed as a tuple of objects. let argTuple = pyTuple(positionalArgs) defer { Py_DecRef(argTuple) } - + // Python calls always return a non-null object when successful. If the // Python function produces the equivalent of C `void`, it returns the // `None` object. A `null` result of `PyObjectCall` happens when there is an // error, like `self` not being a Python callable. let selfObject = base.ownedPyObject defer { Py_DecRef(selfObject) } - + guard let result = PyObject_Call(selfObject, argTuple, kwdictObject) else { // If a Python exception was thrown, throw a corresponding Swift error. try throwPythonErrorIfPresent() @@ -381,7 +386,7 @@ public struct ThrowingPythonObject { } return PythonObject(consuming: result) } - + /// Converts to a 2-tuple, if possible. public var tuple2: (PythonObject, PythonObject)? { let ct = base.checking @@ -390,7 +395,7 @@ public struct ThrowingPythonObject { } return (elt0, elt1) } - + /// Converts to a 3-tuple, if possible. public var tuple3: (PythonObject, PythonObject, PythonObject)? { let ct = base.checking @@ -399,7 +404,7 @@ public struct ThrowingPythonObject { } return (elt0, elt1, elt2) } - + /// Converts to a 4-tuple, if possible. public var tuple4: (PythonObject, PythonObject, PythonObject, PythonObject)? { let ct = base.checking @@ -409,7 +414,7 @@ public struct ThrowingPythonObject { } return (elt0, elt1, elt2, elt3) } - + public var count: Int? { base.checking.count } @@ -434,11 +439,11 @@ public extension PythonObject { public struct CheckingPythonObject { /// The underlying `PythonObject`. private var base: PythonObject - + fileprivate init(_ base: PythonObject) { self.base = base } - + public subscript(dynamicMember name: String) -> PythonObject? { get { let selfObject = base.ownedPyObject @@ -451,7 +456,7 @@ public struct CheckingPythonObject { return PythonObject(consuming: result) } } - + /// Access the element corresponding to the specified `PythonConvertible` /// values representing a key. /// - Note: This is equivalent to `object[key]` in Python. @@ -461,9 +466,10 @@ public struct CheckingPythonObject { let selfObject = base.ownedPyObject defer { Py_DecRef(keyObject) - Py_DecRef(selfObject) + // TODO: Re-enable once the use after free is figured out. + // Py_DecRef(selfObject) } - + // `PyObject_GetItem` returns +1 reference. if let result = PyObject_GetItem(selfObject, keyObject) { return PythonObject(consuming: result) @@ -478,7 +484,7 @@ public struct CheckingPythonObject { Py_DecRef(keyObject) Py_DecRef(selfObject) } - + if let newValue = newValue { let newValueObject = newValue.ownedPyObject PyObject_SetItem(selfObject, keyObject, newValueObject) @@ -489,7 +495,7 @@ public struct CheckingPythonObject { } } } - + /// Access the element corresponding to the specified `PythonConvertible` /// values representing a key. /// - Note: This is equivalent to `object[key]` in Python. @@ -501,7 +507,7 @@ public struct CheckingPythonObject { self[key] = newValue } } - + /// Converts to a 2-tuple, if possible. public var tuple2: (PythonObject, PythonObject)? { guard let elt0 = self[0], let elt1 = self[1] else { @@ -509,7 +515,7 @@ public struct CheckingPythonObject { } return (elt0, elt1) } - + /// Converts to a 3-tuple, if possible. public var tuple3: (PythonObject, PythonObject, PythonObject)? { guard let elt0 = self[0], let elt1 = self[1], let elt2 = self[2] else { @@ -517,7 +523,7 @@ public struct CheckingPythonObject { } return (elt0, elt1, elt2) } - + /// Converts to a 4-tuple, if possible. public var tuple4: (PythonObject, PythonObject, PythonObject, PythonObject)? { guard let elt0 = self[0], let elt1 = self[1], @@ -526,7 +532,7 @@ public struct CheckingPythonObject { } return (elt0, elt1, elt2, elt3) } - + public var count: Int? { Int(Python.len(base)) } @@ -559,7 +565,7 @@ public extension PythonObject { defer { Py_DecRef(selfObject) } let valueObject = newValue.ownedPyObject defer { Py_DecRef(valueObject) } - + if PyObject_SetAttrString(selfObject, memberName, valueObject) == -1 { try! throwPythonErrorIfPresent() fatalError(""" @@ -569,7 +575,7 @@ public extension PythonObject { } } } - + /// Access the element corresponding to the specified `PythonConvertible` /// values representing a key. /// - Note: This is equivalent to `object[key]` in Python. @@ -587,7 +593,7 @@ public extension PythonObject { checking[key] = newValue } } - + /// Converts to a 2-tuple. var tuple2: (PythonObject, PythonObject) { guard let result = checking.tuple2 else { @@ -595,7 +601,7 @@ public extension PythonObject { } return result } - + /// Converts to a 3-tuple. var tuple3: (PythonObject, PythonObject, PythonObject) { guard let result = checking.tuple3 else { @@ -603,7 +609,7 @@ public extension PythonObject { } return result } - + /// Converts to a 4-tuple. var tuple4: (PythonObject, PythonObject, PythonObject, PythonObject) { guard let result = checking.tuple4 else { @@ -611,7 +617,7 @@ public extension PythonObject { } return result } - + /// Call `self` with the specified positional arguments. /// - Precondition: `self` must be a Python callable. /// - Parameter args: Positional arguments for the Python callable. @@ -620,7 +626,7 @@ public extension PythonObject { withArguments args: [PythonConvertible] = []) -> PythonObject { return try! throwing.dynamicallyCall(withArguments: args) } - + /// Call `self` with the specified arguments. /// - Precondition: `self` must be a Python callable. /// - Parameter args: Positional or keyword arguments for the Python callable. @@ -630,7 +636,7 @@ public extension PythonObject { KeyValuePairs = [:]) -> PythonObject { return try! throwing.dynamicallyCall(withKeywordArguments: args) } - + /// Alias for the function above that lets the caller dynamically construct the argument list, without using a dictionary literal. /// This function must be called explicitly on a `PythonObject` because `@dynamicCallable` does not recognize it. @discardableResult @@ -673,16 +679,19 @@ public let Python = PythonInterface() public struct PythonInterface { /// A dictionary of the Python builtins. public let builtins: PythonObject - + + /// Native extension module. + internal let module: PythonModule + init() { Py_Initialize() // Initialize Python builtins = PythonObject(PyEval_GetBuiltins()) - + // Runtime Fixes: PyRun_SimpleString(""" import sys import os - + # Some Python modules expect to have at least one argument in `sys.argv`: sys.argv = [""] @@ -693,8 +702,10 @@ public struct PythonInterface { executable_name = "python{}.{}".format(sys.version_info.major, sys.version_info.minor) sys.executable = os.path.join(sys.exec_prefix, "bin", executable_name) """) + + module = PythonModule() } - + public func attemptImport(_ name: String) throws -> PythonObject { guard let module = PyImport_ImportModule(name) else { try throwPythonErrorIfPresent() @@ -702,27 +713,27 @@ public struct PythonInterface { } return PythonObject(consuming: module) } - + public func `import`(_ name: String) -> PythonObject { return try! attemptImport(name) } - + public subscript(dynamicMember name: String) -> PythonObject { return builtins[name] } - + // The Python runtime version. // Equivalent to `sys.version` in Python. public var version: PythonObject { return self.import("sys").version } - + // The Python runtime version information. // Equivalent to `sys.version_info` in Python. public var versionInfo: PythonObject { return self.import("sys").version_info } - + /// Emulates a Python `with` statement. /// - Parameter object: A context manager object. /// - Parameter body: A closure to call on the result of `object.__enter__()`. @@ -733,6 +744,48 @@ public struct PythonInterface { } } +//===----------------------------------------------------------------------===// +// Helper to enter and leave the GIL for non-Python created threads. +// See https://docs.python.org/3/c-api/init.html#non-python-created-threads +//===----------------------------------------------------------------------===// + +class PythonThread { + private var state: PyGILState_State? + + /// Enter the GIL and initialize Python thread state. + func enterGIL() { + guard state == nil else { + fatalError("The GIL is already held by this thread.") + } + state = PyGILState_Ensure() + } + + /// Exit the GIL. + func leaveGIL() { + guard let s = state else { + fatalError("The GIL is not held by this thread.") + } + PyGILState_Release(s) + state = nil + } +} + +/// Execute body while holding the GIL. +public func withGIL(_ body: () throws -> T) rethrows -> T { + let thread = PythonThread() + thread.enterGIL() + defer { thread.leaveGIL() } + return try body() +} + +/// Leave the GIL, execute the body, then re-enter the GIL. +public func withoutGIL(_ body: () throws -> T) rethrows -> T { + let thread = PythonThread() + thread.leaveGIL() + defer { thread.enterGIL() } + return try body() +} + //===----------------------------------------------------------------------===// // Helpers for Python tuple types //===----------------------------------------------------------------------===// @@ -754,12 +807,12 @@ public extension PythonObject { init(tupleOf elements: PythonConvertible...) { self.init(tupleContentsOf: elements) } - + init(tupleContentsOf elements: T) where T.Element == PythonConvertible { self.init(consuming: pyTuple(elements.map { $0.pythonObject })) } - + init(tupleContentsOf elements: T) where T.Element : PythonConvertible { self.init(consuming: pyTuple(elements)) @@ -775,14 +828,14 @@ public extension PythonObject { private func isType(_ object: PythonObject, type: PyObjectPointer) -> Bool { let typePyRef = PythonObject(type) - + let result = Python.isinstance(object, typePyRef) - + // We cannot use the normal failable Bool initializer from `PythonObject` // here because would cause an infinite loop. let pyObject = result.ownedPyObject defer { Py_DecRef(pyObject) } - + // Anything not equal to `Py_ZeroStruct` is truthy. return pyObject != _Py_ZeroStruct } @@ -790,13 +843,13 @@ private func isType(_ object: PythonObject, extension Bool : PythonConvertible, ConvertibleFromPython { public init?(_ pythonObject: PythonObject) { guard isType(pythonObject, type: PyBool_Type) else { return nil } - + let pyObject = pythonObject.ownedPyObject defer { Py_DecRef(pyObject) } - + self = pyObject == _Py_TrueStruct } - + public var pythonObject: PythonObject { _ = Python // Ensure Python is initialized. return PythonObject(consuming: PyBool_FromLong(self ? 1 : 0)) @@ -807,14 +860,14 @@ extension String : PythonConvertible, ConvertibleFromPython { public init?(_ pythonObject: PythonObject) { let pyObject = pythonObject.ownedPyObject defer { Py_DecRef(pyObject) } - + guard let cString = PyString_AsString(pyObject) else { PyErr_Clear() return nil } self.init(cString: cString) } - + public var pythonObject: PythonObject { _ = Python // Ensure Python is initialized. let v = utf8CString.withUnsafeBufferPointer { @@ -834,17 +887,17 @@ fileprivate extension PythonObject { ) -> T? { let pyObject = ownedPyObject defer { Py_DecRef(pyObject) } - + assert(PyErr_Occurred() == nil, "Python error occurred somewhere but wasn't handled") - + let value = converter(pyObject) guard value != errorValue || PyErr_Occurred() == nil else { PyErr_Clear() return nil } return value - + } } @@ -858,7 +911,7 @@ extension Int : PythonConvertible, ConvertibleFromPython { } self = value } - + public var pythonObject: PythonObject { _ = Python // Ensure Python is initialized. return PythonObject(consuming: PyInt_FromLong(self)) @@ -876,7 +929,7 @@ extension UInt : PythonConvertible, ConvertibleFromPython { } self = value } - + public var pythonObject: PythonObject { _ = Python // Ensure Python is initialized. return PythonObject(consuming: PyInt_FromSize_t(self)) @@ -893,7 +946,7 @@ extension Double : PythonConvertible, ConvertibleFromPython { } self = value } - + public var pythonObject: PythonObject { _ = Python // Ensure Python is initialized. return PythonObject(consuming: PyFloat_FromDouble(self)) @@ -912,7 +965,7 @@ extension Int8 : PythonConvertible, ConvertibleFromPython { guard let i = Int(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return Int(self).pythonObject } @@ -923,7 +976,7 @@ extension Int16 : PythonConvertible, ConvertibleFromPython { guard let i = Int(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return Int(self).pythonObject } @@ -934,7 +987,7 @@ extension Int32 : PythonConvertible, ConvertibleFromPython { guard let i = Int(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return Int(self).pythonObject } @@ -945,7 +998,7 @@ extension Int64 : PythonConvertible, ConvertibleFromPython { guard let i = Int(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return Int(self).pythonObject } @@ -956,7 +1009,7 @@ extension UInt8 : PythonConvertible, ConvertibleFromPython { guard let i = UInt(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return UInt(self).pythonObject } @@ -967,7 +1020,7 @@ extension UInt16 : PythonConvertible, ConvertibleFromPython { guard let i = UInt(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return UInt(self).pythonObject } @@ -978,7 +1031,7 @@ extension UInt32 : PythonConvertible, ConvertibleFromPython { guard let i = UInt(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return UInt(self).pythonObject } @@ -989,7 +1042,7 @@ extension UInt64 : PythonConvertible, ConvertibleFromPython { guard let i = UInt(pythonObject) else { return nil } self.init(i) } - + public var pythonObject: PythonObject { return UInt(self).pythonObject } @@ -1002,7 +1055,7 @@ extension Float : PythonConvertible, ConvertibleFromPython { guard let v = Double(pythonObject) else { return nil } self.init(v) } - + public var pythonObject: PythonObject { return Double(self).pythonObject } @@ -1087,12 +1140,12 @@ extension Dictionary : ConvertibleFromPython where Key : ConvertibleFromPython, Value : ConvertibleFromPython { public init?(_ pythonDict: PythonObject) { self = [:] - + // Iterate over the Python dictionary, converting its keys and values to // Swift `Key` and `Value` pairs. var key, value: PyObjectPointer? var position: Int = 0 - + while PyDict_Next( pythonDict.borrowedPyObject, &position, &key, &value) != 0 { @@ -1204,7 +1257,7 @@ public extension PythonObject { static func + (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return performBinaryOp(PyNumber_Add, lhs: lhs, rhs: rhs) } - + static func - (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return performBinaryOp(PyNumber_Subtract, lhs: lhs, rhs: rhs) } @@ -1212,23 +1265,23 @@ public extension PythonObject { static func * (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return performBinaryOp(PyNumber_Multiply, lhs: lhs, rhs: rhs) } - + static func / (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return performBinaryOp(PyNumber_TrueDivide, lhs: lhs, rhs: rhs) } - + static func += (lhs: inout PythonObject, rhs: PythonObject) { lhs = performBinaryOp(PyNumber_InPlaceAdd, lhs: lhs, rhs: rhs) } - + static func -= (lhs: inout PythonObject, rhs: PythonObject) { lhs = performBinaryOp(PyNumber_InPlaceSubtract, lhs: lhs, rhs: rhs) } - + static func *= (lhs: inout PythonObject, rhs: PythonObject) { lhs = performBinaryOp(PyNumber_InPlaceMultiply, lhs: lhs, rhs: rhs) } - + static func /= (lhs: inout PythonObject, rhs: PythonObject) { lhs = performBinaryOp(PyNumber_InPlaceTrueDivide, lhs: lhs, rhs: rhs) } @@ -1268,9 +1321,9 @@ extension PythonObject : SignedNumeric { public init(exactly value: T) { self.init(Int(value)) } - + public typealias Magnitude = PythonObject - + public var magnitude: PythonObject { return self < 0 ? -self : self } @@ -1284,11 +1337,11 @@ extension PythonObject : SignedNumeric { extension PythonObject : Strideable { public typealias Stride = PythonObject - + public func distance(to other: PythonObject) -> Stride { return other - self } - + public func advanced(by stride: Stride) -> PythonObject { return self + stride } @@ -1318,7 +1371,7 @@ extension PythonObject : Equatable, Comparable { public static func == (lhs: PythonObject, rhs: PythonObject) -> Bool { return lhs.compared(to: rhs, byOp: Py_EQ) } - + public static func != (lhs: PythonObject, rhs: PythonObject) -> Bool { return lhs.compared(to: rhs, byOp: Py_NE) } @@ -1330,11 +1383,11 @@ extension PythonObject : Equatable, Comparable { public static func <= (lhs: PythonObject, rhs: PythonObject) -> Bool { return lhs.compared(to: rhs, byOp: Py_LE) } - + public static func > (lhs: PythonObject, rhs: PythonObject) -> Bool { return lhs.compared(to: rhs, byOp: Py_GT) } - + public static func >= (lhs: PythonObject, rhs: PythonObject) -> Bool { return lhs.compared(to: rhs, byOp: Py_GE) } @@ -1357,27 +1410,27 @@ public extension PythonObject { } return PythonObject(consuming: result) } - + static func == (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_EQ) } - + static func != (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_NE) } - + static func < (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_LT) } - + static func <= (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_LE) } - + static func > (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_GT) } - + static func >= (lhs: PythonObject, rhs: PythonObject) -> PythonObject { return lhs.compared(to: rhs, byOp: Py_GE) } @@ -1395,19 +1448,19 @@ extension PythonObject : Hashable { extension PythonObject : MutableCollection { public typealias Index = PythonObject public typealias Element = PythonObject - + public var startIndex: Index { return 0 } - + public var endIndex: Index { return Python.len(self) } - + public func index(after i: Index) -> Index { return i + PythonObject(1) } - + public subscript(index: PythonObject) -> PythonObject { get { return self[index as PythonConvertible] @@ -1421,7 +1474,7 @@ extension PythonObject : MutableCollection { extension PythonObject : Sequence { public struct Iterator : IteratorProtocol { fileprivate let pythonIterator: PythonObject - + public func next() -> PythonObject? { guard let result = PyIter_Next(self.pythonIterator.borrowedPyObject) else { try! throwPythonErrorIfPresent() @@ -1430,7 +1483,7 @@ extension PythonObject : Sequence { return PythonObject(consuming: result) } } - + public func makeIterator() -> Iterator { guard let result = PyObject_GetIter(borrowedPyObject) else { try! throwPythonErrorIfPresent() @@ -1610,47 +1663,47 @@ final class PyFunction { case varArgs case varArgsWithKeywords } - + /// Allows `PyFunction` to store Python functions with more than one possible calling convention var callingConvention: CallingConvention - + /// `arguments` is a Python tuple. typealias VarArgsFunction = ( _ arguments: PythonObject) throws -> PythonConvertible - + /// `arguments` is a Python tuple. /// `keywordArguments` is an OrderedDict in Python 3.6 and later, or a dict otherwise. typealias VarArgsWithKeywordsFunction = ( _ arguments: PythonObject, _ keywordArguments: PythonObject) throws -> PythonConvertible - + /// Has the same memory layout as any other function with the Swift calling convention private typealias Storage = () throws -> PythonConvertible - + /// Stores all function pointers in the same stored property. `callAsFunction` casts this into the desired type. private var callSwiftFunction: Storage - + init(_ callSwiftFunction: @escaping VarArgsFunction) { self.callingConvention = .varArgs self.callSwiftFunction = unsafeBitCast(callSwiftFunction, to: Storage.self) } - + init(_ callSwiftFunction: @escaping VarArgsWithKeywordsFunction) { self.callingConvention = .varArgsWithKeywords self.callSwiftFunction = unsafeBitCast(callSwiftFunction, to: Storage.self) } - + private func checkConvention(_ calledConvention: CallingConvention) { precondition(callingConvention == calledConvention, "Called PyFunction with convention \(calledConvention), but expected \(callingConvention)") } - + func callAsFunction(_ argumentsTuple: PythonObject) throws -> PythonConvertible { checkConvention(.varArgs) let callSwiftFunction = unsafeBitCast(self.callSwiftFunction, to: VarArgsFunction.self) return try callSwiftFunction(argumentsTuple) } - + func callAsFunction(_ argumentsTuple: PythonObject, _ keywordArguments: PythonObject) throws -> PythonConvertible { checkConvention(.varArgsWithKeywords) let callSwiftFunction = unsafeBitCast(self.callSwiftFunction, to: VarArgsWithKeywordsFunction.self) @@ -1661,21 +1714,76 @@ final class PyFunction { public struct PythonFunction { /// Called directly by the Python C API private var function: PyFunction - + + /// Name of the function in Python. + private var name: StaticString? + @_disfavoredOverload public init(_ fn: @escaping (PythonObject) throws -> PythonConvertible) { function = PyFunction { argumentsAsTuple in return try fn(argumentsAsTuple[0]) } } - + + @_disfavoredOverload + public init(name: StaticString?, _ fn: @escaping (PythonObject) throws -> PythonConvertible) { + self.init(fn) + self.name = name + } + + @_disfavoredOverload + public init(name: StaticString?, awaitable fn: @escaping (PythonObject) async throws -> PythonConvertible) { + function = PyFunction { argumentsAsTuple in + let swiftArg = argumentsAsTuple[0] + let loop = Python.import("asyncio").get_running_loop() + let future = loop.create_future() + Task { + var err: Error? + var result: PythonConvertible? + do { + result = try await fn(swiftArg) + } catch { + err = error + } + Self.completeFuture(loop, future, result, err) + } + return future + } + self.name = name + } + /// For cases where the Swift function should accept more (or less) than one parameter, accept an ordered array of all arguments instead. public init(_ fn: @escaping ([PythonObject]) throws -> PythonConvertible) { function = PyFunction { argumentsAsTuple in return try fn(argumentsAsTuple.map { $0 }) } } - + + public init(name: StaticString?, _ fn: @escaping ([PythonObject]) throws -> PythonConvertible) { + self.init(fn) + self.name = name + } + + public init(name: StaticString?, awaitable fn: @escaping ([PythonObject]) async throws -> PythonConvertible) { + function = PyFunction { argumentsAsTuple in + let swiftArgs = argumentsAsTuple.map { $0 } + let loop = Python.import("asyncio").get_running_loop() + let future = loop.create_future() + Task { + var err: Error? + var result: PythonConvertible? + do { + result = try await fn(swiftArgs) + } catch { + err = error + } + Self.completeFuture(loop, future, result, err) + } + return future + } + self.name = name + } + /// For cases where the Swift function should accept keyword arguments as `**kwargs` in Python. /// `**kwargs` must preserve order from Python 3.6 onward, similarly to /// Swift `KeyValuePairs` and unlike `Dictionary`. `KeyValuePairs` cannot be @@ -1690,6 +1798,52 @@ public struct PythonFunction { return try fn(argumentsAsTuple.map { $0 }, kwargs) } } + + public init(name: StaticString?, _ fn: @escaping ([PythonObject], [(key: String, value: PythonObject)]) throws -> PythonConvertible) { + self.init(fn) + self.name = name + } + + public init(name: StaticString?, awaitable fn: @escaping ([PythonObject], [(key: String, value: PythonObject)]) async throws -> PythonConvertible) { + function = PyFunction { argumentsAsTuple, keywordArgumentsAsDictionary in + let swiftArgs = argumentsAsTuple.map { $0 } + let swiftKwargs = keywordArgumentsAsDictionary.items().compactMap { keyAndValue -> (String, PythonObject)? in + guard let key = String(keyAndValue.tuple2.0) else { + return nil + } + return (key, keyAndValue.tuple2.1) + } + + let loop = Python.import("asyncio").get_running_loop() + let future = loop.create_future() + Task { + var err: Error? + var result: PythonConvertible? + do { + result = try await fn(swiftArgs, swiftKwargs) + } catch { + err = error + } + Self.completeFuture(loop, future, result, err) + } + return future + } + self.name = name + } + + private static func completeFuture( + _ loop: PythonObject, _ future: PythonObject, _ result: PythonConvertible?, _ error: Error?) { + withGIL { + if let error { + loop.call_soon_threadsafe(future.set_exception, Python.ValueError("\(error)")) + } else { + guard let result else { + fatalError("Result should exist if error is nil.") + } + loop.call_soon_threadsafe(future.set_result, result) + } + } + } } extension PythonFunction : PythonConvertible { @@ -1713,12 +1867,22 @@ extension PythonFunction : PythonConvertible { ) var methodDefinition: UnsafeMutablePointer - switch function.callingConvention { - case .varArgs: - methodDefinition = PythonFunction.sharedMethodDefinition - case .varArgsWithKeywords: - methodDefinition = PythonFunction.sharedMethodWithKeywordsDefinition + if let name { + switch function.callingConvention { + case .varArgs: + methodDefinition = PythonFunction.methodDefinition(name: name) + case .varArgsWithKeywords: + methodDefinition = PythonFunction.methodWithKeywordsDefinition(name: name) + } + } else { + switch function.callingConvention { + case .varArgs: + methodDefinition = PythonFunction.sharedMethodDefinition + case .varArgsWithKeywords: + methodDefinition = PythonFunction.sharedMethodWithKeywordsDefinition + } } + let pyFuncPointer = PyCFunction_NewEx( methodDefinition, capsulePointer, @@ -1751,7 +1915,27 @@ fileprivate extension PythonFunction { return pointer }() - + + static func methodDefinition(name: StaticString) -> UnsafeMutablePointer { + let namePointer = UnsafeRawPointer(name.utf8Start).assumingMemoryBound(to: Int8.self) + + let methodImplementationPointer = unsafeBitCast( + PythonFunction.sharedMethodImplementation, to: OpaquePointer.self) + + /// The standard calling convention. See Python C API docs + let METH_VARARGS = 0x0001 as Int32 + + let pointer = UnsafeMutablePointer.allocate(capacity: 1) + pointer.pointee = PyMethodDef( + ml_name: namePointer, + ml_meth: methodImplementationPointer, + ml_flags: METH_VARARGS, + ml_doc: nil + ) + + return pointer + } + static let sharedMethodWithKeywordsDefinition: UnsafeMutablePointer = { let name: StaticString = "pythonkit_swift_function_with_keywords" // `utf8Start` is a property of StaticString, thus, it has a stable pointer. @@ -1775,6 +1959,27 @@ fileprivate extension PythonFunction { return pointer }() + static func methodWithKeywordsDefinition(name: StaticString) -> UnsafeMutablePointer { + let namePointer = UnsafeRawPointer(name.utf8Start).assumingMemoryBound(to: Int8.self) + + let methodImplementationPointer = unsafeBitCast( + PythonFunction.sharedMethodWithKeywordsImplementation, to: OpaquePointer.self) + + /// A combination of flags that supports `**kwargs`. See Python C API docs + let METH_VARARGS = 0x0001 as Int32 + let METH_KEYWORDS = 0x0002 as Int32 + + let pointer = UnsafeMutablePointer.allocate(capacity: 1) + pointer.pointee = PyMethodDef( + ml_name: namePointer, + ml_meth: methodImplementationPointer, + ml_flags: METH_VARARGS | METH_KEYWORDS, + ml_doc: nil + ) + + return pointer + } + private static let sharedMethodImplementation: @convention(c) ( PyObjectPointer?, PyObjectPointer? ) -> PyObjectPointer? = { context, argumentsPointer in @@ -1793,7 +1998,7 @@ fileprivate extension PythonFunction { return nil // This must only be `nil` if an exception has been set } } - + private static let sharedMethodWithKeywordsImplementation: @convention(c) ( PyObjectPointer?, PyObjectPointer?, PyObjectPointer? ) -> PyObjectPointer? = { context, argumentsPointer, keywordArgumentsPointer in @@ -1840,16 +2045,51 @@ fileprivate extension PythonFunction { } } +//===----------------------------------------------------------------------===// +// PythonModule functions that need access to PyRef members. +//===----------------------------------------------------------------------===// + +extension PythonModule { + func addObject(_ object: PyObjectPointer, named: StaticString) -> Bool { + let module = pythonObject.borrowedPyObject + let result = PyModule_AddObject( + module, + UnsafeRawPointer(named.utf8Start).assumingMemoryBound(to: Int8.self), + object) + + guard result >= 0 else { + fatalError("Failed to add type to module: \(result)") + } + return true + } + + func addType(_ typeDefinition: PyTypeObjectPointer) -> Bool { + let result = PyType_Ready(typeDefinition) + guard result >= 0 else { + fatalError("Failed to add type to module: \(result)") + } + return true + } + + static var testAwaitableFunction: PythonFunction? + static let getTestAwaitable: PyCFunction = { _, _ in + guard let testAwaitableFunction = Self.testAwaitableFunction else { + fatalError("testAwaitableFunction not set") + } + return testAwaitableFunction.pythonObject.ownedPyObject + } +} + extension PythonObject: Error {} // From Python's C Headers: struct PyMethodDef { /// The name of the built-in function/method - var ml_name: UnsafePointer + var ml_name: UnsafePointer? /// The C function that implements it. /// Since this accepts multiple function signatures, the Swift type must be opaque here. - var ml_meth: OpaquePointer + var ml_meth: OpaquePointer? /// Combination of METH_xxx flags, which mostly describe the args expected by the C func var ml_flags: Int32 @@ -1864,16 +2104,16 @@ struct PyMethodDef { public struct PythonInstanceMethod { private var function: PythonFunction - + @_disfavoredOverload public init(_ fn: @escaping (PythonObject) throws -> PythonConvertible) { function = PythonFunction(fn) } - + public init(_ fn: @escaping ([PythonObject]) throws -> PythonConvertible) { function = PythonFunction(fn) } - + public init(_ fn: @escaping ([PythonObject], [(key: String, value: PythonObject)]) throws -> PythonConvertible) { function = PythonFunction(fn) } @@ -1893,18 +2133,18 @@ extension PythonInstanceMethod : PythonConvertible { public struct PythonClass { private var typeObject: PythonObject - + public struct Members: ExpressibleByDictionaryLiteral { public typealias Key = String public typealias Value = PythonConvertible - + var dictionary: [String: PythonObject] - + public init(dictionaryLiteral elements: (Key, Value)...) { let castedElements = elements.map { (key, value) in (key, value.pythonObject) } - + dictionary = Dictionary(castedElements, uniquingKeysWith: { _, _ in fatalError("Dictionary literal contains duplicate keys") }) @@ -1914,14 +2154,14 @@ public struct PythonClass { public init(_ name: String, superclasses: [PythonObject] = [], members: Members = [:]) { self.init(name, superclasses: superclasses, members: members.dictionary) } - + @_disfavoredOverload public init(_ name: String, superclasses: [PythonObject] = [], members: [String: PythonObject] = [:]) { var trueSuperclasses = superclasses if !trueSuperclasses.contains(Python.object) { trueSuperclasses.append(Python.object) } - + let superclassesTuple = PythonObject(tupleContentsOf: trueSuperclasses) typeObject = Python.type(name, superclassesTuple, members.pythonObject) } diff --git a/PythonKit/PythonLibrary+Symbols.swift b/PythonKit/PythonLibrary+Symbols.swift index c4db681..15e2f8f 100644 --- a/PythonKit/PythonLibrary+Symbols.swift +++ b/PythonKit/PythonLibrary+Symbols.swift @@ -34,6 +34,10 @@ let Py_NE: Int32 = 3 let Py_GT: Int32 = 4 let Py_GE: Int32 = 5 +typealias PyGILState_State = Int32 +let Py_GILState_Locked: Int32 = 0 +let Py_GILState_Unlocked: Int32 = 1 + //===----------------------------------------------------------------------===// // Python library symbols lazily loaded at runtime. //===----------------------------------------------------------------------===// @@ -47,10 +51,46 @@ let Py_IncRef: @convention(c) (PyObjectPointer?) -> Void = let Py_DecRef: @convention(c) (PyObjectPointer?) -> Void = PythonLibrary.loadSymbol(name: "Py_DecRef") +let PyGILState_Ensure: @convention(c) () -> PyGILState_State = + PythonLibrary.loadSymbol(name: "PyGILState_Ensure") + +let PyGILState_Release: @convention(c) (PyGILState_State) -> Void = + PythonLibrary.loadSymbol(name: "PyGILState_Release") + let PyImport_ImportModule: @convention(c) ( PyCCharPointer) -> PyObjectPointer? = PythonLibrary.loadSymbol(name: "PyImport_ImportModule") +let PyImport_GetModuleDict: @convention(c) () -> PyObjectPointer = + PythonLibrary.loadSymbol(name: "PyImport_GetModuleDict") + +let _PyImport_FixupExtensionObject: @convention(c) ( + PyObjectPointer, PyObjectPointer, PyObjectPointer, PyObjectPointer) -> Int = + PythonLibrary.loadSymbol(name: "_PyImport_FixupExtensionObject") + +let PyUnicode_InternFromString: @convention(c) ( + PyCCharPointer) -> PyObjectPointer = + PythonLibrary.loadSymbol(name: "PyUnicode_InternFromString") + +// The 2nd argument is the api version. +let PyModule_Create: @convention(c) ( + PyModuleDefPointer, Int) -> PyObjectPointer = + PythonLibrary.loadSymbol(name: "PyModule_Create2") + +let PyModule_AddObject: @convention(c) ( + PyObjectPointer, PyCCharPointer, PyObjectPointer) -> Int = + PythonLibrary.loadSymbol(name: "PyModule_AddObject") + +let PyType_Ready: @convention(c) (PyTypeObjectPointer) -> Int = + PythonLibrary.loadSymbol(name: "PyType_Ready") + +let PyType_GenericAlloc: @convention(c) ( + PyTypeObjectPointer, Int) -> PyObjectPointer? = + PythonLibrary.loadSymbol(name: "PyType_GenericAlloc") + +let PyObject_Free: @convention(c) (UnsafeMutableRawPointer) -> Void = + PythonLibrary.loadSymbol(name: "PyObject_Free") + let PyEval_GetBuiltins: @convention(c) () -> PyObjectPointer = PythonLibrary.loadSymbol(name: "PyEval_GetBuiltins") diff --git a/PythonKit/PythonLibrary.swift b/PythonKit/PythonLibrary.swift index 7befe64..5755821 100644 --- a/PythonKit/PythonLibrary.swift +++ b/PythonKit/PythonLibrary.swift @@ -32,7 +32,7 @@ import WinSDK public struct PythonLibrary { public enum Error: Swift.Error, Equatable, CustomStringConvertible { case pythonLibraryNotFound - + public var description: String { switch self { case .pythonLibraryNotFound: @@ -43,23 +43,23 @@ public struct PythonLibrary { } } } - + private static let pythonInitializeSymbolName = "Py_Initialize" private static let pythonLegacySymbolName = "PyString_AsString" - + #if canImport(Darwin) private static let defaultLibraryHandle = UnsafeMutableRawPointer(bitPattern: -2) // RTLD_DEFAULT #else private static let defaultLibraryHandle: UnsafeMutableRawPointer? = nil // RTLD_DEFAULT #endif - + private static var isPythonLibraryLoaded = false private static var _pythonLibraryHandle: UnsafeMutableRawPointer? private static var pythonLibraryHandle: UnsafeMutableRawPointer? { try! PythonLibrary.loadLibrary() return self._pythonLibraryHandle } - + /// Tries to load the Python library, will throw an error if no compatible library is found. public static func loadLibrary() throws { guard !self.isPythonLibraryLoaded else { return } @@ -70,7 +70,7 @@ public struct PythonLibrary { self.isPythonLibraryLoaded = true self._pythonLibraryHandle = pythonLibraryHandle } - + private static let isLegacyPython: Bool = { let isLegacyPython = PythonLibrary.loadSymbol(PythonLibrary.pythonLibraryHandle, PythonLibrary.pythonLegacySymbolName) != nil if isLegacyPython { @@ -78,16 +78,17 @@ public struct PythonLibrary { } return isLegacyPython }() - + internal static func loadSymbol( name: String, legacyName: String? = nil, type: T.Type = T.self) -> T { var name = name if let legacyName = legacyName, self.isLegacyPython { name = legacyName } - + log("Loading symbol '\(name)' from the Python library...") - return unsafeBitCast(self.loadSymbol(self.pythonLibraryHandle, name), to: type) + let ptr = self.loadSymbol(self.pythonLibraryHandle, name) + return unsafeBitCast(ptr, to: type) } } @@ -95,9 +96,9 @@ public struct PythonLibrary { extension PythonLibrary { private static let supportedMajorVersions: [Int] = [3, 2] private static let supportedMinorVersions: [Int] = Array(0...30).reversed() - + private static let libraryPathVersionCharacter: Character = ":" - + #if canImport(Darwin) private static var libraryNames = ["Python.framework/Versions/:/Python"] private static var libraryPathExtensions = [""] @@ -114,7 +115,7 @@ extension PythonLibrary { private static var librarySearchPaths = [""] private static var libraryVersionSeparator = "" #endif - + private static let libraryPaths: [String] = { var libraryPaths: [String] = [] for librarySearchPath in librarySearchPaths { @@ -128,7 +129,7 @@ extension PythonLibrary { } return libraryPaths }() - + private static func loadSymbol( _ libraryHandle: UnsafeMutableRawPointer?, _ name: String) -> UnsafeMutableRawPointer? { #if os(Windows) @@ -141,12 +142,12 @@ extension PythonLibrary { return dlsym(libraryHandle, name) #endif } - + private static func isPythonLibraryLoaded(at pythonLibraryHandle: UnsafeMutableRawPointer? = nil) -> Bool { let pythonLibraryHandle = pythonLibraryHandle ?? self.defaultLibraryHandle return self.loadSymbol(pythonLibraryHandle, self.pythonInitializeSymbolName) != nil } - + private static func loadPythonLibrary() -> UnsafeMutableRawPointer? { let pythonLibraryHandle: UnsafeMutableRawPointer? if self.isPythonLibraryLoaded() { @@ -160,7 +161,7 @@ extension PythonLibrary { } return pythonLibraryHandle } - + private static func findAndLoadExternalPythonLibrary() -> UnsafeMutableRawPointer? { for majorVersion in supportedMajorVersions { for minorVersion in supportedMinorVersions { @@ -176,11 +177,11 @@ extension PythonLibrary { } return nil } - + private static func loadPythonLibrary( at path: String, version: PythonVersion) -> UnsafeMutableRawPointer? { let versionString = version.versionString - + if let requiredPythonVersion = Environment.version.value { let requiredMajorVersion = Int(requiredPythonVersion) if requiredPythonVersion != versionString, @@ -188,7 +189,7 @@ extension PythonLibrary { return nil } } - + let libraryVersionString = versionString .split(separator: PythonVersion.versionSeparator) .joined(separator: libraryVersionSeparator) @@ -196,7 +197,7 @@ extension PythonLibrary { .joined(separator: libraryVersionString) return self.loadPythonLibrary(at: path) } - + private static func loadPythonLibrary(at path: String) -> UnsafeMutableRawPointer? { self.log("Trying to load library at '\(path)'...") #if os(Windows) @@ -206,7 +207,7 @@ extension PythonLibrary { // modules may depend on this .so file. let pythonLibraryHandle = dlopen(path, RTLD_LAZY | RTLD_GLOBAL) #endif - + if pythonLibraryHandle != nil { self.log("Library at '\(path)' was successfully loaded.") } @@ -222,7 +223,7 @@ extension PythonLibrary { has already been loaded. """) } - + /// Use the Python library with the specified version. /// - Parameters: /// - major: Major version or nil to use any Python version. @@ -232,7 +233,7 @@ extension PythonLibrary { let version = PythonVersion(major: major, minor: minor) PythonLibrary.Environment.version.set(version.versionString) } - + /// Use the Python library at the specified path. /// - Parameter path: Path of the Python library to load or nil to use the default search path. public static func useLibrary(at path: String?) { @@ -246,9 +247,9 @@ extension PythonLibrary { private struct PythonVersion { let major: Int? let minor: Int? - + static let versionSeparator: Character = "." - + init(major: Int?, minor: Int?) { precondition(!(major == nil && minor != nil), """ Error: The Python library minor version cannot be specified \ @@ -257,7 +258,7 @@ extension PythonLibrary { self.major = major self.minor = minor } - + var versionString: String { guard let major = major else { return "" } var versionString = String(major) @@ -274,22 +275,22 @@ extension PythonLibrary { private enum Environment: String { private static let keyPrefix = "PYTHON" private static let keySeparator = "_" - + case library = "LIBRARY" case version = "VERSION" case loaderLogging = "LOADER_LOGGING" - + var key: String { return Environment.keyPrefix + Environment.keySeparator + rawValue } - + var value: String? { guard let cString = getenv(key) else { return nil } let value = String(cString: cString) guard !value.isEmpty else { return nil } return value } - + func set(_ value: String) { #if os(Windows) _putenv_s(key, value) diff --git a/PythonKit/PythonModule.swift b/PythonKit/PythonModule.swift new file mode 100644 index 0000000..7baf93d --- /dev/null +++ b/PythonKit/PythonModule.swift @@ -0,0 +1,216 @@ +//===--PythonModule.swift -------------------------------------------------===// +// This file defines a custom extension module for PythonKit. +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Types from the CPython headers and related. +//===----------------------------------------------------------------------===// + +typealias PyModuleDefPointer = UnsafeMutableRawPointer +typealias PyTypeObjectPointer = UnsafeMutableRawPointer + +typealias allocfunc = @convention(c) (PyTypeObjectPointer, Int) -> PyObjectPointer? +typealias destructor = @convention(c) (PyObjectPointer) -> Void +typealias freefunc = @convention(c) (UnsafeMutableRawPointer) -> Void +typealias iternextfunc = @convention(c) (PyObjectPointer) -> PyObjectPointer? +typealias newfunc = @convention(c) (PyTypeObjectPointer, PyObjectPointer?, PyObjectPointer?) -> PyObjectPointer? +typealias sendfunc = @convention(c) (PyObjectPointer, PyObjectPointer, PyObjectPointer) -> Int +typealias unaryfunc = @convention(c) (PyObjectPointer) -> PyObjectPointer? + +typealias PyCFunction = @convention(c) (PyObjectPointer, PyObjectPointer) -> PyObjectPointer? + +// This will be 3 for the lifetime of Python 3. See PEP-384. +let Py_AbiVersion: Int = 3 + +// From the C headers (we're not stackless). +let Py_TPFlagsDefault: UInt64 = 0 + +// Our type is dynamically allocated. +let Py_TPFLAGS_HEAPTYPE: UInt64 = (1 << 9) + +// The immortal value is the 32bit UInt.max value. +let Py_ImmortalRefCount: Int = Int(bitPattern: 0x00000000FFFFFFFF) + +let METH_NOARGS: Int32 = 0x0004 +let METH_O: Int32 = 0x0008 + +struct PyObject { + var ob_refcnt: Int + var ob_type: UnsafeMutablePointer? +} + +struct PyVarObject { + var ob_base: PyObject + var ob_size: Int +} + +//===----------------------------------------------------------------------===// +// PythonModule Types. +//===----------------------------------------------------------------------===// + +struct PyModuleDef_Base { + var ob_base: PyObject + var m_init: OpaquePointer? + var m_index: Int + var m_copy: OpaquePointer? +} + +struct PyModuleDef_Slot { + var slot: Int + var value: OpaquePointer +} + +struct PyModuleDef { + var m_base: PyModuleDef_Base + var m_name: UnsafePointer + var m_doc: UnsafePointer? + var m_size: Int + var m_methods: UnsafePointer? + var m_slots: UnsafePointer? + var m_traverse: OpaquePointer? + var m_clear: OpaquePointer? + var m_free: OpaquePointer? +} + +//===----------------------------------------------------------------------===// +// PythonType Types. +//===----------------------------------------------------------------------===// + +struct PyAsyncMethods { + var am_await: unaryfunc? + var am_aiter: unaryfunc? + var am_anext: unaryfunc? + var am_send: sendfunc? +} + +struct PyGetSetDef { + var name: UnsafePointer + var get: OpaquePointer? + var set: OpaquePointer? + var doc: UnsafePointer? + var closure: OpaquePointer? +} + +struct PyTypeObject { + var ob_base: PyVarObject + var tp_name: UnsafePointer + var tp_basicsize: Int + var tp_itemsize: Int + var tp_dealloc: destructor? + var tp_vectorcall_offset: Int + var tp_getattr: OpaquePointer? + var tp_setattr: OpaquePointer? + var tp_as_async: UnsafePointer? + var tp_repr: OpaquePointer? + var tp_as_number: OpaquePointer? + var tp_as_sequence: OpaquePointer? + var tp_as_mapping: OpaquePointer? + var tp_hash: OpaquePointer? + var tp_call: OpaquePointer? + var tp_str: OpaquePointer? + var tp_getattro: OpaquePointer? + var tp_setattro: OpaquePointer? + var tp_as_buffer: OpaquePointer? + var tp_flags: UInt64 + var tp_doc: UnsafePointer? + var tp_traverse: OpaquePointer? + var tp_clear: OpaquePointer? + var tp_richcompare: OpaquePointer? + var tp_weaklistoffset: Int + var tp_iter: OpaquePointer? + var tp_iternext: iternextfunc? + var tp_methods: UnsafePointer? + var tp_members: UnsafePointer>? + var tp_getset: UnsafePointer? + var tp_base: OpaquePointer? + var tp_dict: OpaquePointer? + var tp_descr_get: OpaquePointer? + var tp_descr_set: OpaquePointer? + var tp_dictoffset: Int + var tp_init: OpaquePointer? + var tp_alloc: allocfunc? + var tp_new: newfunc? + var tp_free: freefunc? + var tp_is_gc: OpaquePointer? + var tp_bases: OpaquePointer? + var tp_mro: OpaquePointer? + var tp_cache: OpaquePointer? + var tp_subclasses: OpaquePointer? + var tp_weaklist: OpaquePointer? + var tp_del: OpaquePointer? + var tp_version_tag: UInt + var tp_finalize: OpaquePointer? + var tp_vectorcall: OpaquePointer? +} + +//===----------------------------------------------------------------------===// +// PythonModule for injecting the `pythonkit` extension. +//===----------------------------------------------------------------------===// + +struct PythonModule : PythonConvertible { + private static let moduleName: StaticString = "pythonkit" + private static let moduleDoc: StaticString = "PythonKit Extension Module" + + // PythonConvertible conformance. + public var pythonObject: PythonObject + + private let moduleDef: PyModuleDef + + init() { + // Define module-level methods. + let methods: [(StaticString, PyCFunction, Int32)] = [ + ("get_test_awaitable", PythonModule.getTestAwaitable, METH_NOARGS), + ] + let methodDefs = Self.generateMethodDefs(from: methods) + + // Define the module. + moduleDef = PyModuleDef( + m_base: PyModuleDef_Base( + ob_base: PyObject( + ob_refcnt: Py_ImmortalRefCount, + ob_type: nil + ), + m_init: nil, + m_index: 0, + m_copy: nil + ), + m_name: UnsafeRawPointer(Self.moduleName.utf8Start).assumingMemoryBound(to: Int8.self), + m_doc: UnsafeRawPointer(Self.moduleDoc.utf8Start).assumingMemoryBound(to: Int8.self), + m_size: -1, + m_methods: methodDefs, + m_slots: nil, + m_traverse: nil, + m_clear: nil, + m_free: nil + ) + + let moduleDefinition: UnsafeMutablePointer = .allocate(capacity: 1) + moduleDefinition.pointee = moduleDef + + let module = PyModule_Create(moduleDefinition, Py_AbiVersion) + let moduleName = PyUnicode_InternFromString( + UnsafeRawPointer(Self.moduleName.utf8Start).assumingMemoryBound(to: Int8.self)) + let modules = PyImport_GetModuleDict() + let success = _PyImport_FixupExtensionObject(module, moduleName, moduleName, modules) + guard success == 0 else { + fatalError("Failed to fixup extension object.") + } + + pythonObject = PythonObject(consuming: module) + } + + private static func generateMethodDefs(from methods: [(StaticString, PyCFunction, Int32)]) -> UnsafeMutablePointer { + let methodDefs = UnsafeMutablePointer.allocate(capacity: methods.count + 1) + for (index, (name, fn, meth)) in methods.enumerated() { + methodDefs[index] = PyMethodDef( + ml_name: UnsafeRawPointer(name.utf8Start).assumingMemoryBound(to: Int8.self), + ml_meth: unsafeBitCast(fn, to: OpaquePointer.self), + ml_flags: meth, + ml_doc: nil) + } + // Sentinel value. + methodDefs[methods.count] = PyMethodDef( + ml_name: nil, ml_meth: nil, ml_flags: 0, ml_doc: nil) + return methodDefs + } +} diff --git a/Tests/PythonKitTests/PythonFunctionTests.swift b/Tests/PythonKitTests/PythonFunctionTests.swift index 7be1afa..f11ad57 100644 --- a/Tests/PythonKitTests/PythonFunctionTests.swift +++ b/Tests/PythonKitTests/PythonFunctionTests.swift @@ -7,24 +7,24 @@ class PythonFunctionTests: XCTestCase { let versionMinor = Python.versionInfo.minor return (versionMajor == 3 && versionMinor >= 1) || versionMajor > 3 } - + func testPythonFunction() { guard canUsePythonFunction else { return } - + let pythonAdd = PythonFunction { args in let lhs = args[0] let rhs = args[1] return lhs + rhs }.pythonObject - + let pythonSum = pythonAdd(2, 3) XCTAssertNotNil(Double(pythonSum)) XCTAssertEqual(pythonSum, 5) - + // Test function with keyword arguments - + // Since there is no alternative function signature, `args` and `kwargs` // can be used without manually stating their type. This differs from // the behavior when there are no keywords. @@ -36,76 +36,76 @@ class PythonFunctionTests: XCTestCase { XCTAssertEqual(kwargs[0].value, 2) XCTAssertEqual(kwargs[1].key, "x") XCTAssertEqual(kwargs[1].value, 3) - + let conditional = Bool(args[0])! let xIndex = kwargs.firstIndex(where: { $0.key == "x" })! let yIndex = kwargs.firstIndex(where: { $0.key == "y" })! - + return kwargs[conditional ? xIndex : yIndex].value }.pythonObject - + let pythonSelectOutput = pythonSelect(true, y: 2, x: 3) XCTAssertEqual(pythonSelectOutput, 3) } - + // From https://www.geeksforgeeks.org/create-classes-dynamically-in-python func testPythonClassConstruction() { guard canUsePythonFunction else { return } - + let constructor = PythonInstanceMethod { args in let `self` = args[0] `self`.constructor_arg = args[1] return Python.None } - + // Instead of calling `print`, use this to test what would be output. var printOutput: String? - + // Example of function using an alternative syntax for `args`. let displayMethod = PythonInstanceMethod { (args: [PythonObject]) in // let `self` = args[0] printOutput = String(args[1]) return Python.None } - + let classMethodOriginal = PythonInstanceMethod { args in // let cls = args[0] printOutput = String(args[1]) return Python.None } - + // Did not explicitly convert `constructor` or `displayMethod` to // PythonObject. This is intentional, as the `PythonClass` initializer // should take any `PythonConvertible` and not just `PythonObject`. let classMethod = Python.classmethod(classMethodOriginal.pythonObject) - + let Geeks = PythonClass("Geeks", members: [ // Constructor "__init__": constructor, - + // Data members "string_attribute": "Geeks 4 geeks!", "int_attribute": 1706256, - + // Member functions "func_arg": displayMethod, "class_func": classMethod, ]).pythonObject - + let obj = Geeks("constructor argument") XCTAssertEqual(obj.constructor_arg, "constructor argument") XCTAssertEqual(obj.string_attribute, "Geeks 4 geeks!") XCTAssertEqual(obj.int_attribute, 1706256) - + obj.func_arg("Geeks for Geeks") XCTAssertEqual(printOutput, "Geeks for Geeks") - + Geeks.class_func("Class Dynamically Created!") XCTAssertEqual(printOutput, "Class Dynamically Created!") } - + // Previously, there was a build error where passing a simple // `PythonClass.Members` literal made the literal's type ambiguous. It was // confused with `[String: PythonObject]`. The solution was adding a @@ -114,7 +114,7 @@ class PythonFunctionTests: XCTestCase { guard canUsePythonFunction else { return } - + let MyClass = PythonClass( "MyClass", superclasses: [Python.object], @@ -122,76 +122,76 @@ class PythonFunctionTests: XCTestCase { "memberName": "memberValue", ] ).pythonObject - + let memberValue = MyClass().memberName XCTAssertEqual(String(memberValue), "memberValue") } - + func testPythonClassInheritance() { guard canUsePythonFunction else { return } - + var helloOutput: String? var helloWorldOutput: String? - + // Declare subclasses of `Python.Exception` - + let HelloException = PythonClass( "HelloException", superclasses: [Python.Exception], members: [ "str_prefix": "HelloException-prefix ", - + "__init__": PythonInstanceMethod { args in let `self` = args[0] let message = "hello \(args[1])" helloOutput = String(message) - + // Conventional `super` syntax does not work; use this instead. Python.Exception.__init__(`self`, message) return Python.None }, - + // Example of function using the `self` convention instead of `args`. "__str__": PythonInstanceMethod { (`self`: PythonObject) in return `self`.str_prefix + Python.repr(`self`) } ] ).pythonObject - + let HelloWorldException = PythonClass( "HelloWorldException", superclasses: [HelloException], members: [ "str_prefix": "HelloWorldException-prefix ", - + "__init__": PythonInstanceMethod { args in let `self` = args[0] let message = "world \(args[1])" helloWorldOutput = String(message) - + `self`.int_param = args[2] - + // Conventional `super` syntax does not work; use this instead. HelloException.__init__(`self`, message) return Python.None }, - + // Example of function using the `self` convention instead of `args`. "custom_method": PythonInstanceMethod { (`self`: PythonObject) in return `self`.int_param } ] ).pythonObject - + // Test that inheritance works as expected - + let error1 = HelloException("test 1") XCTAssertEqual(helloOutput, "hello test 1") XCTAssertEqual(Python.str(error1), "HelloException-prefix HelloException('hello test 1')") XCTAssertEqual(Python.repr(error1), "HelloException('hello test 1')") - + let error2 = HelloWorldException("test 1", 123) XCTAssertEqual(helloOutput, "hello world test 1") XCTAssertEqual(helloWorldOutput, "world test 1") @@ -199,15 +199,15 @@ class PythonFunctionTests: XCTestCase { XCTAssertEqual(Python.repr(error2), "HelloWorldException('hello world test 1')") XCTAssertEqual(error2.custom_method(), 123) XCTAssertNotEqual(error2.custom_method(), "123") - + // Test that subclasses behave like Python exceptions - + // Example of function with no named parameters, which can be stated // ergonomically using an underscore. The ignored input is a [PythonObject]. - let testFunction = PythonFunction { _ in + let _ = PythonFunction { _ in throw HelloWorldException("EXAMPLE ERROR MESSAGE", 2) }.pythonObject - + /* do { try testFunction.throwing.dynamicallyCall(withArguments: []) @@ -217,7 +217,7 @@ class PythonFunctionTests: XCTestCase { XCTFail("A string could not be created from a HelloWorldException.") return } - + XCTAssertTrue(description.contains("EXAMPLE ERROR MESSAGE")) XCTAssertTrue(description.contains("HelloWorldException")) } catch { @@ -225,19 +225,19 @@ class PythonFunctionTests: XCTestCase { } */ } - + // Tests the ability to dynamically construct an argument list with keywords // and instantiate a `PythonInstanceMethod` with keywords. func testPythonClassInheritanceWithKeywords() { guard canUsePythonFunction else { return } - + func getValue(key: String, kwargs: [(String, PythonObject)]) -> PythonObject { let index = kwargs.firstIndex(where: { $0.0 == key })! return kwargs[index].1 } - + // Base class has the following arguments: // __init__(): // - 1 unnamed argument @@ -247,7 +247,7 @@ class PythonFunctionTests: XCTestCase { // test_method(): // - param1 // - param2 - + let BaseClass = PythonClass( "BaseClass", superclasses: [], @@ -259,7 +259,7 @@ class PythonFunctionTests: XCTestCase { `self`.param2 = getValue(key: "param2", kwargs: kwargs) return Python.None }, - + "test_method": PythonInstanceMethod { args, kwargs in let `self` = args[0] `self`.param1 += getValue(key: "param1", kwargs: kwargs) @@ -268,7 +268,7 @@ class PythonFunctionTests: XCTestCase { } ] ).pythonObject - + // Derived class accepts the following arguments: // __init__(): // - param2 @@ -278,7 +278,7 @@ class PythonFunctionTests: XCTestCase { // - param1 // - param2 // - param3 - + let DerivedClass = PythonClass( "DerivedClass", superclasses: [], @@ -286,7 +286,7 @@ class PythonFunctionTests: XCTestCase { "__init__": PythonInstanceMethod { args, kwargs in let `self` = args[0] `self`.param3 = getValue(key: "param3", kwargs: kwargs) - + // Lists the arguments in an order different than they are // specified (self, param2, param3, param1, arg1). The // correct order is (self, arg1, param1, param2, param3). @@ -296,44 +296,44 @@ class PythonFunctionTests: XCTestCase { ("param1", 1), ("", 0) ] - + BaseClass.__init__.dynamicallyCall( withKeywordArguments: newKeywordArguments) return Python.None }, - + "test_method": PythonInstanceMethod { args, kwargs in let `self` = args[0] `self`.param3 += getValue(key: "param3", kwargs: kwargs) - + BaseClass.test_method.dynamicallyCall( withKeywordArguments: args.map { ("", $0) } + kwargs) return Python.None } ] ).pythonObject - + let derivedInstance = DerivedClass(param2: 2, param3: 3) XCTAssertEqual(derivedInstance.arg1, 0) XCTAssertEqual(derivedInstance.param1, 1) XCTAssertEqual(derivedInstance.param2, 2) XCTAssertEqual(derivedInstance.checking.param3, 3) - + derivedInstance.test_method(param1: 1, param2: 2, param3: 3) XCTAssertEqual(derivedInstance.arg1, 0) XCTAssertEqual(derivedInstance.param1, 2) XCTAssertEqual(derivedInstance.param2, 4) XCTAssertEqual(derivedInstance.checking.param3, 6) - + // Validate that subclassing and instantiating the derived class does // not affect behavior of the parent class. - + let baseInstance = BaseClass(0, param1: 10, param2: 20) XCTAssertEqual(baseInstance.arg1, 0) XCTAssertEqual(baseInstance.param1, 10) XCTAssertEqual(baseInstance.param2, 20) XCTAssertEqual(baseInstance.checking.param3, nil) - + baseInstance.test_method(param1: 10, param2: 20) XCTAssertEqual(baseInstance.arg1, 0) XCTAssertEqual(baseInstance.param1, 20) diff --git a/Tests/PythonKitTests/PythonModuleTests.swift b/Tests/PythonKitTests/PythonModuleTests.swift new file mode 100644 index 0000000..50adbda --- /dev/null +++ b/Tests/PythonKitTests/PythonModuleTests.swift @@ -0,0 +1,33 @@ +import XCTest +@testable import PythonKit + +class PythonModuleTests: XCTestCase { + func testPythonModule() { + let pythonKit = Python.import("pythonkit") + XCTAssertNotNil(pythonKit) + } + + func testCanAwait() throws { + _ = Python + + PythonModule.testAwaitableFunction = + PythonFunction(name: "test_awaitable") { (_, _) async throws -> PythonConvertible in + let result = 42 + return result + } + + // TODO: Find a way to assert the result in Swift. + PyRun_SimpleString(""" + import asyncio + import pythonkit + + async def main(): + awaitable = pythonkit.get_test_awaitable() + result = await awaitable() + print(f"Python: result == {result}") + assert result == 42 + + asyncio.run(main()) + """) + } +}