Merge branch 'main' into pandas

Christopher Doris · Christopher Doris · commit 00cf9d8d2c20 · 2022-04-21T14:51:53.000+01:00
diff --git a/CITATION.bib b/CITATION.bib
@@ -0,0 +1,6 @@
+@misc{PythonCall.jl,
+  author = {Rowley, Christopher},
+  title = {PythonCall.jl: Python and Julia in harmony},
+  year  = {2022},
+  url   = {https://github.com/cjdoris/PythonCall.jl},
+}
diff --git a/Project.toml b/Project.toml
@@ -21,7 +21,7 @@ MacroTools = "0.5"
 Requires = "1"
 Tables = "1"
 UnsafePointers = "1"
-julia = "1"
+julia = "1.6"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
diff --git a/docs/src/pycall.md b/docs/src/pycall.md
@@ -6,8 +6,11 @@ On this page, we give some tips for migrating between the two modules and a comp
 
 ## Tips
 
-- You can use both PyCall and PythonCall in the same Julia session (this might be platform dependent).
-- To force PythonCall to use the same Python interpreter as PyCall, set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`.
+- You can use both PyCall and PythonCall in the same Julia session. This is platform-dependent:
+  - On Unix (Linux, Mac, etc.) the Python interpreter used by PythonCall and PyCall must be the same (see below).
+  - On Windows, it appears to be possible for PythonCall and PyCall to use different interpreters.
+- To force PythonCall to use the same Python interpreter as PyCall, set the environment variable `JULIA_PYTHONCALL_EXE` to `"@PyCall"`. Note that this will opt out of automatic dependency management using CondaPkg.
+- Alternatively, to force PyCall to use the same interpreter as PythonCall, set the environment variable `PYTHON` to `PythonCall.C.CTX.exe_path` and then `Pkg.build("PyCall")`. You will need to do this each time you change project, because PythonCall by default uses a different Python for each project.
 
 ## Comparison
 
diff --git a/docs/src/pythoncall-reference.md b/docs/src/pythoncall-reference.md
@@ -51,6 +51,7 @@ pyexec
 pygetattr
 pygetitem
 pyhasattr
+pyhasitem
 pyhash
 pyhelp
 pyimport
diff --git a/docs/src/releasenotes.md b/docs/src/releasenotes.md
@@ -1,5 +1,11 @@
 # Release Notes
 
+## Unreleased
+* Adds `pyhasitem` and 3-arg `pygetitem`.
+* Extends `Base.get`, `Base.get!`, `Base.haskey` and 2-arg `Base.hash` for `Py`.
+* `PyArray` can now have any element type when the underlying array is of Python objects.
+* Bug fixes.
+
 ## v0.8.0 (2022-03-17)
 * **Breaking:** Removes `pymethod` and `pyclass`. In the future, `pyclass` may become sugar
   for `types.new_class` (namely you can specify a metaclass).
diff --git a/src/Py.jl b/src/Py.jl
@@ -313,6 +313,9 @@ Base.showable(mime::MIME, o::Py) = pyshowable(mime, o)
 Base.getproperty(x::Py, k::Symbol) = pygetattr(x, string(k))
 Base.getproperty(x::Py, k::String) = pygetattr(x, k)
 
+Base.hasproperty(x::Py, k::Symbol) = pyhasattr(x, string(k))
+Base.hasproperty(x::Py, k::String) = pyhasattr(x, k)
+
 Base.setproperty!(x::Py, k::Symbol, v) = pysetattr(x, string(k), v)
 Base.setproperty!(x::Py, k::String, v) = pysetattr(x, k, v)
 
@@ -350,6 +353,25 @@ Base.setindex!(x::Py, v, i...) = (pysetitem(x, i, v); x)
 
 Base.delete!(x::Py, i) = (pydelitem(x, i); x)
 
+Base.haskey(x::Py, i) = pyhasitem(x, i)
+
+Base.get(x::Py, i, d) = pygetitem(x, i, d)
+
+function Base.get(f::Base.Callable, x::Py, i)
+    v = pygetitem(x, i, nothing)
+    v === nothing ? f() : v
+end
+
+Base.get!(x::Py, i, d) = get(x, i) do
+    pysetitem(x, i, d)
+    pygetitem(x, i)
+end
+
+Base.get!(f::Base.Callable, x::Py, i) = get(x, i) do
+    pysetitem(x, i, f())
+    pygetitem(x, i)
+end
+
 Base.eltype(::Type{Py}) = Py
 
 Base.IteratorSize(::Type{Py}) = Base.SizeUnknown()
@@ -366,7 +388,7 @@ end
 
 Base.in(v, x::Py) = pycontains(x, v)
 
-Base.hash(x::Py) = reinterpret(UInt, Int(pyhash(x)))
+Base.hash(x::Py, h::UInt) = reinterpret(UInt, Int(pyhash(x))) - 3h
 
 (f::Py)(args...; kwargs...) = pycall(f, args...; kwargs...)
 
@@ -464,7 +486,7 @@ Base.powermod(x::Number, y::Py, z::Number) = pypow(x, y, z)
 Base.powermod(x::Py, y::Number, z::Number) = pypow(x, y, z)
 
 # documentation
-function Base.Docs.getdoc(x::Py)
+function Base.Docs.getdoc(x::Py, @nospecialize(sig))
     parts = []
     inspect = pyimport("inspect")
     # head line
@@ -498,4 +520,5 @@ function Base.Docs.getdoc(x::Py)
     end
     return Markdown.MD(parts)
 end
+Base.Docs.doc(x::Py, sig::Type=Union{}) = Base.Docs.getdoc(x, sig)
 Base.Docs.Binding(x::Py, k::Symbol) = getproperty(x, k)
diff --git a/src/abstract/object.jl b/src/abstract/object.jl
@@ -30,8 +30,24 @@ export pyascii
     pyhasattr(x, k)
 
 Equivalent to `hasattr(x, k)` in Python.
+
+Tests if `getattr(x, k)` raises an `AttributeError`.
 """
-pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(getptr(x_), getptr(k_))) == 1
+# pyhasattr(x, k) = errcheck(@autopy x k C.PyObject_HasAttr(getptr(x_), getptr(k_))) == 1
+function pyhasattr(x, k)
+    ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_))
+    if iserrset(ptr)
+        if errmatches(pybuiltins.AttributeError)
+            errclear()
+            return false
+        else
+            pythrow()
+        end
+    else
+        decref(ptr)
+        return true
+    end
+end
 export pyhasattr
 
 """
@@ -45,8 +61,12 @@ pygetattr(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetAttr(getptr(x_), getp
 function pygetattr(x, k, d)
     ptr = @autopy x k C.PyObject_GetAttr(getptr(x_), getptr(k_))
     if iserrset(ptr)
-        errclear()
-        return d
+        if errmatches(pybuiltins.AttributeError)
+            errclear()
+            return d
+        else
+            pythrow()
+        end
     else
         return pynew(ptr)
     end
@@ -118,11 +138,48 @@ pylen(x) = errcheck(@autopy x C.PyObject_Length(getptr(x_)))
 export pylen
 
 """
-    pygetitem(x, k)
+    pyhasitem(x, k)
+
+Test if `pygetitem(x, k)` raises a `KeyError` or `AttributeError`.
+"""
+function pyhasitem(x, k)
+    ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_))
+    if iserrset(ptr)
+        if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError)
+            errclear()
+            return false
+        else
+            pythrow()
+        end
+    else
+        decref(ptr)
+        return true
+    end
+end
+export pyhasitem
+
+"""
+    pygetitem(x, k, [d])
+
+Equivalent `x[k]` in Python.
 
-Equivalent to `getitem(x, k)` or `x[k]` in Python.
+If `d` is specified, it is returned if the item does not exist (i.e. if `x[k]` raises a
+`KeyError` or `IndexError`).
 """
 pygetitem(x, k) = pynew(errcheck(@autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_))))
+function pygetitem(x, k, d)
+    ptr = @autopy x k C.PyObject_GetItem(getptr(x_), getptr(k_))
+    if iserrset(ptr)
+        if errmatches(pybuiltins.KeyError) || errmatches(pybuiltins.IndexError)
+            errclear()
+            return d
+        else
+            pythrow()
+        end
+    else
+        return pynew(ptr)
+    end
+end
 export pygetitem
 
 """
diff --git a/src/concrete/range.jl b/src/concrete/range.jl
@@ -23,8 +23,8 @@ function pyconvert_rule_range(::Type{R}, x::Py, ::Type{StepRange{T0,S0}}=Utils._
 end
 
 function pyconvert_rule_range(::Type{R}, x::Py, ::Type{UnitRange{T0}}=Utils._type_lb(R), ::Type{UnitRange{T1}}=Utils._type_ub(R)) where {R<:UnitRange,T0,T1}
-    b = @pyconvert(Utils._typeintersect(Integer, S1), x.step)
-    b == 1 && return pyconvert_unconverted()
+    b = @pyconvert(Int, x.step)
+    b == 1 || return pyconvert_unconverted()
     a = @pyconvert(Utils._typeintersect(Integer, T1), x.start)
     c = @pyconvert(Utils._typeintersect(Integer, T1), x.stop)
     a′, c′ = promote(a, c - oftype(c, 1))
diff --git a/src/convert.jl b/src/convert.jl
@@ -444,6 +444,7 @@ function init_pyconvert()
     pyconvert_add_rule("io:IOBase", PyIO, pyconvert_rule_io, priority)
     pyconvert_add_rule("_io:_IOBase", PyIO, pyconvert_rule_io, priority)
     pyconvert_add_rule("pandas.core.frame:DataFrame", PyPandasDataFrame, pyconvert_rule_pandasdataframe, priority)
+    pyconvert_add_rule("pandas.core.arrays.base:ExtensionArray", PyList, pyconvert_rule_sequence, priority)
     pyconvert_add_rule("builtins:tuple", Tuple, pyconvert_rule_iterable, priority)
     pyconvert_add_rule("datetime:datetime", DateTime, pyconvert_rule_datetime, priority)
     pyconvert_add_rule("datetime:date", Date, pyconvert_rule_date, priority)
diff --git a/src/jlwrap/any.jl b/src/jlwrap/any.jl
@@ -134,7 +134,11 @@ end
 
 function pyjlany_help(self, mime_::Py)
     mime = pyconvertarg(Union{Nothing,String}, mime_, "mime")
-    x = Utils.ExtraNewline(Docs.doc(self))
+    doc = Docs.getdoc(self)
+    if doc === nothing
+        doc = Docs.doc(self)
+    end
+    x = Utils.ExtraNewline(doc)
     if mime === nothing
         display(x)
     else
diff --git a/src/jlwrap/array.jl b/src/jlwrap/array.jl
@@ -314,7 +314,7 @@ function init_jlwrap_array()
         @property
         def __array_interface__(self):
             return self._jl_callmethod($(pyjl_methodnum(pyjlarray_array_interface)))
-        def __array__(self):
+        def __array__(self, dtype=None):
             # convert to an array-like object
             arr = self
             if not (hasattr(arr, "__array_interface__") or hasattr(arr, "__array_struct__")):
@@ -326,7 +326,7 @@ function init_jlwrap_array()
             # convert to a numpy array if numpy is available
             try:
                 import numpy
-                arr = numpy.array(arr)
+                arr = numpy.array(arr, dtype=dtype)
             except ImportError:
                 pass
             return arr
diff --git a/src/jlwrap/dict.jl b/src/jlwrap/dict.jl
@@ -97,6 +97,8 @@ function init_jlwrap_dict()
                 self._jl_callmethod($(pyjl_methodnum(pyjldict_update)), items)
             if kwargs:
                 self.update(kwargs)
+        def copy(self):
+            return self._jl_callmethod($(pyjl_methodnum(Py ∘ copy)))
     import collections.abc
     collections.abc.MutableMapping.register(DictValue)
     del collections
diff --git a/src/pywrap/PyArray.jl b/src/pywrap/PyArray.jl
@@ -438,6 +438,7 @@ pyarray_load(::Type{T}, p::Ptr{UnsafePyObject}) where {T} = begin
 end
 
 pyarray_store!(p::Ptr{R}, x::R) where {R} = unsafe_store!(p, x)
+pyarray_store!(p::Ptr{UnsafePyObject}, x::UnsafePyObject) = unsafe_store!(p, x)
 pyarray_store!(p::Ptr{UnsafePyObject}, x) = @autopy x begin
     decref(unsafe_load(p).ptr)
     unsafe_store!(p, UnsafePyObject(incref(getptr(x_))))
diff --git a/test/abstract.jl b/test/abstract.jl
@@ -212,6 +212,7 @@ end
 @testset "iter" begin
     @test_throws PyException pyiter(pybuiltins.None)
     @test_throws PyException pyiter(pybuiltins.True)
+    # unsafe_pynext
     it = pyiter(pyrange(2))
     x = PythonCall.unsafe_pynext(it)
     @test !PythonCall.pyisnull(x)
@@ -221,6 +222,13 @@ end
     @test pyeq(Bool, x, 1)
     x = PythonCall.unsafe_pynext(it)
     @test PythonCall.pyisnull(x)
+    # pynext
+    it = pyiter(pyrange(2))
+    x = pynext(it)
+    @test pyeq(Bool, x, 0)
+    x = pynext(it)
+    @test pyeq(Bool, x, 1)
+    @test_throws PyException pynext(it)
 end
 
 @testset "number" begin
@@ -355,3 +363,35 @@ end
     end
     # TODO: in-place operators
 end
+
+@testset "builtins" begin
+    @testset "pyprint" begin
+        buf = pyimport("io").StringIO()
+        ans = pyprint("hello", 12, file=buf)
+        @test ans === nothing
+        buf.seek(0)
+        @test pyeq(Bool, buf.read().strip(), "hello 12")
+    end
+    @testset "pyall" begin
+        for val in [[true, true], [true, false], [false, false]]
+            @test pyall(pylist(val)) === all(val)
+        end
+    end
+    @testset "pyany" begin
+        for val in [[true, true], [true, false], [false, false]]
+            @test pyany(pylist(val)) === any(val)
+        end
+    end
+    @testset "pycallable" begin
+        @test pycallable(pybuiltins.str) === true
+        @test pycallable(pybuiltins.any) === true
+        @test pycallable(pybuiltins.None) === false
+        @test pycallable(pybuiltins.True) === false
+        @test pycallable(pybuiltins.object) === true
+    end
+    @testset "pycompile" begin
+        ans = pycompile("3+4", "foo.py", "eval")
+        @test pyeq(Bool, ans.co_filename, "foo.py")
+        @test pyeq(Bool, pybuiltins.eval(ans, pydict()), 7)
+    end
+end
diff --git a/test/concrete.jl b/test/concrete.jl
@@ -259,7 +259,28 @@ end
 end
 
 @testset "datetime" begin
-    # TODO
+    dt = pyimport("datetime")
+    x1 = pydate(2001, 2, 3)
+    @test pyisinstance(x1, dt.date)
+    @test pyeq(Bool, x1, dt.date(2001, 2, 3))
+    x2 = pydate(Date(2002, 3, 4))
+    @test pyisinstance(x2, dt.date)
+    @test pyeq(Bool, x2, dt.date(2002, 3, 4))
+    x3 = pytime(12, 3, 4, 5)
+    @test pyisinstance(x3, dt.time)
+    @test pyeq(Bool, x3, dt.time(12, 3, 4, 5))
+    x4 = pytime(Time(23, 4, 5, 0, 6))
+    @test pyisinstance(x4, dt.time)
+    @test pyeq(Bool, x4, dt.time(23, 4, 5, 6))
+    x5 = pydatetime(2001, 2, 3, 4, 5, 6, 7)
+    @test pyisinstance(x5, dt.datetime)
+    @test pyeq(Bool, x5, dt.datetime(2001, 2, 3, 4, 5, 6, 7))
+    x6 = pydatetime(Date(2007, 8, 9))
+    @test pyisinstance(x6, dt.datetime)
+    @test pyeq(Bool, x6, dt.datetime(2007, 8, 9))
+    x7 = pydatetime(DateTime(2001, 2, 3, 4, 5, 6, 7))
+    @test pyisinstance(x7, dt.datetime)
+    @test pyeq(Bool, x7, dt.datetime(2001, 2, 3, 4, 5, 6, 7000))
 end
 
 @testset "code" begin
diff --git a/test/convert.jl b/test/convert.jl
