no-copy conversion

Author: Christopher Doris

src/concrete/numpy.jl

@@ -39,7 +39,7 @@ function init_numpy()
         rule = pyconvert_rule_numpysimplevalue{T, false}()
         saferule = pyconvert_rule_numpysimplevalue{T, true}()
 
-        pyconvert_add_rule(name, T, saferule, PYCONVERT_PRIORITY_CANONICAL)
+        pyconvert_add_rule(name, T, saferule, PYCONVERT_PRIORITY_ARRAY)
         isuint && pyconvert_add_rule(name, UInt, sizeof(T) ≤ sizeof(UInt) ? saferule : rule)
         isuint && pyconvert_add_rule(name, Int, sizeof(T) < sizeof(Int) ? saferule : rule)
         isint && !isuint && pyconvert_add_rule(name, Int, sizeof(T) ≤ sizeof(Int) ? saferule : rule)

src/convert.jl

@@ -479,6 +479,10 @@ function init_pyconvert()
     pyconvert_add_rule("<arrayinterface>", AbstractArray, pyconvert_rule_array, priority)
     pyconvert_add_rule("<array>", AbstractArray, pyconvert_rule_array, priority)
     pyconvert_add_rule("<buffer>", AbstractArray, pyconvert_rule_array, priority)
+    # more sequence types
+    pyconvert_add_rule("pandas.core.series:Series", PyList, pyconvert_rule_sequence)
+    pyconvert_add_rule("pandas.core.indexes.base:Index", PyList, pyconvert_rule_sequence)
+    pyconvert_add_rule("pandas.core.arrays.base:ExtensionArray", PyList, pyconvert_rule_sequence)
 
     priority = PYCONVERT_PRIORITY_FALLBACK
     pyconvert_add_rule("builtins:object", Py, pyconvert_rule_object, priority)

src/pywrap/PyArray.jl

@@ -26,7 +26,7 @@ struct PyArray{T,N,M,L,R} <: AbstractArray{T,N}
     py::Py                  # underlying python object
     handle::Py              # the data in this array is valid as long as this handle is alive
     function PyArray{T,N,M,L,R}(::Val{:new}, ptr::Ptr{R}, size::NTuple{N,Int}, strides::NTuple{N,Int}, py::Py, handle::Py) where {T,N,M,L,R}
-        T isa DataType || error("T must be a DataType")
+        T isa Type || error("T must be a Type")
         N isa Int || error("N must be an Int")
         M isa Bool || error("M must be a Bool")
         L isa Bool || error("L must be a Bool")

src/pywrap/PyPandasDataFrame.jl

@@ -55,11 +55,11 @@ function _columns(df, columnnames, columntypes)
     pycolumns = Py[]
     if df.indexname !== nothing
         push!(colnames, df.indexname)
-        push!(pycolumns, df.py.index)
+        push!(pycolumns, df.py.index.values)
     end
     for pycolname in df.py.columns
         colname = columnnames(pycolname)::Symbol
-        pycolumn = df.py[pycolname]
+        pycolumn = df.py[pycolname].values
         push!(colnames, colname)
         push!(pycolumns, pycolumn)
     end
@@ -78,20 +78,28 @@ function _columns(df, columnnames, columntypes)
     for (colname, pycolumn) in zip(colnames, pycolumns)
         coltype = columntypes(colname)::Union{Nothing,Type}
         if coltype !== nothing
-            column = pyconvert_and_del(AbstractVector{coltype}, pycolumn)
+            column = pyconvert(AbstractVector{coltype}, pycolumn)
         else
-            column = pyconvert_and_del(AbstractVector, pycolumn)
-            # narrow the type
-            column = identity.(column)
-            # convert any Py to something more useful
-            if Py <: eltype(column)
-                column = [x isa Py ? pyconvert(Any, x) : x for x in column]
-            end
-            # convert NaN to missing
-            if eltype(column) != Float64 && Float64 <: eltype(column)
-                column = [x isa Float64 && isnan(x) ? missing : x for x in column]
+            column = pyconvert(AbstractVector, pycolumn)
+            @show colname typeof(column)
+            if eltype(column) == Py
+                # guess a column type based on the types of the elements
+                ts = pybuiltins.set(pybuiltins.map(pybuiltins.type, pycolumn))
+                Ts = Type[pyconvert_preferred_type(t) for t in ts]
+                T = isempty(Ts) ? Any : reduce(promote_type, Ts)
+                column = pyconvert(AbstractVector{T}, pycolumn)
+                @show ts Ts T typeof(column)
+                # if all items are either NaN or not Float64, convert NaN to missing
+                if T != Float64 && Float64 in Ts && !any(x isa Float64 && !isnan(x) for x in column)
+                    Ts = Type[T for T in Ts if T != Float64]
+                    push!(Ts, Missing)
+                    T = reduce(promote_type, Ts)
+                    column = pyconvert(AbstractVector{T}, pycolumn)
+                    @show Ts T typeof(column)
+                end
             end
         end
+        pydel!(pycolumn)
         push!(columns, column)
         push!(coltypes, eltype(column))
     end