From 09f93479078bb87796e4461d3c6c0dada206741e Mon Sep 17 00:00:00 2001 From: "Stefan H. Muller" Date: Tue, 6 Aug 2013 19:40:42 +0200 Subject: [PATCH 01/10] Increase compatibility with numpypy and matplotlib - numpy/lib/utils.py: get_include() now calls numpypy.get_include(). - numpy/core/fromnumeric.py: Add implementations for np.take() and np.ptp(). - numpy/distutils/misc_utip.py: Add comment what was done to deactivate building of external modules. - numpy/f2py/rules.py: Replace array->data calls in wrapper generator by PyArray_DATA(array). --- numpy/core/fromnumeric.py | 48 +++++++++++++++++++++++++++--------- numpy/distutils/misc_util.py | 1 + numpy/f2py/rules.py | 2 +- numpy/lib/utils.py | 4 +++ 4 files changed, 43 insertions(+), 12 deletions(-) diff --git a/numpy/core/fromnumeric.py b/numpy/core/fromnumeric.py index d73f1313c6ee..2090b65e62cd 100644 --- a/numpy/core/fromnumeric.py +++ b/numpy/core/fromnumeric.py @@ -97,11 +97,26 @@ def take(a, indices, axis=None, out=None, mode='raise'): array([4, 3, 6]) """ - try: - take = a.take - except AttributeError: - return _wrapit(a, 'take', indices, axis, out, mode) - return take(indices, axis, out, mode) + # XXX: numpypy doesn't support ndarray.take() + assert mode == 'raise' + if axis is None: + res = a.ravel()[indices] + else: + if axis < 0: axis += len(a.shape) + s0, s1 = a.shape[:axis], a.shape[axis+1:] + l0 = prod(s0) if s0 else 1 + l1 = prod(s1) if s1 else 1 + res = a.reshape((l0, -1, l1))[:,indices,:].reshape(s0 + (-1,) + s1) + if out is not None: + out[:] = res + return out + else: + return res + #try: + # take = a.take + #except AttributeError: + # return _wrapit(a, 'take', indices, axis, out, mode) + #return take(indices, axis, out, mode) # not deprecated --- copy if necessary, view otherwise @@ -1837,11 +1852,18 @@ def ptp(a, axis=None, out=None): array([1, 1]) """ - try: - ptp = a.ptp - except AttributeError: - return _wrapit(a, 'ptp', axis, out) - return ptp(axis, out) + #XXX: numpypy does not have ndarray.ptp() + res = amax(a, axis) - amin(a, axis) + if out is not None: + out[:] = res + return out + else: + return res + #try: + # ptp = a.ptp + #except AttributeError: + # return _wrapit(a, 'ptp', axis, out) + #return ptp(axis, out) def amax(a, axis=None, out=None, keepdims=False): @@ -2395,11 +2417,15 @@ def round_(a, decimals=0, out=None): around : equivalent function """ + # XXX: numpypy doesn't support 'out' in ndarray.round() try: round = a.round except AttributeError: return _wrapit(a, 'round', decimals, out) - return round(decimals, out) + if out is None: + return round(decimals) + else: + return round(decimals, out) def mean(a, axis=None, dtype=None, out=None, keepdims=False): diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index c9b984d475f4..77a51828ff9c 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -1487,6 +1487,7 @@ def add_extension(self,name,sources,**kw): from numpy.distutils.core import Extension ext = Extension(**ext_args) + #XXX: This was commented out to prevent the building of extension modules #self.ext_modules.append(ext) dist = self.get_distribution() diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index 83f5811e5956..82dd1173c851 100644 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -1046,7 +1046,7 @@ \t\tif (!PyErr_Occurred()) \t\t\tPyErr_SetString(#modulename#_error,\"failed in converting #nth# `#varname#\' of #pyname# to C/Fortran array\" ); \t} else { -\t\t#varname# = (#ctype# *)(capi_#varname#_tmp->data); +\t\t#varname# = (#ctype# *)(PyArray_DATA(capi_#varname#_tmp)); """, {hasinitvalue:[ {isintent_nothide:'\tif (#varname#_capi == Py_None) {'}, diff --git a/numpy/lib/utils.py b/numpy/lib/utils.py index 227e64070a84..ea9bd4911245 100644 --- a/numpy/lib/utils.py +++ b/numpy/lib/utils.py @@ -29,6 +29,7 @@ def get_include(): include_dirs=[np.get_include()]) ... + """ """ import numpy if numpy.show_config is None: @@ -39,6 +40,9 @@ def get_include(): import numpy.core as core d = os.path.join(os.path.dirname(core.__file__), 'include') return d + """ + import numpypy + return numpypy.get_include() def get_numarray_include(type=None): """ From b579493fcc1ae5118f82be54d0d3328f0eb4045a Mon Sep 17 00:00:00 2001 From: "Stefan H. Muller" Date: Tue, 6 Aug 2013 22:26:23 +0200 Subject: [PATCH 02/10] Start making f2py PyPy compatible - numpy/f2py/f90mod_rules.py: Replace "np.intp().itemsize" by "np.dtype('intp').itemsize", since the latter is supported by PyPy. - numpy/f2py/src/fortranobject.c: check_and_fix_dimensions(): Refactor using PyArray_NDIM() and PyArray_DIM() instead of direct field access. Tested by recompiling and testing CPython numpy. Works. --- numpy/f2py/f90mod_rules.py | 2 +- numpy/f2py/src/fortranobject.c | 39 +++++++++++++++++----------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index e4a4b0e96fc7..c5c75729f351 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -56,7 +56,7 @@ def findf90modules(m): deallocate(d) end if end if - if ((.not.allocated(d)).and.(s(1).ge.1)) then""" % np.intp().itemsize + if ((.not.allocated(d)).and.(s(1).ge.1)) then""" % np.dtype('intp').itemsize fgetdims2="""\ end if diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index f5e2c62445af..164b4c0354af 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -778,20 +778,21 @@ int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *d the dimensions from arr. It also checks that non-blank dims will match with the corresponding values in arr dimensions. */ - const npy_intp arr_size = (arr->nd)?PyArray_Size((PyObject *)arr):1; + const int nd = PyArray_NDIM(arr); + const npy_intp arr_size = (nd)?PyArray_Size((PyObject *)arr):1; #ifdef DEBUG_COPY_ND_ARRAY dump_attrs(arr); printf("check_and_fix_dimensions:init: dims="); dump_dims(rank,dims); #endif - if (rank > arr->nd) { /* [1,2] -> [[1],[2]]; 1 -> [[1]] */ + if (rank > nd) { /* [1,2] -> [[1],[2]]; 1 -> [[1]] */ npy_intp new_size = 1; int free_axe = -1; int i; npy_intp d; /* Fill dims where -1 or 0; check dimensions; calc new_size; */ - for(i=0;ind;++i) { - d = arr->dimensions[i]; + for(i=0;i= 0) { if (d>1 && dims[i]!=d) { fprintf(stderr,"%d-th dimension must be fixed to %" NPY_INTP_FMT @@ -805,7 +806,7 @@ int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *d } new_size *= dims[i]; } - for(i=arr->nd;i1) { fprintf(stderr,"%d-th dimension must be %" NPY_INTP_FMT " but got 0 (not defined).\n", @@ -825,12 +826,12 @@ int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *d " indices)\n", new_size,arr_size); return 1; } - } else if (rank==arr->nd) { + } else if (rank==nd) { npy_intp new_size = 1; int i; npy_intp d; for (i=0; idimensions[i]; + d = PyArray_DIM(arr,i); if (dims[i]>=0) { if (d > 1 && d!=dims[i]) { fprintf(stderr,"%d-th dimension must be fixed to %" NPY_INTP_FMT @@ -852,19 +853,19 @@ int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *d npy_intp d; int effrank; npy_intp size; - for (i=0,effrank=0;ind;++i) - if (arr->dimensions[i]>1) ++effrank; + for (i=0,effrank=0;i1) ++effrank; if (dims[rank-1]>=0) if (effrank>rank) { fprintf(stderr,"too many axes: %d (effrank=%d), expected rank=%d\n", - arr->nd,effrank,rank); + nd,effrank,rank); return 1; } for (i=0,j=0;ind && arr->dimensions[j]<2) ++j; - if (j>=arr->nd) d = 1; - else d = arr->dimensions[j++]; + while (j=nd) d = 1; + else d = PyArray_DIM(arr,j++); if (dims[i]>=0) { if (d>1 && d!=dims[i]) { fprintf(stderr,"%d-th dimension must be fixed to %" NPY_INTP_FMT @@ -877,20 +878,20 @@ int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *d dims[i] = d; } - for (i=rank;ind;++i) { /* [[1,2],[3,4]] -> [1,2,3,4] */ - while (jnd && arr->dimensions[j]<2) ++j; - if (j>=arr->nd) d = 1; - else d = arr->dimensions[j++]; + for (i=rank;i [1,2,3,4] */ + while (j=nd) d = 1; + else d = PyArray_DIM(arr,j++); dims[rank-1] *= d; } for (i=0,size=1;ind); + size,arr_size,rank,effrank,nd); for (i=0;ind;++i) fprintf(stderr," %" NPY_INTP_FMT,arr->dimensions[i]); + for (i=0;i Date: Wed, 7 Aug 2013 14:05:33 +0200 Subject: [PATCH 03/10] More refactoring of fortranobject.* for PyPy - const PyArrayObject -> ArrayObject - Use PyArray_ISCONTIGUOUS(), PyArray_FLAGS() - Replace PyArray_MultiplyList() by direct loop. - Deactivate swap_arrays() for now. - PyArray_Size((PyObject*)arr) -> PyArray_Size(arr). --- numpy/f2py/src/fortranobject.c | 53 +++++++++++++++++++--------------- numpy/f2py/src/fortranobject.h | 2 +- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index 164b4c0354af..032a911a65eb 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -267,7 +267,7 @@ fortran_getattr(PyFortranObject *fp, char *name) { static int fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) { - int i,j,flag; + int i,j,k,flag; PyArrayObject *arr = NULL; for (i=0,j=1;ilen && (j=strcmp(name,fp->defs[i].name));i++); if (j==0) { @@ -277,7 +277,6 @@ fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) { } if (fp->defs[i].func!=NULL) { /* is allocatable array */ npy_intp dims[F2PY_MAX_DIMS]; - int k; save_def = &fp->defs[i]; if (v!=Py_None) { /* set new value (reallocate if needed -- see f2py generated code for more @@ -297,11 +296,15 @@ fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) { return -1; } if (fp->defs[i].data!=NULL) { /* copy Python object to Fortran array */ - npy_intp s = PyArray_MultiplyList(fp->defs[i].dims.d,arr->nd); + int nd = PyArray_NDIM(arr); + npy_intp s; + for (k=0,s=1;kdefs[i].dims.d[k]; + //npy_intp s = PyArray_MultiplyList(fp->defs[i].dims.d,arr->nd); if (s==-1) - s = PyArray_MultiplyList(arr->dimensions,arr->nd); + for (k=0,s=1;kdimensions,arr->nd); if (s<0 || - (memcpy(fp->defs[i].data,arr->data,s*PyArray_ITEMSIZE(arr)))==NULL) { + (memcpy(fp->defs[i].data,PyArray_DATA(arr),s*PyArray_ITEMSIZE(arr)))==NULL) { if ((PyObject*)arr!=v) { Py_DECREF(arr); } @@ -493,7 +496,7 @@ void f2py_report_on_exit(int exit_flag,void *name) { #ifdef F2PY_REPORT_ON_ARRAY_COPY static void f2py_report_on_array_copy(PyArrayObject* arr) { - const long arr_size = PyArray_Size((PyObject *)arr); + const long arr_size = PyArray_Size(arr); if (arr_size>F2PY_REPORT_ON_ARRAY_COPY) { fprintf(stderr,"copied an array: size=%ld, elsize=%d\n", arr_size, PyArray_ITEMSIZE(arr)); @@ -543,7 +546,7 @@ count_nonpos(const int rank, return r; } -static int check_and_fix_dimensions(const PyArrayObject* arr, +static int check_and_fix_dimensions(PyArrayObject* arr, const int rank, npy_intp *dims); @@ -556,11 +559,11 @@ void dump_dims(int rank, npy_intp* dims) { } printf("]\n"); } -void dump_attrs(const PyArrayObject* arr) { - int rank = arr->nd; - npy_intp size = PyArray_Size((PyObject *)arr); +void dump_attrs(PyArrayObject* arr) { + int rank = PyArray_NDIM(arr); + npy_intp size = PyArray_Size(arr); printf("\trank = %d, flags = %d, size = %" NPY_INTP_FMT "\n", - rank,arr->flags,size); + rank,PyArray_FLAGS(arr),size); printf("\tstrides = "); dump_dims(rank,arr->strides); printf("\tdimensions = "); @@ -570,17 +573,19 @@ void dump_attrs(const PyArrayObject* arr) { #define SWAPTYPE(a,b,t) {t c; c = (a); (a) = (b); (b) = c; } -static int swap_arrays(PyArrayObject* arr1, PyArrayObject* arr2) { - SWAPTYPE(arr1->data,arr2->data,char*); - SWAPTYPE(arr1->nd,arr2->nd,int); - SWAPTYPE(arr1->dimensions,arr2->dimensions,npy_intp*); - SWAPTYPE(arr1->strides,arr2->strides,npy_intp*); - SWAPTYPE(arr1->base,arr2->base,PyObject*); - SWAPTYPE(arr1->descr,arr2->descr,PyArray_Descr*); - SWAPTYPE(arr1->flags,arr2->flags,int); - /* SWAPTYPE(arr1->weakreflist,arr2->weakreflist,PyObject*); */ - return 0; -} +#define swap_arrays(arr1, arr2) (1) /* fail if called */ + +//static int swap_arrays(PyArrayObject* arr1, PyArrayObject* arr2) { +// SWAPTYPE(arr1->data,arr2->data,char*); +// SWAPTYPE(arr1->nd,arr2->nd,int); +// SWAPTYPE(arr1->dimensions,arr2->dimensions,npy_intp*); +// SWAPTYPE(arr1->strides,arr2->strides,npy_intp*); +// SWAPTYPE(arr1->base,arr2->base,PyObject*); +// SWAPTYPE(arr1->descr,arr2->descr,PyArray_Descr*); +// SWAPTYPE(arr1->flags,arr2->flags,int); +// /* SWAPTYPE(arr1->weakreflist,arr2->weakreflist,PyObject*); */ +// return 0; +//} #define ARRAY_ISCOMPATIBLE(arr,type_num) \ ( (PyArray_ISINTEGER(arr) && PyTypeNum_ISINTEGER(type_num)) \ @@ -772,14 +777,14 @@ PyArrayObject* array_from_pyobj(const int type_num, /*****************************************/ static -int check_and_fix_dimensions(const PyArrayObject* arr,const int rank,npy_intp *dims) { +int check_and_fix_dimensions(PyArrayObject* arr,const int rank,npy_intp *dims) { /* This function fills in blanks (that are -1\'s) in dims list using the dimensions from arr. It also checks that non-blank dims will match with the corresponding values in arr dimensions. */ const int nd = PyArray_NDIM(arr); - const npy_intp arr_size = (nd)?PyArray_Size((PyObject *)arr):1; + const npy_intp arr_size = (nd)?PyArray_Size(arr):1; #ifdef DEBUG_COPY_ND_ARRAY dump_attrs(arr); printf("check_and_fix_dimensions:init: dims="); diff --git a/numpy/f2py/src/fortranobject.h b/numpy/f2py/src/fortranobject.h index 283021aa12d6..947c806b6f3e 100644 --- a/numpy/f2py/src/fortranobject.h +++ b/numpy/f2py/src/fortranobject.h @@ -135,7 +135,7 @@ int F2PyCapsule_Check(PyObject *ptr); #endif -#define ISCONTIGUOUS(m) ((m)->flags & NPY_CONTIGUOUS) +#define ISCONTIGUOUS PyArray_ISCONTIGUOUS #define F2PY_INTENT_IN 1 #define F2PY_INTENT_INOUT 2 #define F2PY_INTENT_OUT 4 From 90b79281d98e029cc3f77f7f814fd4966adaaf4f Mon Sep 17 00:00:00 2001 From: "Stefan H. Muller" Date: Wed, 7 Aug 2013 17:37:11 +0200 Subject: [PATCH 04/10] fortranobject.c compiles now with PyPy - Manually copy arr->dimensions into array arr_dims[]. - Replace PyArray_FromAny() by PyArray_FROM_OTF(), to avoid the need to create a PyArray_Descr instance. - Replace PyArray_DescrFromType() to investigate element size by PyArray_SimpleNew() + PyArray_ITEMSIZE(). - Modify docstrings and error messages that output character codes arr->descr->type, which are impossible to get with PyPy. --- numpy/f2py/src/fortranobject.c | 51 ++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/numpy/f2py/src/fortranobject.c b/numpy/f2py/src/fortranobject.c index 032a911a65eb..3ea75ca70bd0 100644 --- a/numpy/f2py/src/fortranobject.c +++ b/numpy/f2py/src/fortranobject.c @@ -136,12 +136,12 @@ fortran_doc (FortranDataDef def) { goto fail; } } else { - PyArray_Descr *d = PyArray_DescrFromType(def.type); - if (sprintf(p+strlen(p),"'%c'-",d->type)==0) { - Py_DECREF(d); - goto fail; - } - Py_DECREF(d); + //PyArray_Descr *d = PyArray_DescrFromType(def.type); + //if (sprintf(p+strlen(p),"'%c'-",d->type)==0) { + // Py_DECREF(d); + // goto fail; + //} + //Py_DECREF(d); if (def.data==NULL) { if (sprintf(p+strlen(p),"array(%" NPY_INTP_FMT,def.dims.d[0])==0) goto fail; @@ -276,7 +276,7 @@ fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) { return -1; } if (fp->defs[i].func!=NULL) { /* is allocatable array */ - npy_intp dims[F2PY_MAX_DIMS]; + npy_intp dims[F2PY_MAX_DIMS], arr_dims[F2PY_MAX_DIMS]; save_def = &fp->defs[i]; if (v!=Py_None) { /* set new value (reallocate if needed -- see f2py generated code for more @@ -284,7 +284,9 @@ fortran_setattr(PyFortranObject *fp, char *name, PyObject *v) { for(k=0;kdefs[i].rank;k++) dims[k]=-1; if ((arr = array_from_pyobj(fp->defs[i].type,dims,fp->defs[i].rank,F2PY_INTENT_IN,v))==NULL) return -1; - (*(fp->defs[i].func))(&fp->defs[i].rank,arr->dimensions,set_data,&flag); + for(k=0;kdefs[i].rank;k++) arr_dims[k]=PyArray_DIM(arr,k); + (*(fp->defs[i].func))(&fp->defs[i].rank,arr_dims,set_data,&flag); + //(*(fp->defs[i].func))(&fp->defs[i].rank,arr->dimensions,set_data,&flag); } else { /* deallocate */ for(k=0;kdefs[i].rank;k++) dims[k]=0; (*(fp->defs[i].func))(&fp->defs[i].rank,dims,set_data,&flag); @@ -613,8 +615,8 @@ PyArrayObject* array_from_pyobj(const int type_num, */ char mess[200]; PyArrayObject *arr = NULL; - PyArray_Descr *descr; - char typechar; + //PyArray_Descr *descr; + //char typechar; int elsize; if ((intent & F2PY_INTENT_HIDE) @@ -643,10 +645,13 @@ PyArrayObject* array_from_pyobj(const int type_num, return arr; } - descr = PyArray_DescrFromType(type_num); - elsize = descr->elsize; - typechar = descr->type; - Py_DECREF(descr); + PyObject* descr_arr = PyArray_SimpleNew(0, NULL, type_num); + elsize = PyArray_ITEMSIZE(descr_arr); + Py_DECREF(descr_arr); + //descr = PyArray_DescrFromType(type_num); + //elsize = descr->elsize; + //typechar = descr->type; + //Py_DECREF(descr); if (PyArray_Check(obj)) { arr = (PyArrayObject *)obj; @@ -710,8 +715,10 @@ PyArrayObject* array_from_pyobj(const int type_num, PyArray_ITEMSIZE(arr) ); if (!(ARRAY_ISCOMPATIBLE(arr,type_num))) - sprintf(mess+strlen(mess)," -- input '%c' not compatible to '%c'", - arr->descr->type,typechar); + sprintf(mess+strlen(mess)," -- input type '%d' not compatible to type '%d'", + PyArray_TYPE(arr), type_num); + //sprintf(mess+strlen(mess)," -- input '%c' not compatible to '%c'", + // arr->descr->type,typechar); if (!(F2PY_CHECK_ALIGNMENT(arr, intent))) sprintf(mess+strlen(mess)," -- input not %d-aligned", F2PY_GET_ALIGNMENT(intent)); PyErr_SetString(PyExc_ValueError,mess); @@ -721,8 +728,12 @@ PyArrayObject* array_from_pyobj(const int type_num, /* here we have always intent(in) or intent(inplace) */ { + int k, nd = PyArray_NDIM(arr); + npy_intp arr_dims[F2PY_MAX_DIMS]; + for (k=0;knd, arr->dimensions, type_num, + PyArray_New(&PyArray_Type, nd, arr_dims, type_num, NULL,NULL,0, !(intent&F2PY_INTENT_C), NULL); @@ -759,10 +770,14 @@ PyArrayObject* array_from_pyobj(const int type_num, { F2PY_REPORT_ON_ARRAY_COPY_FROMANY; + arr = (PyArrayObject *) PyArray_FROM_OTF(obj, type_num, + ((intent & F2PY_INTENT_C)?NPY_CARRAY:NPY_FARRAY) | NPY_FORCECAST); + /* arr = (PyArrayObject *) \ PyArray_FromAny(obj,PyArray_DescrFromType(type_num), 0,0, ((intent & F2PY_INTENT_C)?NPY_CARRAY:NPY_FARRAY) \ | NPY_FORCECAST, NULL); + */ if (arr==NULL) return NULL; if (check_and_fix_dimensions(arr,rank,dims)) @@ -784,7 +799,7 @@ int check_and_fix_dimensions(PyArrayObject* arr,const int rank,npy_intp *dims) { match with the corresponding values in arr dimensions. */ const int nd = PyArray_NDIM(arr); - const npy_intp arr_size = (nd)?PyArray_Size(arr):1; + const npy_intp arr_size = (nd)?PyArray_Size((PyObject*)arr):1; #ifdef DEBUG_COPY_ND_ARRAY dump_attrs(arr); printf("check_and_fix_dimensions:init: dims="); From 5c2e11f6c57feceb28dd8dff2da23e30b3e5b01b Mon Sep 17 00:00:00 2001 From: "Stefan H. Muller" Date: Sat, 10 Aug 2013 00:37:14 +0200 Subject: [PATCH 05/10] Reactivate building of lapack_lite - numpy/linalg/lapack_lite.py: Delete placeholder file. - numpy/distutils/misc_util.py: Add 'activate' flag to selectively activate building of a module. Use in numpy/linalg/setup.py. Note that numpy/core/include must be renamed so that the PyPy headers are found. - numpy/core/multiarray.py: Add implemenations for copyto() and fastCopyAndTranspose(). - numpy/core/numeric.py: ones(): Use numpypy's implementation. --- numpy/core/multiarray.py | 11 +++++++++-- numpy/core/numeric.py | 7 ++++--- numpy/distutils/misc_util.py | 4 +++- numpy/linalg/lapack_lite.py | 0 numpy/linalg/setup.py | 4 ++-- 5 files changed, 18 insertions(+), 8 deletions(-) delete mode 100644 numpy/linalg/lapack_lite.py diff --git a/numpy/core/multiarray.py b/numpy/core/multiarray.py index 429ac286a924..60cdeffe4309 100644 --- a/numpy/core/multiarray.py +++ b/numpy/core/multiarray.py @@ -7,11 +7,18 @@ set_string_function, typeinfo, where, zeros) ''' +def _fastCopyAndTranspose(a): + return a.T.copy() + +def copyto(dst, src, casting='same_kind', where=None, preservena=False): + dst.fill(src) + + undef = '''CLIP WRAP RAISE MAXDIMS ALLOW_THREADS BUFSIZE nditer nested_iters broadcast empty_like fromiter fromfile frombuffer newbuffer getbuffer -int_asbuffer _fastCopyAndTranspose set_numeric_ops can_cast promote_types +int_asbuffer set_numeric_ops can_cast promote_types min_scalar_type result_type lexsort compare_chararrays putmask einsum inner -_vec_string copyto datetime_data format_longfloat +_vec_string datetime_data format_longfloat datetime_as_string busday_offset busday_count is_busday busdaycalendar _flagdict flagsobj '''.split() diff --git a/numpy/core/numeric.py b/numpy/core/numeric.py index 45b998b61b3a..7f865e63a03e 100644 --- a/numpy/core/numeric.py +++ b/numpy/core/numeric.py @@ -145,9 +145,10 @@ def ones(shape, dtype=None, order='C'): [ 1., 1.]]) """ - a = empty(shape, dtype, order) - multiarray.copyto(a, 1, casting='unsafe') - return a + #a = empty(shape, dtype, order) + #multiarray.copyto(a, 1, casting='unsafe') + #return a + return multiarray.ones(shape, dtype, order) def ones_like(a, dtype=None, order='K', subok=True): """ diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index 77a51828ff9c..4313f61866d1 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -1442,6 +1442,7 @@ def add_extension(self,name,sources,**kw): The self.paths(...) method is applied to all lists that may contain paths. """ + activate = kw.pop('activate', False) ext_args = copy.copy(kw) ext_args['name'] = dot_join(self.name,name) ext_args['sources'] = sources @@ -1488,7 +1489,8 @@ def add_extension(self,name,sources,**kw): from numpy.distutils.core import Extension ext = Extension(**ext_args) #XXX: This was commented out to prevent the building of extension modules - #self.ext_modules.append(ext) + if activate: + self.ext_modules.append(ext) dist = self.get_distribution() if dist is not None: diff --git a/numpy/linalg/lapack_lite.py b/numpy/linalg/lapack_lite.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 1fb7a3acd0f8..5f668d3c52bc 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -27,8 +27,8 @@ def get_lapack_lite_sources(ext, build_dir): 'zlapack_lite.c', 'dlapack_lite.c', 'blas_lite.c', 'dlamch.c', 'f2c_lite.c','f2c.h'], - extra_info = lapack_info - ) + extra_info = lapack_info, + activate = True) return config From 8bf181d6ac1cacd79edebe8cbb2ba3c4ee6d1641 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sat, 5 Oct 2013 22:17:00 +0300 Subject: [PATCH 06/10] 'setup.py install' works for a clean checkout --- numpy/core/setup.py | 5 +++-- numpy/distutils/misc_util.py | 12 +++++++++--- numpy/linalg/setup.py | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 5a40e75072a1..f49b97c6904f 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -376,7 +376,6 @@ def configuration(parent_package='',top_path=None): from numpy.distutils.system_info import get_info, default_lib_dirs config = Configuration('core',parent_package,top_path) - return config local_dir = config.local_path codegen_dir = join(local_dir,'code_generators') @@ -627,9 +626,11 @@ def generate_api(ext, build_dir): sources = [join('src','dummymodule.c'), generate_config_h, generate_numpyconfig_h, - generate_numpy_api] + generate_numpy_api], + activate = True, ) + return config ####################################################################### # npymath library # ####################################################################### diff --git a/numpy/distutils/misc_util.py b/numpy/distutils/misc_util.py index 4313f61866d1..777a027b54c8 100644 --- a/numpy/distutils/misc_util.py +++ b/numpy/distutils/misc_util.py @@ -17,6 +17,8 @@ except NameError: from sets import Set as set +is_pypy = '__pypy__' in sys.builtin_module_names + from numpy.distutils.compat import get_exception __all__ = ['Configuration', 'get_numpy_include_dirs', 'default_config_dict', @@ -27,7 +29,7 @@ 'has_f_sources', 'has_cxx_sources', 'filter_sources', 'get_dependencies', 'is_local_src_dir', 'get_ext_source_files', 'get_script_files', 'get_lib_source_files', 'get_data_files', - 'dot_join', 'get_frame', 'minrelpath','njoin', + 'dot_join', 'get_frame', 'minrelpath','njoin', 'is_pypy', 'is_sequence', 'is_string', 'as_list', 'gpaths', 'get_language', 'quote_args', 'get_build_architecture', 'get_info', 'get_pkg_info'] @@ -719,7 +721,12 @@ class Configuration(object): _dict_keys = ['package_dir', 'installed_pkg_config'] _extra_keys = ['name', 'version'] - numpy_include_dirs = [] + if is_pypy: + import numpypy as np + numpy_include_dirs = [np.get_include()] + del np + else: + numpy_include_dirs = [] def __init__(self, package_name=None, @@ -2141,7 +2148,6 @@ def get_cmd(cmdname, _cache={}): def get_numpy_include_dirs(): # numpy_include_dirs are set by numpy/core/setup.py, otherwise [] include_dirs = Configuration.numpy_include_dirs[:] - return include_dirs if not include_dirs: import numpy include_dirs = [ numpy.get_include() ] diff --git a/numpy/linalg/setup.py b/numpy/linalg/setup.py index 5f668d3c52bc..24a33e09c818 100644 --- a/numpy/linalg/setup.py +++ b/numpy/linalg/setup.py @@ -28,7 +28,8 @@ def get_lapack_lite_sources(ext, build_dir): 'blas_lite.c', 'dlamch.c', 'f2c_lite.c','f2c.h'], extra_info = lapack_info, - activate = True) + activate = True, + ) return config From 3ad9bdafe76a5ada19725fbdc4a70b98b81896cd Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 6 Oct 2013 22:56:42 +0300 Subject: [PATCH 07/10] allow 'from numpy.random import *' to work --- numpy/random/info.py | 88 +++++++++++++++++++++--------------------- numpy/random/mtrand.py | 7 ++++ 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/numpy/random/info.py b/numpy/random/info.py index 6139e5784170..796733f0ceac 100644 --- a/numpy/random/info.py +++ b/numpy/random/info.py @@ -86,49 +86,49 @@ depends = ['core'] __all__ = [ - 'beta', - 'binomial', - 'bytes', - 'chisquare', - 'exponential', - 'f', - 'gamma', - 'geometric', - 'get_state', - 'gumbel', - 'hypergeometric', - 'laplace', - 'logistic', - 'lognormal', - 'logseries', - 'multinomial', - 'multivariate_normal', - 'negative_binomial', - 'noncentral_chisquare', - 'noncentral_f', - 'normal', - 'pareto', - 'permutation', - 'poisson', - 'power', - 'rand', - 'randint', - 'randn', - 'random_integers', - 'random_sample', - 'rayleigh', + #'beta', + #'binomial', + #'bytes', + #'chisquare', + #'exponential', + #'f', + #'gamma', + #'geometric', + #'get_state', + #'gumbel', + #'hypergeometric', + #'laplace', + #'logistic', + #'lognormal', + #'logseries', + #'multinomial', + #'multivariate_normal', + #'negative_binomial', + #'noncentral_chisquare', + #'noncentral_f', + #'normal', + #'pareto', + #'permutation', + #'poisson', + #'power', + #'rand', + #'randint', + #'randn', + #'random_integers', + #'random_sample', + #'rayleigh', 'seed', - 'set_state', - 'shuffle', - 'standard_cauchy', - 'standard_exponential', - 'standard_gamma', - 'standard_normal', - 'standard_t', - 'triangular', - 'uniform', - 'vonmises', - 'wald', - 'weibull', - 'zipf' + #'set_state', + #'shuffle', + #'standard_cauchy', + #'standard_exponential', + #'standard_gamma', + #'standard_normal', + #'standard_t', + #'triangular', + #'uniform', + #'vonmises', + #'wald', + #'weibull', + #'zipf' ] diff --git a/numpy/random/mtrand.py b/numpy/random/mtrand.py index eab67bcedfd0..5b3069f829bb 100644 --- a/numpy/random/mtrand.py +++ b/numpy/random/mtrand.py @@ -1,6 +1,13 @@ from random import Random from numpy import zeros +# This file is used in pypy's version of mtrand, +# cpython compiles a mtrand capi module. + +# as functionality is added here, uncomment the function +# in __all__ located in numpy/random/info.py + + _rand = Random() def random_sample(length=0): if length == 0: From 882fe4e06596ad7dd25d038b245080187489797d Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 13 Oct 2013 02:07:52 +0300 Subject: [PATCH 08/10] rework header files while maintaining backward compatability --- .../code_generators/generate_numpy_api.py | 23 ++++++++++++++++--- numpy/core/setup.py | 4 ++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index 7cd4b9f6a99b..bf227cf2ebb8 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -1,4 +1,5 @@ import os +import sys import genapi from genapi import \ @@ -141,6 +142,18 @@ #endif """ +h_pypy_template = r""" + +typedef struct { + PyObject_HEAD + npy_bool obval; +} PyBoolScalarObject; + +#define import_array() +extern PyTypeObject PyArray_Type; +#define PyArray_New _PyArray_New + +""" c_template = r""" /* These pointers will be stored in the C-object for use in other @@ -233,14 +246,18 @@ def do_generate_api(targets, sources): # Write to header fid = open(header_file, 'w') - s = h_template % ('\n'.join(module_list), '\n'.join(extension_list)) + if '__pypy__' in sys.builtin_module_names: + s = h_pypy_template + else: + s = h_template % ('\n'.join(module_list), '\n'.join(extension_list)) fid.write(s) fid.close() # Write to c-code fid = open(c_file, 'w') - s = c_template % ',\n'.join(init_list) - fid.write(s) + if '__pypy__' not in sys.builtin_module_names: + s = c_template % ',\n'.join(init_list) + fid.write(s) fid.close() # write to documentation diff --git a/numpy/core/setup.py b/numpy/core/setup.py index f49b97c6904f..4e7d86c35f76 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -594,6 +594,10 @@ def generate_api(ext, build_dir): config.add_include_dirs(join(local_dir)) config.add_data_files('include/numpy/*.h') + if '__pypy__' in sys.builtin_module_names: + # Copy pypy's compatability file, too much bother to clean up the + # multiple levels of ifdefs + config.add_data_files(('include/numpy', sys.exec_prefix + '/include/numpy/npy_3kcompat.h')) config.add_include_dirs(join('src', 'npymath')) config.add_include_dirs(join('src', 'multiarray')) config.add_include_dirs(join('src', 'umath')) From 210f7ca3e988254dcea4b003cfb5e12668f8c7f0 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 13 Oct 2013 22:43:24 +0300 Subject: [PATCH 09/10] just copy numpypy's headers into site-packages --- numpy/core/setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 4e7d86c35f76..33e51de0d531 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -595,9 +595,8 @@ def generate_api(ext, build_dir): config.add_data_files('include/numpy/*.h') if '__pypy__' in sys.builtin_module_names: - # Copy pypy's compatability file, too much bother to clean up the - # multiple levels of ifdefs - config.add_data_files(('include/numpy', sys.exec_prefix + '/include/numpy/npy_3kcompat.h')) + # Copy pypy's builting headers + config.add_data_files(('include/numpy', sys.exec_prefix + '/include/numpy/*.h')) config.add_include_dirs(join('src', 'npymath')) config.add_include_dirs(join('src', 'multiarray')) config.add_include_dirs(join('src', 'umath')) From 229e2459140862e369e8f31dde6df351cfd67bb0 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Sun, 13 Oct 2013 23:16:43 +0300 Subject: [PATCH 10/10] clarify and add a pypy conditional to difference between pypy and cpython --- numpy/linalg/linalg.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/numpy/linalg/linalg.py b/numpy/linalg/linalg.py index 7f465ed2c0de..ba11e081899d 100644 --- a/numpy/linalg/linalg.py +++ b/numpy/linalg/linalg.py @@ -33,6 +33,9 @@ fortran_int = intc +import sys +ispypy = '__pypy__' in sys.builtin_module_names + # Error object class LinAlgError(Exception): """ @@ -63,9 +66,15 @@ class LinAlgError(Exception): """ pass +def do_nothing(*args): + return args[0] + def _makearray(a): new = asarray(a) - wrap = a + if ispypy: + wrap = do_nothing + else: + wrap = getattr(a, "__array_prepare__", new.__array_wrap__) return new, wrap def isComplexType(t):