Merge branch 'master' into valgrind

.gitignore

@@ -1,6 +1,10 @@
 # HPy autogen
 hpy/tools/autogen/autogen_pypy.txt
 
+# generated by setup.py:get_scm_config()
+hpy/devel/include/common/version.h
+hpy/devel/version.py
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

Makefile

@@ -7,5 +7,13 @@ debug:
 autogen:
 	python3 -m hpy.tools.autogen .
 
+PYTHON_INCLUDE_DIR=$(shell python $$PWD/hpy/tools/include_path.py)
+
+cppcheck-build-dir:
+	mkdir -p $(or ${CPPCHECK_BUILD_DIR}, .cppcheck)
+
+cppcheck: cppcheck-build-dir
+	@cppcheck --error-exitcode=1 --cppcheck-build-dir=$(or ${CPPCHECK_BUILD_DIR}, .cppcheck) --enable=warning,performance,portability,information,unusedFunction,missingInclude --report-progress -I hpy/devel/include/ -I hpy/devel/include/common/ -I hpy/devel/include/cpython/ -I hpy/devel/include/universal/ -I hpy/universal/src/ -I ${PYTHON_INCLUDE_DIR} --force -D NULL=0 .
+
 valgrind:
 	PYTHONMALLOC=malloc valgrind --suppressions=hpy/tools/valgrind/python.supp --suppressions=hpy/tools/valgrind/hpy.supp --leak-check=full --show-leak-kinds=definite,indirect --log-file=/tmp/valgrind-output python -m pytest --valgrind --valgrind-log=/tmp/valgrind-output test/

azure-pipelines.yml

@@ -22,16 +22,16 @@ jobs:
     displayName: 'Use Python $(python.version)'
 
   - script: |
-      python -m pip install --upgrade pip
+      python -m pip install --upgrade pip wheel
     displayName: 'Install dependencies'
 
   - script: |
       python -m pip install .
     displayName: 'Build project'
 
   - script: |
-      pip install pytest pytest-azurepipelines
-      pytest test/
+      pip install pytest pytest-azurepipelines pytest-xdist
+      pytest -n auto test/
     displayName: 'pytest'
 
 - job: ExampleTests
@@ -117,15 +117,35 @@ jobs:
       python test/check_py27_compat.py
     displayName: 'check_py27_compat.py'
 
-- job: Valgrind
+- job: CPPCheck
   pool:
     vmImage: 'ubuntu-latest'
-  displayName: 'Check memory leaks'
+  displayName: "Run CPPCheck"
+  variables:
+    CPPCHECK_BUILD_DIR: $(Pipeline.Workspace)/.cppcheck
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '3.8'
     displayName: 'Use Python 3.8'
+
+
+  - script: sudo snap install cppcheck --channel=latest/stable
+    displayName: Install CPPCheck
+
+  - task: Cache@2
+    inputs:
+      key: 'cppcheck | "$(Agent.OS)"'
+      path: $(CPPCHECK_BUILD_DIR)
+    displayName: Cache CPPCheck analysis files
+
+  - script: make cppcheck
+    displayName: Run CPPCheck
+
+- job: Valgrind
+  pool:
+    vmImage: 'ubuntu-latest'
+  displayName: 'Check memory leaks'
   - script: sudo apt update && sudo apt install -y valgrind
     displayName: 'Install valgrind'
   - script: python -m pip install --upgrade pip
@@ -135,4 +155,4 @@ jobs:
   - script: |
       pip install pytest pytest-azurepipelines pytest-valgrind
       make valgrind
-    displayName: 'Run valgrind'
+    displayName: 'Run valgrind'

docs/module-state.txt

@@ -0,0 +1,44 @@
+How to replace global variables
+-------------------------------
+
+In a given .c source, write:
+
+
+typedef struct {
+    long x;
+    HPy y;
+} my_globals_t;
+
+static void my_globals_traverse(traversefunc traverse, my_globals_t *g)
+{
+    traverse(g->y);
+}
+
+HPyGlobalSpec my_globals = {
+    .m_size = sizeof(my_globals_t),
+    .m_traverse = my_globals_traverse
+};
+
+
+There can be several HPyGlobalSpec structures around; in CPython it's done as
+part of the PyModuleDef type, but there is no real reason for why it should
+be tightly tied to a module.
+
+
+To use:
+
+    my_globals_t *g = HPy_GetState(ctx, &my_globals);
+    g->x++;
+    HPy_DoRandomStuffWithHandle(ctx, g->y);
+
+
+Implementation: the type HPyGlobalSpec contains extra internal fields
+which should give us a very fast cache: _last_ctx and _last_result,
+and HPy_GetState() can be:
+
+    if (ctx == globspec->_last_ctx)
+        return globspec->_last_result;
+    else
+        look up globspec in a dictionary attached to ctx, or vice-versa,
+        or maybe initialize globspec->_index with a unique incrementing
+        index and use that to index an array attached to ctx

hpy/devel/include/common/hpytype.h

@@ -31,6 +31,18 @@ typedef struct {
     HPyDef **defines;   /* points to an array of 'HPyDef *' */
 } HPyType_Spec;
 
+typedef enum {
+    HPyType_SpecParam_Base = 1,
+    HPyType_SpecParam_BasesTuple = 2,
+    //HPyType_SpecParam_Metaclass = 3,
+    //HPyType_SpecParam_Module = 4,
+} HPyType_SpecParam_Kind;
+
+typedef struct {
+    HPyType_SpecParam_Kind kind;
+    HPy object;
+} HPyType_SpecParam;
+
 /* All types are dynamically allocated */
 #define _Py_TPFLAGS_HEAPTYPE (1UL << 9)
 

hpy/devel/include/common/runtime/ctx_type.h

@@ -6,7 +6,8 @@
 #include "common/hpytype.h"
 
 _HPy_HIDDEN void* ctx_Cast(HPyContext ctx, HPy h);
-_HPy_HIDDEN HPy ctx_Type_FromSpec(HPyContext ctx, HPyType_Spec *hpyspec);
+_HPy_HIDDEN HPy ctx_Type_FromSpec(HPyContext ctx, HPyType_Spec *hpyspec,
+                                  HPyType_SpecParam *params);
 _HPy_HIDDEN HPy ctx_New(HPyContext ctx, HPy h_type, void **data);
 _HPy_HIDDEN HPy ctx_Type_GenericNew(HPyContext ctx, HPy h_type, HPy *args,
                                     HPy_ssize_t nargs, HPy kw);

hpy/devel/include/cpython/hpy.h

@@ -151,9 +151,9 @@ HPyModule_Create(HPyContext ctx, HPyModuleDef *mdef)
 }
 
 HPyAPI_FUNC(HPy)
-HPyType_FromSpec(HPyContext ctx, HPyType_Spec *spec)
+HPyType_FromSpec(HPyContext ctx, HPyType_Spec *spec, HPyType_SpecParam *params)
 {
-    return ctx_Type_FromSpec(ctx, spec);
+    return ctx_Type_FromSpec(ctx, spec, params);
 }
 
 HPyAPI_FUNC(HPy)

hpy/devel/src/runtime/ctx_type.c

@@ -310,10 +310,85 @@ create_slot_defs(HPyType_Spec *hpyspec)
     return result;
 }
 
+static int check_unknown_params(HPyType_SpecParam *params, const char *name)
+{
+    if (params == NULL)
+        return 0;
+
+    int found_base = 0, found_basestuple = 0;
+    for (HPyType_SpecParam *p = params; p->kind != 0; p++) {
+        switch (p->kind) {
+            case HPyType_SpecParam_Base:
+                found_base++;
+                break;
+            case HPyType_SpecParam_BasesTuple:
+                found_basestuple++;
+                break;
+
+            default:
+                PyErr_Format(PyExc_TypeError,
+                    "unknown HPyType_SpecParam specification for '%s'",
+                    name);
+                return -1;
+        }
+    }
+    if (found_basestuple > 1) {
+        PyErr_SetString(PyExc_TypeError,
+            "multiple specifications of HPyType_SpecParam_BasesTuple");
+        return -1;
+    }
+    if (found_base && found_basestuple) {
+        PyErr_SetString(PyExc_TypeError,
+            "cannot specify both HPyType_SpecParam_Base and "
+            "HPytype_SpecParam_BasesTuple");
+        return -1;
+    }
+    return 0;
+}
+
+static PyObject *build_bases_from_params(HPyType_SpecParam *params)
+{
+    if (params == NULL)
+        return NULL;
+
+    int found_base = 0;
+    for (HPyType_SpecParam *p = params; p->kind != 0; p++) {
+        switch (p->kind) {
+            case HPyType_SpecParam_Base:
+                /* count the base entries (multiple entries are fine) */
+                found_base++;
+                break;
+            case HPyType_SpecParam_BasesTuple:
+                /* if there is instead a complete base tuple, just return it */
+                return _h2py(p->object);
+        }
+    }
+    if (found_base == 0)
+        return NULL;
+
+    PyObject *tup = PyTuple_New(found_base);
+    if (tup == NULL)
+        return NULL;
+
+    found_base = 0;
+    for (HPyType_SpecParam *p = params; p->kind != 0; p++) {
+        if (p->kind == HPyType_SpecParam_Base) {
+            PyObject *base = _h2py(p->object);
+            Py_INCREF(base);
+            PyTuple_SET_ITEM(tup, found_base, base);
+            found_base++;
+        }
+    }
+    return tup;
+}
 
 _HPy_HIDDEN HPy
-ctx_Type_FromSpec(HPyContext ctx, HPyType_Spec *hpyspec)
+ctx_Type_FromSpec(HPyContext ctx, HPyType_Spec *hpyspec,
+                  HPyType_SpecParam *params)
 {
+    if (check_unknown_params(params, hpyspec->name) < 0) {
+        return HPy_NULL;
+    }
     PyType_Spec *spec = PyMem_Calloc(1, sizeof(PyType_Spec));
     if (spec == NULL) {
         PyErr_NoMemory();
@@ -328,7 +403,11 @@ ctx_Type_FromSpec(HPyContext ctx, HPyType_Spec *hpyspec)
         PyMem_Free(spec);
         return HPy_NULL;
     }
-    PyObject *result = PyType_FromSpec(spec);
+    PyObject *bases = build_bases_from_params(params);
+    if (PyErr_Occurred()) {
+        return HPy_NULL;
+    }
+    PyObject *result = PyType_FromSpecWithBases(spec, bases);
     /* note that we do NOT free the memory which was allocated by
        create_method_defs, because that one is referenced internally by
        CPython (which probably assumes it's statically allocated) */
@@ -349,6 +428,11 @@ ctx_New(HPyContext ctx, HPy h_type, void **data)
     PyObject *result = PyObject_New(PyObject, (PyTypeObject*)tp);
     if (!result)
         return HPy_NULL;
+#if PY_VERSION_HEX < 0x03080000
+    // Workaround for Python issue 35810; no longer necessary in Python 3.8
+    // TODO: Remove this workaround once we no longer support Python versions older than 3.8
+    Py_INCREF(tp);
+#endif
 
     *data = (void*)result;
     return _py2h(result);

hpy/tools/autogen/public_api.h

@@ -3,6 +3,7 @@ typedef int HPy;
 typedef int HPyContext;
 typedef int HPyModuleDef;
 typedef int HPyType_Spec;
+typedef int HPyType_SpecParam;
 typedef int HPyCFunction;
 typedef int HPy_ssize_t;
 typedef int HPy_hash_t;
@@ -90,7 +91,8 @@ HPy HPyErr_NoMemory(HPyContext ctx);
 
 /* object.h */
 int HPy_IsTrue(HPyContext ctx, HPy h);
-HPy HPyType_FromSpec(HPyContext ctx, HPyType_Spec *spec);
+HPy HPyType_FromSpec(HPyContext ctx, HPyType_Spec *spec,
+                     HPyType_SpecParam *params);
 HPy HPyType_GenericNew(HPyContext ctx, HPy type, HPy *args, HPy_ssize_t nargs, HPy kw);
 
 HPy HPy_GetAttr(HPyContext ctx, HPy obj, HPy name);

hpy/tools/include_path.py

@@ -0,0 +1,4 @@
+"""Prints the include path for the current Python interpreter."""
+
+from sysconfig import get_paths as gp
+print(gp()['include'])

hpy/universal/src/api.h

@@ -6,16 +6,4 @@
 #define HPyAPI_STORAGE _HPy_HIDDEN
 extern struct _HPyContext_s global_ctx;
 
-/* declare alloca() */
-#if defined(_MSC_VER)
-# include <malloc.h>   /* for alloca() */
-#else
-# include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
-#  include <alloca.h>
-# endif
-#endif
-
-
-
 #endif /* HPY_API_H */

hpy/universal/src/ctx_meth.c

@@ -23,7 +23,7 @@ ctx_CallRealFunctionFromTrampoline(HPyContext ctx, HPyFunc_Signature sig,
         HPyFunc_varargs f = (HPyFunc_varargs)func;
         _HPyFunc_args_VARARGS *a = (_HPyFunc_args_VARARGS*)args;
         Py_ssize_t nargs = PyTuple_GET_SIZE(a->args);
-        HPy *h_args = alloca(nargs * sizeof(HPy));
+        HPy h_args[nargs * sizeof(HPy)];
         for (Py_ssize_t i = 0; i < nargs; i++) {
             h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i));
         }
@@ -34,7 +34,7 @@ ctx_CallRealFunctionFromTrampoline(HPyContext ctx, HPyFunc_Signature sig,
        HPyFunc_keywords f = (HPyFunc_keywords)func;
        _HPyFunc_args_KEYWORDS *a = (_HPyFunc_args_KEYWORDS*)args;
        Py_ssize_t nargs = PyTuple_GET_SIZE(a->args);
-       HPy *h_args = alloca(nargs * sizeof(HPy));
+       HPy h_args[nargs * sizeof(HPy)];
        for (Py_ssize_t i = 0; i < nargs; i++) {
            h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i));
        }
@@ -45,7 +45,7 @@ ctx_CallRealFunctionFromTrampoline(HPyContext ctx, HPyFunc_Signature sig,
        HPyFunc_initproc f = (HPyFunc_initproc)func;
        _HPyFunc_args_INITPROC *a = (_HPyFunc_args_INITPROC*)args;
        Py_ssize_t nargs = PyTuple_GET_SIZE(a->args);
-       HPy *h_args = alloca(nargs * sizeof(HPy));
+       HPy h_args[nargs * sizeof(HPy)];
        for (Py_ssize_t i = 0; i < nargs; i++) {
            h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i));
        }