From daf3e1f537cf891bb7eeaf08af18a1bc3fade05e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 27 Feb 2025 09:15:36 +0100 Subject: [PATCH 1/3] Update HPy inlined files: b0fbdf73 --- .../include/hpy/version.h | 4 - .../src/hpytest/conftest.py | 2 +- .../src/hpytest/debug/__init__.py | 0 .../src/hpytest/debug/test_charptr.py | 4 +- .../src/hpytest/debug/test_handles_invalid.py | 13 +- .../src/hpytest/debug/test_misc.py | 2 +- .../src/hpytest/hpy_devel/__init__.py | 0 .../src/hpytest/hpy_devel/test_distutils.py | 2 +- .../src/hpytest/support.py | 28 +-- .../src/hpytest/test_00_basic.py | 22 +- .../src/hpytest/test_argparse.py | 18 +- .../src/hpytest/test_capsule.py | 4 + .../src/hpytest/test_eval.py | 2 +- .../src/hpytest/test_hpyfield.py | 3 +- .../src/hpytest/test_hpyiter.py | 90 +++++++ .../src/hpytest/test_hpylist.py | 35 +++ .../src/hpytest/test_hpymodule.py | 10 +- .../src/hpytest/test_hpyslice.py | 28 +++ .../src/hpytest/test_hpytype.py | 10 +- .../src/hpytest/test_hpytype_legacy.py | 52 ++++ .../src/hpytest/test_hpyunicode.py | 10 +- .../src/hpytest/test_importing.py | 2 +- .../src/hpytest/test_legacy_forbidden.py | 2 - .../src/hpytest/test_object.py | 223 ++++++++++++++++++ .../src/hpytest/test_slots.py | 26 +- .../include/hpy.h | 0 .../include/hpy/autogen_hpyfunc_declare.h | 0 .../include/hpy/autogen_hpyslot.h | 2 + .../include/hpy/cpy_types.h | 0 .../include/hpy/cpython/autogen_api_impl.h | 39 ++- .../include/hpy/cpython/autogen_ctx.h | 1 + .../hpy/cpython/autogen_hpyfunc_trampolines.h | 0 .../include/hpy/cpython/hpyfunc_trampolines.h | 0 .../include/hpy/cpython/misc.h | 15 +- .../include/hpy/forbid_python_h/Python.h | 0 .../include/hpy/hpydef.h | 2 +- .../include/hpy/hpyexports.h | 0 .../include/hpy/hpyfunc.h | 0 .../include/hpy/hpymodule.h | 10 +- .../include/hpy/hpytype.h | 18 +- .../include/hpy/inline_helpers.h | 0 .../include/hpy/macros.h | 0 .../include/hpy/runtime/argparse.h | 0 .../include/hpy/runtime/buildvalue.h | 0 .../include/hpy/runtime/ctx_funcs.h | 4 +- .../include/hpy/runtime/ctx_module.h | 0 .../include/hpy/runtime/ctx_type.h | 0 .../include/hpy/runtime/format.h | 0 .../include/hpy/runtime/helpers.h | 0 .../include/hpy/runtime/structseq.h | 0 .../include/hpy/universal/autogen_ctx.h | 12 +- .../universal/autogen_hpyfunc_trampolines.h | 0 .../hpy/universal/autogen_trampolines.h | 38 ++- .../hpy/universal/hpyfunc_trampolines.h | 0 .../include/hpy/universal/misc_trampolines.h | 0 .../include/hpy/version.h | 4 + .../src/debug/_debugmod.c | 2 +- .../src/debug/autogen_debug_ctx_init.h | 21 +- .../src/debug/autogen_debug_wrappers.c | 100 ++++++++ .../src/debug/debug_ctx.c | 5 +- .../src/trace/autogen_trace_ctx_init.h | 25 +- .../src/trace/autogen_trace_func_table.c | 14 +- .../src/trace/autogen_trace_wrappers.c | 119 +++++++++- .../modules/hpy/devel/__init__.py | 2 +- .../hpy/devel/src/runtime/ctx_object.c | 14 +- .../modules/hpy/devel/src/runtime/ctx_tuple.c | 2 +- .../modules/hpy/devel/src/runtime/ctx_type.c | 9 + .../modules/hpy/devel/version.py | 4 +- .../modules/hpy/trace/leakdetector.py | 43 ++++ .../modules/hpy/trace/pytest.py | 31 +++ 70 files changed, 1047 insertions(+), 81 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py create mode 100644 graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/autogen_hpyfunc_declare.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/autogen_hpyslot.h (98%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpy_types.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_api_impl.h (92%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_ctx.h (99%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/cpython/misc.h (97%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/forbid_python_h/Python.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpydef.h (99%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpyexports.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpyfunc.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpymodule.h (92%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/hpytype.h (94%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/inline_helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/macros.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/argparse.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/buildvalue.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_funcs.h (96%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_module.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/ctx_type.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/format.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/runtime/structseq.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_ctx.h (95%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/autogen_trampolines.h (94%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy.llvm => com.oracle.graal.python.hpy}/include/hpy/universal/misc_trampolines.h (100%) create mode 100644 graalpython/com.oracle.graal.python.hpy/include/hpy/version.h create mode 100644 graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py create mode 100644 graalpython/lib-graalpython/modules/hpy/trace/pytest.py diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h deleted file mode 100644 index decdc2922f..0000000000 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/version.h +++ /dev/null @@ -1,4 +0,0 @@ - -// automatically generated by setup.py:get_scm_config() -#define HPY_VERSION "0.9.0" -#define HPY_GIT_REVISION "f6114734" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py index 8c5c78aa08..372843459f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py @@ -1,7 +1,6 @@ import pytest from .support import ExtensionCompiler, DefaultExtensionTemplate,\ PythonSubprocessRunner, HPyDebugCapture, make_hpy_abi_fixture -from hpy.debug.leakdetector import LeakDetector from pathlib import Path IS_VALGRIND_RUN = False @@ -48,6 +47,7 @@ def leakdetector(hpy_abi): """ Automatically detect leaks when the hpy_abi == 'debug' """ + from hpy.debug.leakdetector import LeakDetector if 'debug' in hpy_abi: with LeakDetector() as ld: yield ld diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py index cf59cd5e9f..7df8d8f2c3 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py @@ -1,6 +1,6 @@ import os import pytest -from test.support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION # Tests detection of usage of char pointers associated with invalid already # closed handles. For now, the debug mode does not provide any hook for this @@ -293,4 +293,4 @@ def clear_raw_data_in_closed_handles(): assert h.raw_data_size == -1 finally: _debug.set_protected_raw_data_max_size(old_raw_data_max_size) - _debug.set_closed_handles_queue_max_size(old_closed_handles_max_size) \ No newline at end of file + _debug.set_closed_handles_queue_max_size(old_closed_handles_max_size) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py index 32aec8e415..36622eece9 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py @@ -1,7 +1,8 @@ import pytest +import sys from hpy.debug.leakdetector import LeakDetector -from test.support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD -from test.conftest import IS_VALGRIND_RUN +from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD +from ..conftest import IS_VALGRIND_RUN @pytest.fixture def hpy_abi(): @@ -9,6 +10,8 @@ def hpy_abi(): yield "debug" +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_no_invalid_handle(compiler, hpy_debug_capture): # Basic sanity check that valid code does not trigger any error reports mod = compiler.make_module(""" @@ -33,6 +36,8 @@ def test_no_invalid_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 0 +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_cant_use_closed_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPyDef_METH(f, "f", HPyFunc_O, .doc="double close") @@ -106,6 +111,8 @@ def test_cant_use_closed_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 6 +@pytest.mark.skipif(sys.implementation.name == 'pypy', + reason="Cannot recover from use-after-close on pypy") def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPy keep; @@ -197,4 +204,4 @@ def test_invalid_handle_crashes_python_if_no_hook(compiler, python_subprocess, f """) result = python_subprocess.run(mod, "mod.f(42);") assert result.returncode == fatal_exit_code - assert b"Invalid usage of already closed handle" in result.stderr \ No newline at end of file + assert b"Invalid usage of already closed handle" in result.stderr diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py index 0c6e0a4ca2..2994da4b3e 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py @@ -1,5 +1,5 @@ import pytest -from test.support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION @pytest.fixture def hpy_abi(): diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py index 00bbb3a52a..f6b2daae39 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py @@ -16,7 +16,7 @@ import py import pytest -from test.support import atomic_run, HPY_ROOT +from ..support import atomic_run, HPY_ROOT # ====== IMPORTANT DEVELOPMENT TIP ===== # You can use py.test --reuse-venv to speed up local testing. diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py index 4278c3da13..250640548f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py @@ -49,16 +49,19 @@ def make_hpy_abi_fixture(ABIs, class_fixture=False): class TestFoo(HPyTest): hpy_abi = make_hpy_abi_fixture('with hybrid', class_fixture=True) """ - if ABIs == 'default': - ABIs = ['cpython', 'universal', 'debug'] - elif ABIs == 'with hybrid': - ABIs = ['cpython', 'hybrid', 'hybrid+debug'] - elif isinstance(ABIs, list): + is_cpython = not hasattr(sys, "implementation") or sys.implementation.name == 'cpython' + if isinstance(ABIs, list): pass else: - raise ValueError("ABIs must be 'default', 'with hybrid' " - "or a list of strings. Got: %s" % ABIs) - + if ABIs == 'default': + ABIs = ['universal', 'debug'] + elif ABIs == 'with hybrid': + ABIs = ['hybrid', 'hybrid+debug'] + else: + raise ValueError("ABIs must be 'default', 'with hybrid' " + "or a list of strings. Got: %s" % ABIs) + if is_cpython: + ABIs.append('cpython') if class_fixture: @pytest.fixture(params=ABIs) def hpy_abi(self, request): @@ -495,15 +498,6 @@ def supports_ordinary_make_module_imports(self): """ return True - def supports_refcounts(self): - """ Returns True if the underlying Python implementation supports - the vectorcall protocol. - - By default, this returns True for Python version 3.8+ on all - implementations. - """ - return sys.version_info >= (3, 8) - class HPyDebugCapture: """ diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py index a17f10e3a6..7fc483407e 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py @@ -8,7 +8,6 @@ """ from .support import HPyTest from hpy.devel.abitag import HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR -import shutil class TestBasic(HPyTest): @@ -48,6 +47,7 @@ def test_abi_version_check(self): assert False, "Expected exception" def test_abi_tag_check(self): + import shutil if self.compiler.hpy_abi != 'universal': return @@ -262,6 +262,7 @@ def test_builtin_handles(self): case 20: h = ctx->h_MemoryViewType; break; case 21: h = ctx->h_SliceType; break; case 22: h = ctx->h_Builtins; break; + case 23: h = ctx->h_DictType; break; case 2048: h = ctx->h_CapsuleType; break; default: HPyErr_SetString(ctx, ctx->h_ValueError, "invalid choice"); @@ -278,7 +279,7 @@ def test_builtin_handles(self): '', None, False, True, ValueError, TypeError, IndexError, SystemError, object, type, bool, int, float, str, tuple, list, NotImplemented, Ellipsis, complex, bytes, memoryview, slice, - builtins.__dict__ + builtins.__dict__, dict ) for i, obj in enumerate(builtin_objs): if i == 0: @@ -560,3 +561,20 @@ def test_leave_python(self): @INIT """) assert mod.f("abraka") == 3 + + def test_dup_null(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_NOARGS) + static HPy f_impl(HPyContext *ctx, HPy self) + { + HPy h = HPy_Dup(ctx, HPy_NULL); + if (HPy_IsNull(h)) { + return HPyLong_FromSize_t(ctx, 0); + } + HPy_Close(ctx, h); + return HPyLong_FromSize_t(ctx, -1); + } + @EXPORT(f) + @INIT + """) + assert mod.f() == 0 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py index 51aa35c71e..55bdd31c6d 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py @@ -75,7 +75,7 @@ def test_s(self): "function a str is required" ) - def test_B(self): + def test_upper_b(self): mod = self.make_parse_item("B", "char", "char_to_hpybytes") assert mod.f(0) == b"\x00" assert mod.f(1) == b"\x01" @@ -102,7 +102,7 @@ def test_h(self): "function signed short integer is less than minimum" ) - def test_H_short(self): + def test_upper_h_short(self): mod = self.make_parse_item("H", "short", "HPyLong_FromLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -114,7 +114,7 @@ def test_H_short(self): assert mod.f(2**16) == 0 assert mod.f(-2**16) == 0 - def test_H_unsigned_short(self): + def test_upper_h_unsigned_short(self): mod = self.make_parse_item( "H", "unsigned short", "HPyLong_FromUnsignedLong" ) @@ -147,7 +147,7 @@ def test_i(self): "Python int too large to convert to C long", # where sizeof(long) == 4 ) - def test_I_signed(self): + def test_upper_i_signed(self): mod = self.make_parse_item("I", "int", "HPyLong_FromLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -159,7 +159,7 @@ def test_I_signed(self): assert mod.f(2**32) == 0 assert mod.f(-2**32) == 0 - def test_I_unsigned(self): + def test_upper_i_unsigned(self): mod = self.make_parse_item( "I", "unsigned int", "HPyLong_FromUnsignedLong" ) @@ -211,7 +211,7 @@ def test_k_unsigned(self): assert mod.f(2**ULONG_BITS) == 0 assert mod.f(-2**ULONG_BITS) == 0 - def test_L(self): + def test_upper_l(self): import pytest mod = self.make_parse_item("L", "long long", "HPyLong_FromLongLong") assert mod.f(0) == 0 @@ -224,7 +224,7 @@ def test_L(self): with pytest.raises(OverflowError): mod.f(-2**63 - 1) - def test_K_signed(self): + def test_upper_k_signed(self): mod = self.make_parse_item("K", "long long", "HPyLong_FromLongLong") assert mod.f(0) == 0 assert mod.f(1) == 1 @@ -236,7 +236,7 @@ def test_K_signed(self): assert mod.f(2**64) == 0 assert mod.f(-2**64) == 0 - def test_K_unsigned(self): + def test_upper_k_unsigned(self): mod = self.make_parse_item( "K", "unsigned long long", "HPyLong_FromUnsignedLongLong" ) @@ -277,7 +277,7 @@ def test_d(self): with pytest.raises(TypeError): mod.f("x") - def test_O(self): + def test_upper_o(self): mod = self.make_parse_item("O", "HPy", "HPy_Dup") assert mod.f("a") == "a" assert mod.f(5) == 5 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py index 74c7134b34..546b264f01 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py @@ -156,6 +156,7 @@ class TestHPyCapsule(HPyTest): ExtensionTemplate = CapsuleTemplate def test_capsule_new(self): + import pytest mod = self.make_module(""" @DEFINE_SomeObject @DEFINE_Capsule_New @@ -178,6 +179,7 @@ def test_capsule_new(self): mod.capsule_new() def test_capsule_getter_and_setter(self): + import pytest mod = self.make_module(""" #include @@ -385,6 +387,7 @@ def test_capsule_getter_and_setter(self): mod.capsule_freename(p) def test_capsule_isvalid(self): + import pytest mod = self.make_module(""" @DEFINE_SomeObject @DEFINE_Capsule_New @@ -455,6 +458,7 @@ def test_capsule_new_with_destructor(self): assert mod.pointer_freed() def test_capsule_new_with_invalid_destructor(self): + import pytest mod = self.make_module(""" static HPyCapsule_Destructor mydtor = { NULL, NULL }; diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py index ffb89df8e4..441c0ab838 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py @@ -1,9 +1,9 @@ -from textwrap import dedent from .support import HPyTest class TestEval(HPyTest): def test_compile(self): import pytest + from textwrap import dedent mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py index 55e68e5cdc..29ef99dba7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py @@ -326,6 +326,7 @@ def test_tp_finalize(self): // nicely with pytest, but if the finalizer gets called after the // test has finished, we have no choice but to abort to signal // the error + static void on_unexpected_finalize_call(void); static void on_unexpected_finalize_call() { if (test_finished) abort(); @@ -420,4 +421,4 @@ def test_tp_finalize(self): owner.set_a(42) from gc import collect collect() - assert mod.check_finalize_calls() \ No newline at end of file + assert mod.check_finalize_calls() diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py new file mode 100644 index 0000000000..58387253f9 --- /dev/null +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py @@ -0,0 +1,90 @@ +from .support import HPyTest + +class TestIter(HPyTest): + + def test_Check(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + if (HPyIter_Check(ctx, arg)) + return HPy_Dup(ctx, ctx->h_True); + return HPy_Dup(ctx, ctx->h_False); + } + @EXPORT(f) + @INIT + """) + + class CustomIterable: + def __init__(self): + self._iter = iter([1, 2, 3]) + + def __iter__(self): + return self._iter + + class CustomIterator: + def __init__(self): + self._iter = iter([1, 2, 3]) + + def __iter__(self): + return self._iter + + def __next__(self): + return next(self._iter) + + assert mod.f(object()) is False + assert mod.f(10) is False + + assert mod.f((1, 2)) is False + assert mod.f(iter((1, 2))) is True + + assert mod.f([]) is False + assert mod.f(iter([])) is True + + assert mod.f('hello') is False + assert mod.f(iter('hello')) is True + + assert mod.f(map(int, ("1", "2"))) is True + assert mod.f(range(1, 10)) is False + + assert mod.f(CustomIterable()) is False + assert mod.f(iter(CustomIterable())) is True + assert mod.f(CustomIterator()) is True + + def test_Next(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + HPy result = HPyIter_Next(ctx, arg); + int is_null = HPy_IsNull(result); + + if (is_null && HPyErr_Occurred(ctx)) + return HPy_NULL; + if (is_null) + return HPyErr_SetObject(ctx, ctx->h_StopIteration, ctx->h_None); + return result; + } + @EXPORT(f) + @INIT + """) + + class CustomIterator: + def __init__(self): + self._iter = iter(["a", "b", "c"]) + + def __iter__(self): + return self._iter + + def __next__(self): + return next(self._iter) + + assert mod.f(iter([3, 2, 1])) == 3 + assert mod.f((i for i in range(1, 10))) == 1 + assert mod.f(CustomIterator()) == "a" + + import pytest + with pytest.raises(StopIteration): + assert mod.f(iter([])) + + diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py index 9a048ccb5d..42270cdc6d 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py @@ -75,3 +75,38 @@ def test_ListBuilder(self): @INIT """) assert mod.f("xy") == ["xy", True, -42] + + def test_Insert(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy_ssize_t index; + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_ValueError, "expected exactly three arguments"); + return HPy_NULL; + } + index = HPyLong_AsSsize_t(ctx, args[1]); + if (index == -1 && HPyErr_Occurred(ctx)) { + return HPy_NULL; + } + if (HPyList_Insert(ctx, args[0], index, args[2]) == -1) + return HPy_NULL; + return HPy_Dup(ctx, args[0]); + } + @EXPORT(f) + @INIT + """) + l = [] + assert mod.f(l, 0, 0) == [0] + l = [] + assert mod.f(l, -1, 0) == [0] + l = [1, 2, 4] + assert mod.f(l, 0, 0) == [0, 1, 2, 4] + assert mod.f(l, -1, 3) == [0, 1, 2, 3, 4] + assert mod.f(l, -3, 1.5) == [0, 1, 1.5, 2, 3, 4] + assert mod.f(l, 1000, 5) == [0, 1, 1.5, 2, 3, 4, 5] + assert mod.f(l, -1000, -1) == [-1, 0, 1, 1.5, 2, 3, 4, 5] + with pytest.raises(SystemError): + mod.f(None, 0, 0) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py index f154485ffe..5b675c98df 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py @@ -1,5 +1,3 @@ -import types -import pytest from .support import HPyTest class TestModule(HPyTest): @@ -93,6 +91,7 @@ def test_HPyModule_custom_create_returns_non_module(self): runtime and the extension can only populate that module object in the init slots. """ + import types mod = self.make_module(""" HPyDef_SLOT(create, HPy_mod_create) static HPy create_impl(HPyContext *ctx, HPy spec) @@ -104,7 +103,7 @@ def test_HPyModule_custom_create_returns_non_module(self): return HPy_NULL; ns_type = HPy_GetAttr_s(ctx, types, "SimpleNamespace"); - if (HPy_IsNull(types)) + if (HPy_IsNull(ns_type)) goto cleanup; dict = HPyDict_New(ctx); HPy_SetItem_s(ctx, dict, "spec", spec); @@ -144,6 +143,7 @@ def test_HPyModule_error_when_create_returns_module(self): there are any actual use-cases, the purpose of the 'create' slot is to create non-builtin-module objects. """ + import pytest expected_message = "HPy_mod_create slot returned a builtin module " \ "object. This is currently not supported." with pytest.raises(SystemError, match=expected_message): @@ -171,6 +171,7 @@ def test_HPyModule_error_when_create_returns_module(self): """) def test_HPyModule_create_raises(self): + import pytest with pytest.raises(RuntimeError, match="Test error"): self.make_module(""" HPyDef_SLOT(create, HPy_mod_create) @@ -197,6 +198,7 @@ def test_HPyModule_create_raises(self): """) def test_HPyModule_create_and_nondefault_values(self): + import pytest expected_message = r'^HPyModuleDef defines a HPy_mod_create slot.*' with pytest.raises(SystemError, match=expected_message): self.make_module(""" @@ -224,6 +226,7 @@ def test_HPyModule_create_and_nondefault_values(self): """) def test_HPyModule_create_and_exec_slots(self): + import pytest expected_message = r'^HPyModuleDef defines a HPy_mod_create slot.*' with pytest.raises(SystemError, match=expected_message): self.make_module(""" @@ -261,6 +264,7 @@ def test_HPyModule_negative_size(self): """ The simplest fully declarative module creation. """ + import pytest expected_message = "HPy does not permit HPyModuleDef.size < 0" with pytest.raises(SystemError, match=expected_message): self.make_module(""" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py index 8f8126828f..97f45b77b7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py @@ -108,3 +108,31 @@ def test_adjust_indices(self): assert mod.f(10, 9, 0, -3) == (3, 9, 0, -3) assert mod.f(10, -1, -10, -3) == (3, 9, 0, -3) assert mod.f(10, 5, 5, -3) == (0, 5, 5, -3) + + def test_new(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy start, stop, step; + + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_TypeError, + "expected exactly 3 arguments"); + return HPy_NULL; + } + + if (HPyArg_Parse(ctx, NULL, args, nargs, "OOO", + &start, &stop, &step) < 0) { + return HPy_NULL; + } + return HPySlice_New(ctx, start, stop, step); + } + @EXPORT(f) + @INIT + """) + + assert mod.f(0, 10, 1) == slice(0, 10, 1) + assert mod.f(None, 10, 1) == slice(None, 10, 1) + assert mod.f(1, None, 1) == slice(1, None, 1) + assert mod.f(0, 10, None) == slice(0, 10, None) \ No newline at end of file diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py index aa14006b47..3b8469fd16 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py @@ -1102,10 +1102,12 @@ def test_call_invalid_specs(self): @EXPORT(create_vcall) @INIT """) - with pytest.raises(TypeError): + with pytest.raises(TypeError) as err: mod.create_var_type() - with pytest.raises(TypeError): + assert "Cannot use HPy call protocol with var" in str(err.value) + with pytest.raises(TypeError) as err: mod.create_call_and_vectorcalloffset_type() + # assert "Cannot have HPy_tp_call and explicit member" in str(err.value) def test_call_explicit_offset(self): mod = self.make_module(""" @@ -1483,7 +1485,7 @@ def __new__(self): mod.create_type("mytest.DummyIntMeta", int) def test_get_name(self): - import array + import struct mod = self.make_module(""" static HPyType_Spec Dummy_spec = { .name = "mytest.Dummy", @@ -1508,7 +1510,7 @@ def test_get_name(self): assert mod.Dummy.__name__ == "Dummy" assert mod.get_name(mod.Dummy) == "Dummy" assert mod.get_name(str) == "str" - assert mod.get_name(array.array) == "array" + assert mod.get_name(struct.Struct) == "Struct" def test_issubtype(self): mod = self.make_module(""" diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py index 776c1a2c03..fa9710c58f 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py @@ -1,6 +1,7 @@ """ HPyType tests on legacy types. """ import pytest +import sys from .support import HPyTest, make_hpy_abi_fixture from .test_hpytype import PointTemplate, TestType as _TestType @@ -257,6 +258,57 @@ def test_call_zero_basicsize(self): @INIT """) + @pytest.mark.skipif(sys.version_info < (3, 9), reason="not for python<3.9") + def test_legacy_class_method(self): + mod = self.make_module(""" + @DEFINE_PointObject + @DEFINE_Point_xy + static PyObject * point_class_getitem(PyObject *cls, PyObject *args) + { + Py_ssize_t args_len = PyTuple_Check(args) ? PyTuple_Size(args) : 1; + if (args_len != 1) { + return PyErr_Format(PyExc_TypeError, + "Too %s arguments for %s", + args_len > 1 ? "many" : "few", + ((PyTypeObject *)cls)->tp_name); + } + return Py_GenericAlias(cls, args); + } + + static PyMethodDef point_methods[] = { + /* for typing; requires python >= 3.9 */ + {"__class_getitem__", + (PyCFunction)point_class_getitem, + METH_CLASS | METH_O, NULL}, + {NULL, NULL, 0, NULL} /* sentinel */ + }; + + + static PyType_Slot Point_slots[] = { + {Py_tp_methods, point_methods}, + {Py_tp_new, (void*)PyType_GenericNew}, + {0, NULL}, + }; + static HPyDef *Point_defines[] = {&Point_x, &Point_y, NULL}; + static HPyType_Spec Point_spec = { + .name = "mytest.Point", + .basicsize = sizeof(PointObject), + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_slots, + .defines = Point_defines, + }; + + @EXPORT_TYPE("Point", Point_spec) + @INIT + """) + # Calls __class_getitem__ + t = mod.Point[int] + assert str(t) == "mytest.Point[int]" + pt = mod.Point() + assert pt.x == 0 + assert pt.y == 0 + + class TestCustomLegacyFeatures(HPyTest): def test_legacy_methods(self): diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py index 481d3029ca..6f6ce29d16 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py @@ -1,10 +1,6 @@ # -*- encoding: utf-8 -*- -import itertools -import re import sys -import pytest - from .support import HPyTest class TestUnicode(HPyTest): @@ -226,6 +222,9 @@ def test_FromFormat(self, hpy_abi): # Later we generate an HPy function for each case described below: # Most of the test cases are taken from CPython:Modules/_testcapi/unicode.c # Future work can improve this to add tests from Lib/test/test_capi/test_unicode.py + import itertools + import re + import pytest cases = [ # Unrecognized ( "%y%d", (SystemError, "invalid format string"), "'w', 42"), @@ -692,6 +691,7 @@ def check_cpython_raises_any(name): def test_FromFormat_Ptr(self): # '%p' is platform dependent to some extent, so we need to use regex + import re mod = self.make_module(""" HPyDef_METH(p, "p", HPyFunc_NOARGS) static HPy p_impl(HPyContext *ctx, HPy self) @@ -750,6 +750,7 @@ def __repr__(self): assert mod.A(MyObj()) == 'prefix-MyObj.__repr__\\xfc-suffix' def test_FromFormat_NoAsciiEncodedFmt(self): + import pytest mod = self.make_module(""" HPyDef_METH(no_ascii_fmt, "no_ascii_fmt", HPyFunc_O) static HPy no_ascii_fmt_impl(HPyContext *ctx, HPy self, HPy arg) @@ -809,6 +810,7 @@ def test_FromFormat_LongFormat(self): def test_FromFormat_Limits(self): import sys + import pytest mod = self.make_module(""" #include diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py index c903d2faec..a16fa74c74 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py @@ -1,6 +1,5 @@ import pytest from .support import HPyTest -from hpy.devel.abitag import get_hpy_ext_suffix @pytest.fixture(params=['cpython', 'universal', 'hybrid', 'debug']) def hpy_abi(request): @@ -36,6 +35,7 @@ def test_importing_attributes(self, hpy_abi, tmpdir): import pytest if not self.supports_ordinary_make_module_imports(): pytest.skip() + from hpy.devel.abitag import get_hpy_ext_suffix mod = self.make_module(""" @INIT """, name='mytest') diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py index ceeb69413c..6dfe29be51 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py @@ -3,8 +3,6 @@ get the expected compile time errors """ -import sys -import pytest from .support import HPyTest, make_hpy_abi_fixture, ONLY_LINUX # this is not strictly correct, we should check whether the actual compiler diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py index 46e82e9e83..8e50450457 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py @@ -530,6 +530,54 @@ def test_getitem_s(self): with pytest.raises(TypeError): mod.f([]) + def test_getslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + HPy seq; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "Onn", &seq, &i1, &i2)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + return HPy_GetSlice(ctx, seq, i1, i2); + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + assert mod.f(l, 0, 5) == l + assert mod.f(l, 2, 3) == [3] + assert mod.f(l, 1, 3) == [2, 3] + + s = "hello" + assert mod.f(s, 0, 5) == "hello" + assert mod.f(s, 1, 3) == "el" + + class Sliceable: + def __getitem__(self, key): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + return key.start + key.stop + + o = Sliceable() + assert mod.f(o, 5, 13) == 18 + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1) + with pytest.raises(TypeError): + mod.f(123, 0, 1) + def test_setitem(self): import pytest mod = self.make_module(""" @@ -603,6 +651,67 @@ def test_setitem_s(self): with pytest.raises(TypeError): mod.f([]) + def test_setslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + int res; + HPy seq, value; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "OnnO", &seq, &i1, &i2, &value)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + res = HPy_SetSlice(ctx, seq, i1, i2, value); + if (res == 0) { + return HPy_Dup(ctx, seq); + } else if (res == -1) { + return HPy_NULL; + } + HPyErr_SetString(ctx, ctx->h_SystemError, + "HPy_SetSlice returned an invalid result code"); + return HPy_NULL; + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + val = [11,22,33,44,55] + x = mod.f(l, 0, 5, val) + assert x is l + assert x is not val + assert l == val + + l = [1,2,3,4,5] + assert mod.f(l, 0, 5, []) == [] + + l = [1,2,3,4,5] + assert mod.f(l, 1, 3, [10, 11, 12]) == [1, 10, 11, 12, 4, 5] + + class Sliceable: + def __setitem__(self, key, value): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + self.value = (key.start, key.stop, value) + + o = Sliceable() + assert mod.f(o, 5, 13, [7, 8, 9]).value == (5, 13, [7, 8, 9]) + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1, []) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1, []) + with pytest.raises(TypeError): + mod.f(123, 0, 1, []) + def test_delitem(self): import pytest mod = self.make_module(""" @@ -675,6 +784,62 @@ def test_delitem(self): with pytest.raises(TypeError): mod.delitem_s3([1, 2, 3, 4]) + def test_delslice(self): + import pytest + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_VARARGS) + static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) + { + int res; + HPy seq; + HPy_ssize_t i1, i2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "Onn", &seq, &i1, &i2)) { + return HPy_NULL; + } + if (HPy_Is(ctx, seq, ctx->h_None)) { + seq = HPy_NULL; + } + res = HPy_DelSlice(ctx, seq, i1, i2); + if (res == 0) { + return HPy_Dup(ctx, seq); + } else if (res == -1) { + return HPy_NULL; + } + HPyErr_SetString(ctx, ctx->h_SystemError, + "HPy_DelSlice returned an invalid result code"); + return HPy_NULL; + } + @EXPORT(f) + @INIT + """) + l = [1,2,3,4,5] + x = mod.f(l, 0, 5) + assert x is l + assert l == [] + + l = [1,2,3,4,5] + assert mod.f(l, 1, 3) == [1, 4, 5] + + class Sliceable: + def __delitem__(self, key): + # assume 'key' is a slice + if key.start < 0: + raise ValueError + self.value = key.start + key.stop + + o = Sliceable() + assert mod.f(o, 5, 13).value == 18 + + # test that errors are propagated + with pytest.raises(ValueError): + mod.f(o, -1, 1) + + # 'None' will be mapped to 'HPy_NULL' by the C function + with pytest.raises(SystemError): + mod.f(None, 0, 1) + with pytest.raises(TypeError): + mod.f(123, 0, 1) + def test_length(self): mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_O) @@ -740,6 +905,64 @@ class Dummy: import pytest with pytest.raises(TypeError): mod.f(Dummy(), 42) + + def test_getiter(self): + mod = self.make_module(""" + HPyDef_METH(f, "f", HPyFunc_O) + static HPy f_impl(HPyContext *ctx, HPy self, HPy arg) + { + HPy iterator; + iterator = HPy_GetIter(ctx, arg); + if HPy_IsNull(iterator) + return HPy_NULL; + return iterator; + } + @EXPORT(f) + @INIT + """) + + def test_for_loop(iterator): + results = [] + for obj in iterator: + results.append(obj) + return results + + class WithIter: + def __iter__(self): + return (1, 2, 3).__iter__() + + class WithoutIter: + pass + + case = [1, 2, 3] + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + case = iter([1, 2, 3]) + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + case = zip((1, 2, 3), [4, 5, 6]) + result = mod.f(case) + assert result + assert test_for_loop(result) == [(1, 4), (2, 5), (3, 6)] + + case = range(10) + result = mod.f(case) + assert result + assert test_for_loop(result) == [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + case = WithIter() + result = mod.f(case) + assert result + assert test_for_loop(result) == [1, 2, 3] + + import pytest + with pytest.raises(TypeError): + assert mod.f(WithoutIter()) + def test_dump(self): # _HPy_Dump is supposed to be used e.g. inside a gdb session: it diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py index 356fb73b0e..ac9cbe2fc7 100644 --- a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py +++ b/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py @@ -462,6 +462,7 @@ def test_tp_repr_and_tp_str(self): assert repr(p) == 'repr(Point(1, 2))' def test_tp_hash(self): + import pytest mod = self.make_module(""" @DEFINE_PointObject @DEFINE_Point_new @@ -771,7 +772,6 @@ def test_sq_contains(self): 'hello' in p def test_tp_richcompare(self): - import pytest mod = self.make_module(""" @DEFINE_PointObject @DEFINE_Point_new @@ -807,3 +807,27 @@ def test_tp_richcompare(self): # assert not p1 >= p2 assert p1 >= p1 + + def test_tp_descr_get(self): + mod = self.make_module(""" + @DEFINE_PointObject + @DEFINE_Point_new + + HPyDef_SLOT(Point_get, HPy_tp_descr_get); + static HPy + Point_get_impl(HPyContext *ctx, HPy self, HPy obj, HPy type) + { + if (HPy_IsNull(obj) || HPy_Is(ctx, self, ctx->h_None)) { + return HPy_Dup(ctx, self); + } + return HPyLong_FromLong(ctx, 123); + } + + @EXPORT_POINT_TYPE(&Point_new, &Point_get) + @INIT + """) + p = mod.Point(10, 10) + class Dummy: + point_func = p + assert Dummy.point_func is p + assert Dummy().point_func == 123 diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy.h b/graalpython/com.oracle.graal.python.hpy/include/hpy.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyfunc_declare.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyfunc_declare.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h similarity index 98% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h index 0fcf6608a2..117e6ccae0 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/autogen_hpyslot.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h @@ -58,6 +58,7 @@ typedef enum { HPy_sq_length = 45, HPy_sq_repeat = 46, HPy_tp_call = 50, + HPy_tp_descr_get = 54, HPy_tp_hash = 59, HPy_tp_init = 60, HPy_tp_new = 65, @@ -120,6 +121,7 @@ typedef enum { #define _HPySlot_SIG__HPy_sq_length HPyFunc_LENFUNC #define _HPySlot_SIG__HPy_sq_repeat HPyFunc_SSIZEARGFUNC #define _HPySlot_SIG__HPy_tp_call HPyFunc_KEYWORDS +#define _HPySlot_SIG__HPy_tp_descr_get HPyFunc_TERNARYFUNC #define _HPySlot_SIG__HPy_tp_hash HPyFunc_HASHFUNC #define _HPySlot_SIG__HPy_tp_init HPyFunc_INITPROC #define _HPySlot_SIG__HPy_tp_new HPyFunc_NEWFUNC diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpy_types.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpy_types.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h similarity index 92% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h index bde41f9e62..4a55fc6dc6 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_api_impl.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h @@ -240,6 +240,21 @@ HPyAPI_FUNC int HPyCallable_Check(HPyContext *ctx, HPy h) return PyCallable_Check(_h2py(h)); } +HPyAPI_FUNC HPy HPy_GetIter(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_GetIter(_h2py(obj))); +} + +HPyAPI_FUNC HPy HPyIter_Next(HPyContext *ctx, HPy obj) +{ + return _py2h(PyIter_Next(_h2py(obj))); +} + +HPyAPI_FUNC int HPyIter_Check(HPyContext *ctx, HPy obj) +{ + return PyIter_Check(_h2py(obj)); +} + HPyAPI_FUNC HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) { PyErr_SetString(_h2py(h_type), utf8_message); @@ -339,6 +354,11 @@ HPyAPI_FUNC HPy HPy_GetItem(HPyContext *ctx, HPy obj, HPy key) return _py2h(PyObject_GetItem(_h2py(obj), _h2py(key))); } +HPyAPI_FUNC HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PySequence_GetSlice(_h2py(obj), start, end)); +} + HPyAPI_FUNC int HPy_Contains(HPyContext *ctx, HPy container, HPy key) { return PySequence_Contains(_h2py(container), _h2py(key)); @@ -349,14 +369,19 @@ HPyAPI_FUNC int HPy_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value) return PyObject_SetItem(_h2py(obj), _h2py(key), _h2py(value)); } +HPyAPI_FUNC int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + return PySequence_SetSlice(_h2py(obj), start, end, _h2py(value)); +} + HPyAPI_FUNC int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key) { return PyObject_DelItem(_h2py(obj), _h2py(key)); } -HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy obj) +HPyAPI_FUNC int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { - return _py2h(PyObject_Type(_h2py(obj))); + return PySequence_DelSlice(_h2py(obj), start, end); } HPyAPI_FUNC HPy HPy_Repr(HPyContext *ctx, HPy obj) @@ -514,6 +539,11 @@ HPyAPI_FUNC int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item) return PyList_Append(_h2py(h_list), _h2py(h_item)); } +HPyAPI_FUNC int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + return PyList_Insert(_h2py(h_list), index, _h2py(h_item)); +} + HPyAPI_FUNC int HPyDict_Check(HPyContext *ctx, HPy h) { return PyDict_Check(_h2py(h)); @@ -539,6 +569,11 @@ HPyAPI_FUNC int HPyTuple_Check(HPyContext *ctx, HPy h) return PyTuple_Check(_h2py(h)); } +HPyAPI_FUNC HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) +{ + return _py2h(PySlice_New(_h2py(start), _h2py(stop), _h2py(step))); +} + HPyAPI_FUNC int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { return PySlice_Unpack(_h2py(slice), start, stop, step); diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h similarity index 99% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h index 87813ec23b..6f9fb0d7dc 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_ctx.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h @@ -96,4 +96,5 @@ struct _HPyContext_s { HPy h_CapsuleType; HPy h_SliceType; HPy h_Builtins; + HPy h_DictType; }; diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/autogen_hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h similarity index 97% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h index 5665460de7..6df576e7ff 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/cpython/misc.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h @@ -144,6 +144,7 @@ HPyAPI_FUNC HPyContext * _HPyGetContext(void) { ctx->h_MemoryViewType = _py2h((PyObject *)&PyMemoryView_Type); ctx->h_CapsuleType = _py2h((PyObject *)&PyCapsule_Type); ctx->h_SliceType = _py2h((PyObject *)&PySlice_Type); + ctx->h_DictType = _py2h((PyObject *)&PyDict_Type); /* Reflection */ ctx->h_Builtins = _py2h(PyEval_GetBuiltins()); } @@ -271,6 +272,11 @@ HPyAPI_FUNC void* _HPy_AsStruct_List(HPyContext *ctx, HPy h) return ctx_AsStruct_List(ctx, h); } +HPyAPI_FUNC void* _HPy_AsStruct_Dict(HPyContext *ctx, HPy h) +{ + return ctx_AsStruct_Dict(ctx, h); +} + HPyAPI_FUNC HPy HPy_CallTupleDict(HPyContext *ctx, HPy callable, HPy args, HPy kw) { return ctx_CallTupleDict(ctx, callable, args, kw); @@ -305,6 +311,13 @@ HPyAPI_FUNC void _HPy_Dump(HPyContext *ctx, HPy h) ctx_Dump(ctx, h); } +HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy h_obj) +{ + PyTypeObject *tp = Py_TYPE(_h2py(h_obj)); + Py_INCREF(tp); + return _py2h((PyObject *)tp); +} + HPyAPI_FUNC int HPy_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type) { return ctx_TypeCheck(ctx, h_obj, h_type); @@ -312,7 +325,7 @@ HPyAPI_FUNC int HPy_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type) HPyAPI_FUNC int HPy_Is(HPyContext *ctx, HPy h_obj, HPy h_other) { - return ctx_Is(ctx, h_obj, h_other); + return _h2py(h_obj) == _h2py(h_other); } HPyAPI_FUNC HPyListBuilder HPyListBuilder_New(HPyContext *ctx, HPy_ssize_t initial_size) diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/forbid_python_h/Python.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/forbid_python_h/Python.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h similarity index 99% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h index a0478727f1..db45de2e80 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpydef.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h @@ -10,7 +10,7 @@ extern "C" { #include "hpy/autogen_hpyslot.h" #include "hpy/cpy_types.h" -typedef void* (*HPyCFunction)(); +typedef void* (*HPyCFunction)(void); typedef void (*HPyFunc_Capsule_Destructor)(const char *name, void *pointer, void *context); /** diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyexports.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyexports.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyfunc.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpyfunc.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h similarity index 92% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h index 1ad5802aa2..fc6a72f608 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpymodule.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h @@ -129,17 +129,23 @@ typedef struct { */ #define HPy_MODINIT(ext_name, mod_def) \ HPy_EXPORTED_FUNC uint32_t \ - get_required_hpy_major_version_##ext_name() \ + get_required_hpy_major_version_##ext_name(void); \ + uint32_t \ + get_required_hpy_major_version_##ext_name(void) \ { \ return HPY_ABI_VERSION; \ } \ HPy_EXPORTED_FUNC uint32_t \ - get_required_hpy_minor_version_##ext_name() \ + get_required_hpy_minor_version_##ext_name(void); \ + uint32_t \ + get_required_hpy_minor_version_##ext_name(void) \ { \ return HPY_ABI_VERSION_MINOR; \ } \ _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ HPy_EXPORTED_FUNC void \ + HPyInitGlobalContext_##ext_name(HPyContext *ctx); \ + void \ HPyInitGlobalContext_##ext_name(HPyContext *ctx) \ { \ _ctx_for_trampolines = ctx; \ diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h similarity index 94% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h index 7ba2a5640b..167bd6def5 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/hpytype.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h @@ -73,6 +73,12 @@ typedef enum { * need to specify base class ``ctx->h_ListType``. */ HPyType_BuiltinShape_List = 6, + + /** + * The type inherits from built-in type ``dict``. If using this shape, you + * need to specify base class ``ctx->h_DictType``. + */ + HPyType_BuiltinShape_Dict = 7, } HPyType_BuiltinShape; typedef struct { @@ -202,7 +208,16 @@ typedef struct { #define HPy_TPFLAGS_HAVE_GC (1UL << 14) /** Convenience macro which is equivalent to: - ``HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy)`` */ + ``HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy)`` + For instance, HPyType_LEGACY_HELPERS(DummyMeta) will produce:: + + enum { DummyMeta_SHAPE = (int)HPyType_BuiltinShape_Legacy }; + __attribute__((unused)) static inline + DummyMeta * + DummyMeta_AsStruct(HPyContext *ctx, HPy h) { + return (DummyMeta *) _HPy_AsStruct_Legacy(ctx, h); + } +*/ #define HPyType_LEGACY_HELPERS(TYPE) \ HPyType_HELPERS(TYPE, HPyType_BuiltinShape_Legacy) @@ -283,5 +298,6 @@ _HPyType_HELPER_X(_HPyType_HELPER_FNAME(__VA_ARGS__))(HPyContext *ctx, HPy h) \ #define HPyType_BuiltinShape_Unicode_AsStruct _HPy_AsStruct_Unicode #define HPyType_BuiltinShape_Tuple_AsStruct _HPy_AsStruct_Tuple #define HPyType_BuiltinShape_List_AsStruct _HPy_AsStruct_List +#define HPyType_BuiltinShape_Dict_AsStruct _HPy_AsStruct_Dict #endif /* HPY_UNIVERSAL_HPYTYPE_H */ diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/inline_helpers.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/inline_helpers.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/macros.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/macros.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/argparse.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/argparse.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/buildvalue.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/buildvalue.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h similarity index 96% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h index 4fc1f319fc..d32aa75592 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_funcs.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h @@ -31,6 +31,7 @@ _HPy_HIDDEN HPy ctx_Module_Create(HPyContext *ctx, HPyModuleDef *hpydef); // ctx_object.c _HPy_HIDDEN void ctx_Dump(HPyContext *ctx, HPy h); +_HPy_HIDDEN HPy ctx_Type(HPyContext *ctx, HPy h_obj); _HPy_HIDDEN int ctx_TypeCheck(HPyContext *ctx, HPy h_obj, HPy h_type); _HPy_HIDDEN int ctx_Is(HPyContext *ctx, HPy h_obj, HPy h_other); _HPy_HIDDEN HPy ctx_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); @@ -56,7 +57,7 @@ _HPy_HIDDEN void ctx_TupleBuilder_Cancel(HPyContext *ctx, HPyTupleBuilder builder); // ctx_tuple.c -_HPy_HIDDEN HPy ctx_Tuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n); +_HPy_HIDDEN HPy ctx_Tuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n); // ctx_capsule.c _HPy_HIDDEN HPy ctx_Capsule_New(HPyContext *ctx, @@ -87,6 +88,7 @@ _HPy_HIDDEN void* ctx_AsStruct_Float(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Unicode(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Tuple(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_List(HPyContext *ctx, HPy h); +_HPy_HIDDEN void* ctx_AsStruct_Dict(HPyContext *ctx, HPy h); _HPy_HIDDEN void* ctx_AsStruct_Slow(HPyContext *ctx, HPy h); _HPy_HIDDEN HPy ctx_Type_FromSpec(HPyContext *ctx, HPyType_Spec *hpyspec, HPyType_SpecParam *params); diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_module.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_module.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_type.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/ctx_type.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/format.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/format.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/helpers.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/helpers.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/structseq.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/runtime/structseq.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h similarity index 95% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h index 6133fc633f..1abb8e5d1c 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_ctx.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h @@ -218,7 +218,7 @@ struct _HPyContext_s { int (*ctx_Dict_Check)(HPyContext *ctx, HPy h); HPy (*ctx_Dict_New)(HPyContext *ctx); int (*ctx_Tuple_Check)(HPyContext *ctx, HPy h); - HPy (*ctx_Tuple_FromArray)(HPyContext *ctx, HPy items[], HPy_ssize_t n); + HPy (*ctx_Tuple_FromArray)(HPyContext *ctx, const HPy items[], HPy_ssize_t n); HPy (*ctx_Import_ImportModule)(HPyContext *ctx, const char *utf8_name); HPy (*ctx_FromPyObject)(HPyContext *ctx, cpy_PyObject *obj); cpy_PyObject *(*ctx_AsPyObject)(HPyContext *ctx, HPy h); @@ -277,4 +277,14 @@ struct _HPyContext_s { int (*ctx_SetCallFunction)(HPyContext *ctx, HPy h, HPyCallFunction *func); HPy (*ctx_Call)(HPyContext *ctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); HPy (*ctx_CallMethod)(HPyContext *ctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); + HPy h_DictType; + void *(*ctx_AsStruct_Dict)(HPyContext *ctx, HPy h); + int (*ctx_List_Insert)(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item); + HPy (*ctx_GetSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + int (*ctx_SetSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); + int (*ctx_DelSlice)(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + HPy (*ctx_GetIter)(HPyContext *ctx, HPy obj); + HPy (*ctx_Iter_Next)(HPyContext *ctx, HPy obj); + int (*ctx_Iter_Check)(HPyContext *ctx, HPy obj); + HPy (*ctx_Slice_New)(HPyContext *ctx, HPy start, HPy stop, HPy step); }; diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h similarity index 94% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h index 0e57d3f1be..4579b5591b 100644 --- a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/autogen_trampolines.h +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h @@ -254,6 +254,18 @@ HPyAPI_FUNC HPy HPy_CallMethod(HPyContext *ctx, HPy name, const HPy *args, size_ return ctx->ctx_CallMethod ( ctx, name, args, nargs, kwnames ); } +HPyAPI_FUNC HPy HPy_GetIter(HPyContext *ctx, HPy obj) { + return ctx->ctx_GetIter ( ctx, obj ); +} + +HPyAPI_FUNC HPy HPyIter_Next(HPyContext *ctx, HPy obj) { + return ctx->ctx_Iter_Next ( ctx, obj ); +} + +HPyAPI_FUNC int HPyIter_Check(HPyContext *ctx, HPy obj) { + return ctx->ctx_Iter_Check ( ctx, obj ); +} + HPyAPI_FUNC HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) { ctx->ctx_Err_SetString ( ctx, h_type, utf8_message ); return HPy_NULL; } @@ -350,6 +362,10 @@ HPyAPI_FUNC HPy HPy_GetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key) { return ctx->ctx_GetItem_s ( ctx, obj, utf8_key ); } +HPyAPI_FUNC HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { + return ctx->ctx_GetSlice ( ctx, obj, start, end ); +} + HPyAPI_FUNC int HPy_Contains(HPyContext *ctx, HPy container, HPy key) { return ctx->ctx_Contains ( ctx, container, key ); } @@ -366,6 +382,10 @@ HPyAPI_FUNC int HPy_SetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key, HP return ctx->ctx_SetItem_s ( ctx, obj, utf8_key, value ); } +HPyAPI_FUNC int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) { + return ctx->ctx_SetSlice ( ctx, obj, start, end, value ); +} + HPyAPI_FUNC int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key) { return ctx->ctx_DelItem ( ctx, obj, key ); } @@ -378,6 +398,10 @@ HPyAPI_FUNC int HPy_DelItem_s(HPyContext *ctx, HPy obj, const char *utf8_key) { return ctx->ctx_DelItem_s ( ctx, obj, utf8_key ); } +HPyAPI_FUNC int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) { + return ctx->ctx_DelSlice ( ctx, obj, start, end ); +} + HPyAPI_FUNC HPy HPy_Type(HPyContext *ctx, HPy obj) { return ctx->ctx_Type ( ctx, obj ); } @@ -430,6 +454,10 @@ HPyAPI_FUNC void *_HPy_AsStruct_List(HPyContext *ctx, HPy h) { return ctx->ctx_AsStruct_List ( ctx, h ); } +HPyAPI_FUNC void *_HPy_AsStruct_Dict(HPyContext *ctx, HPy h) { + return ctx->ctx_AsStruct_Dict ( ctx, h ); +} + HPyAPI_FUNC HPyType_BuiltinShape _HPyType_GetBuiltinShape(HPyContext *ctx, HPy h_type) { return ctx->ctx_Type_GetBuiltinShape ( ctx, h_type ); } @@ -562,6 +590,10 @@ HPyAPI_FUNC int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item) { return ctx->ctx_List_Append ( ctx, h_list, h_item ); } +HPyAPI_FUNC int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) { + return ctx->ctx_List_Insert ( ctx, h_list, index, h_item ); +} + HPyAPI_FUNC int HPyDict_Check(HPyContext *ctx, HPy h) { return ctx->ctx_Dict_Check ( ctx, h ); } @@ -582,10 +614,14 @@ HPyAPI_FUNC int HPyTuple_Check(HPyContext *ctx, HPy h) { return ctx->ctx_Tuple_Check ( ctx, h ); } -HPyAPI_FUNC HPy HPyTuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n) { +HPyAPI_FUNC HPy HPyTuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n) { return ctx->ctx_Tuple_FromArray ( ctx, items, n ); } +HPyAPI_FUNC HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) { + return ctx->ctx_Slice_New ( ctx, start, stop, step ); +} + HPyAPI_FUNC int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { return ctx->ctx_Slice_Unpack ( ctx, slice, start, stop, step ); } diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/hpyfunc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/hpyfunc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/misc_trampolines.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.llvm/include/hpy/universal/misc_trampolines.h rename to graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h new file mode 100644 index 0000000000..81d60ed7d5 --- /dev/null +++ b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h @@ -0,0 +1,4 @@ + +// automatically generated by setup.py:get_scm_config() +#define HPY_VERSION "0.9.1.dev79+gb0fbdf73" +#define HPY_GIT_REVISION "b0fbdf73" diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c b/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c index ce3a57e886..2bd71f3262 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c @@ -10,7 +10,7 @@ static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, DebugHandle *handle); -HPY_MOD_EMBEDDABLE(_trace) +HPY_MOD_EMBEDDABLE(_debug) HPyDef_METH(new_generation, "new_generation", HPyFunc_NOARGS) static UHPy new_generation_impl(HPyContext *uctx, UHPy self) diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h index 6c242cfe48..c394c362ce 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h +++ b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h @@ -71,6 +71,9 @@ int debug_ctx_Callable_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_CallTupleDict(HPyContext *dctx, DHPy callable, DHPy args, DHPy kw); DHPy debug_ctx_Call(HPyContext *dctx, DHPy callable, const DHPy *args, size_t nargs, DHPy kwnames); DHPy debug_ctx_CallMethod(HPyContext *dctx, DHPy name, const DHPy *args, size_t nargs, DHPy kwnames); +DHPy debug_ctx_GetIter(HPyContext *dctx, DHPy obj); +DHPy debug_ctx_Iter_Next(HPyContext *dctx, DHPy obj); +int debug_ctx_Iter_Check(HPyContext *dctx, DHPy obj); void debug_ctx_FatalError(HPyContext *dctx, const char *message); void debug_ctx_Err_SetString(HPyContext *dctx, DHPy h_type, const char *utf8_message); void debug_ctx_Err_SetObject(HPyContext *dctx, DHPy h_type, DHPy h_value); @@ -96,13 +99,16 @@ int debug_ctx_SetAttr_s(HPyContext *dctx, DHPy obj, const char *utf8_name, DHPy DHPy debug_ctx_GetItem(HPyContext *dctx, DHPy obj, DHPy key); DHPy debug_ctx_GetItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx); DHPy debug_ctx_GetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key); +DHPy debug_ctx_GetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end); int debug_ctx_Contains(HPyContext *dctx, DHPy container, DHPy key); int debug_ctx_SetItem(HPyContext *dctx, DHPy obj, DHPy key, DHPy value); int debug_ctx_SetItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx, DHPy value); int debug_ctx_SetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key, DHPy value); +int debug_ctx_SetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end, DHPy value); int debug_ctx_DelItem(HPyContext *dctx, DHPy obj, DHPy key); int debug_ctx_DelItem_i(HPyContext *dctx, DHPy obj, HPy_ssize_t idx); int debug_ctx_DelItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key); +int debug_ctx_DelSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end); DHPy debug_ctx_Type(HPyContext *dctx, DHPy obj); int debug_ctx_TypeCheck(HPyContext *dctx, DHPy obj, DHPy type); const char *debug_ctx_Type_GetName(HPyContext *dctx, DHPy type); @@ -116,6 +122,7 @@ void *debug_ctx_AsStruct_Float(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_Unicode(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_Tuple(HPyContext *dctx, DHPy h); void *debug_ctx_AsStruct_List(HPyContext *dctx, DHPy h); +void *debug_ctx_AsStruct_Dict(HPyContext *dctx, DHPy h); HPyType_BuiltinShape debug_ctx_Type_GetBuiltinShape(HPyContext *dctx, DHPy h_type); DHPy debug_ctx_New(HPyContext *dctx, DHPy h_type, void **data); DHPy debug_ctx_Repr(HPyContext *dctx, DHPy obj); @@ -150,12 +157,14 @@ DHPy debug_ctx_Unicode_Substring(HPyContext *dctx, DHPy str, HPy_ssize_t start, int debug_ctx_List_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_List_New(HPyContext *dctx, HPy_ssize_t len); int debug_ctx_List_Append(HPyContext *dctx, DHPy h_list, DHPy h_item); +int debug_ctx_List_Insert(HPyContext *dctx, DHPy h_list, HPy_ssize_t index, DHPy h_item); int debug_ctx_Dict_Check(HPyContext *dctx, DHPy h); DHPy debug_ctx_Dict_New(HPyContext *dctx); DHPy debug_ctx_Dict_Keys(HPyContext *dctx, DHPy h); DHPy debug_ctx_Dict_Copy(HPyContext *dctx, DHPy h); int debug_ctx_Tuple_Check(HPyContext *dctx, DHPy h); -DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, DHPy items[], HPy_ssize_t n); +DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, const DHPy items[], HPy_ssize_t n); +DHPy debug_ctx_Slice_New(HPyContext *dctx, DHPy start, DHPy stop, DHPy step); int debug_ctx_Slice_Unpack(HPyContext *dctx, DHPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); DHPy debug_ctx_Import_ImportModule(HPyContext *dctx, const char *utf8_name); DHPy debug_ctx_Capsule_New(HPyContext *dctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); @@ -275,6 +284,7 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->h_MemoryViewType = DHPy_open_immortal(dctx, uctx->h_MemoryViewType); dctx->h_CapsuleType = DHPy_open_immortal(dctx, uctx->h_CapsuleType); dctx->h_SliceType = DHPy_open_immortal(dctx, uctx->h_SliceType); + dctx->h_DictType = DHPy_open_immortal(dctx, uctx->h_DictType); dctx->h_Builtins = DHPy_open_immortal(dctx, uctx->h_Builtins); dctx->ctx_Dup = &debug_ctx_Dup; dctx->ctx_Close = &debug_ctx_Close; @@ -337,6 +347,9 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_CallTupleDict = &debug_ctx_CallTupleDict; dctx->ctx_Call = &debug_ctx_Call; dctx->ctx_CallMethod = &debug_ctx_CallMethod; + dctx->ctx_GetIter = &debug_ctx_GetIter; + dctx->ctx_Iter_Next = &debug_ctx_Iter_Next; + dctx->ctx_Iter_Check = &debug_ctx_Iter_Check; dctx->ctx_FatalError = &debug_ctx_FatalError; dctx->ctx_Err_SetString = &debug_ctx_Err_SetString; dctx->ctx_Err_SetObject = &debug_ctx_Err_SetObject; @@ -362,13 +375,16 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_GetItem = &debug_ctx_GetItem; dctx->ctx_GetItem_i = &debug_ctx_GetItem_i; dctx->ctx_GetItem_s = &debug_ctx_GetItem_s; + dctx->ctx_GetSlice = &debug_ctx_GetSlice; dctx->ctx_Contains = &debug_ctx_Contains; dctx->ctx_SetItem = &debug_ctx_SetItem; dctx->ctx_SetItem_i = &debug_ctx_SetItem_i; dctx->ctx_SetItem_s = &debug_ctx_SetItem_s; + dctx->ctx_SetSlice = &debug_ctx_SetSlice; dctx->ctx_DelItem = &debug_ctx_DelItem; dctx->ctx_DelItem_i = &debug_ctx_DelItem_i; dctx->ctx_DelItem_s = &debug_ctx_DelItem_s; + dctx->ctx_DelSlice = &debug_ctx_DelSlice; dctx->ctx_Type = &debug_ctx_Type; dctx->ctx_TypeCheck = &debug_ctx_TypeCheck; dctx->ctx_Type_GetName = &debug_ctx_Type_GetName; @@ -382,6 +398,7 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_AsStruct_Unicode = &debug_ctx_AsStruct_Unicode; dctx->ctx_AsStruct_Tuple = &debug_ctx_AsStruct_Tuple; dctx->ctx_AsStruct_List = &debug_ctx_AsStruct_List; + dctx->ctx_AsStruct_Dict = &debug_ctx_AsStruct_Dict; dctx->ctx_Type_GetBuiltinShape = &debug_ctx_Type_GetBuiltinShape; dctx->ctx_New = &debug_ctx_New; dctx->ctx_Repr = &debug_ctx_Repr; @@ -416,12 +433,14 @@ static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx) dctx->ctx_List_Check = &debug_ctx_List_Check; dctx->ctx_List_New = &debug_ctx_List_New; dctx->ctx_List_Append = &debug_ctx_List_Append; + dctx->ctx_List_Insert = &debug_ctx_List_Insert; dctx->ctx_Dict_Check = &debug_ctx_Dict_Check; dctx->ctx_Dict_New = &debug_ctx_Dict_New; dctx->ctx_Dict_Keys = &debug_ctx_Dict_Keys; dctx->ctx_Dict_Copy = &debug_ctx_Dict_Copy; dctx->ctx_Tuple_Check = &debug_ctx_Tuple_Check; dctx->ctx_Tuple_FromArray = &debug_ctx_Tuple_FromArray; + dctx->ctx_Slice_New = &debug_ctx_Slice_New; dctx->ctx_Slice_Unpack = &debug_ctx_Slice_Unpack; dctx->ctx_Import_ImportModule = &debug_ctx_Import_ImportModule; dctx->ctx_Capsule_New = &debug_ctx_Capsule_New; diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c index 59663daecc..e9eb6a15c3 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c @@ -731,6 +731,42 @@ DHPy debug_ctx_CallTupleDict(HPyContext *dctx, DHPy callable, DHPy args, DHPy kw return DHPy_open(dctx, universal_result); } +DHPy debug_ctx_GetIter(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPy_GetIter(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + +DHPy debug_ctx_Iter_Next(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPyIter_Next(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + +int debug_ctx_Iter_Check(HPyContext *dctx, DHPy obj) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPyIter_Check(get_info(dctx)->uctx, dh_obj); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + void debug_ctx_FatalError(HPyContext *dctx, const char *message) { if (!get_ctx_info(dctx)->is_valid) { @@ -1007,6 +1043,18 @@ DHPy debug_ctx_GetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key) return DHPy_open(dctx, universal_result); } +DHPy debug_ctx_GetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPy_GetSlice(get_info(dctx)->uctx, dh_obj, start, end); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + int debug_ctx_Contains(HPyContext *dctx, DHPy container, DHPy key) { if (!get_ctx_info(dctx)->is_valid) { @@ -1060,6 +1108,19 @@ int debug_ctx_SetItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key, DHPy v return universal_result; } +int debug_ctx_SetSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end, DHPy value) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + HPy dh_value = DHPy_unwrap(dctx, value); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPy_SetSlice(get_info(dctx)->uctx, dh_obj, start, end, dh_value); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + int debug_ctx_DelItem(HPyContext *dctx, DHPy obj, DHPy key) { if (!get_ctx_info(dctx)->is_valid) { @@ -1097,6 +1158,18 @@ int debug_ctx_DelItem_s(HPyContext *dctx, DHPy obj, const char *utf8_key) return universal_result; } +int debug_ctx_DelSlice(HPyContext *dctx, DHPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_obj = DHPy_unwrap(dctx, obj); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPy_DelSlice(get_info(dctx)->uctx, dh_obj, start, end); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + DHPy debug_ctx_Type(HPyContext *dctx, DHPy obj) { if (!get_ctx_info(dctx)->is_valid) { @@ -1476,6 +1549,19 @@ int debug_ctx_List_Append(HPyContext *dctx, DHPy h_list, DHPy h_item) return universal_result; } +int debug_ctx_List_Insert(HPyContext *dctx, DHPy h_list, HPy_ssize_t index, DHPy h_item) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_h_list = DHPy_unwrap(dctx, h_list); + HPy dh_h_item = DHPy_unwrap(dctx, h_item); + get_ctx_info(dctx)->is_valid = false; + int universal_result = HPyList_Insert(get_info(dctx)->uctx, dh_h_list, index, dh_h_item); + get_ctx_info(dctx)->is_valid = true; + return universal_result; +} + int debug_ctx_Dict_Check(HPyContext *dctx, DHPy h) { if (!get_ctx_info(dctx)->is_valid) { @@ -1535,6 +1621,20 @@ int debug_ctx_Tuple_Check(HPyContext *dctx, DHPy h) return universal_result; } +DHPy debug_ctx_Slice_New(HPyContext *dctx, DHPy start, DHPy stop, DHPy step) +{ + if (!get_ctx_info(dctx)->is_valid) { + report_invalid_debug_context(); + } + HPy dh_start = DHPy_unwrap(dctx, start); + HPy dh_stop = DHPy_unwrap(dctx, stop); + HPy dh_step = DHPy_unwrap(dctx, step); + get_ctx_info(dctx)->is_valid = false; + HPy universal_result = HPySlice_New(get_info(dctx)->uctx, dh_start, dh_stop, dh_step); + get_ctx_info(dctx)->is_valid = true; + return DHPy_open(dctx, universal_result); +} + int debug_ctx_Slice_Unpack(HPyContext *dctx, DHPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { if (!get_ctx_info(dctx)->is_valid) { diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c b/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c index 1f021daa5c..de661ea2c9 100644 --- a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c +++ b/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c @@ -256,7 +256,7 @@ const char *debug_ctx_Bytes_AS_STRING(HPyContext *dctx, DHPy h) return (const char *)protect_and_associate_data_ptr(h, (void *)ptr, data_size); } -DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, DHPy dh_items[], HPy_ssize_t n) +DHPy debug_ctx_Tuple_FromArray(HPyContext *dctx, const DHPy dh_items[], HPy_ssize_t n) { if (!get_ctx_info(dctx)->is_valid) { report_invalid_debug_context(); @@ -328,6 +328,7 @@ static const char *get_builtin_shape_name(HPyType_BuiltinShape shape) SHAPE_NAME(HPyType_BuiltinShape_Unicode) SHAPE_NAME(HPyType_BuiltinShape_Tuple) SHAPE_NAME(HPyType_BuiltinShape_List) + SHAPE_NAME(HPyType_BuiltinShape_Dict) } return ""; #undef SHAPE_NAME @@ -372,6 +373,8 @@ MAKE_debug_ctx_AsStruct(Tuple) MAKE_debug_ctx_AsStruct(List) +MAKE_debug_ctx_AsStruct(Dict) + /* ~~~ debug mode implementation of HPyTracker ~~~ This is a bit special and it's worth explaining what is going on. diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h index 4e452b7e28..fef5028491 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h @@ -71,6 +71,9 @@ int trace_ctx_Callable_Check(HPyContext *tctx, HPy h); HPy trace_ctx_CallTupleDict(HPyContext *tctx, HPy callable, HPy args, HPy kw); HPy trace_ctx_Call(HPyContext *tctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); HPy trace_ctx_CallMethod(HPyContext *tctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); +HPy trace_ctx_GetIter(HPyContext *tctx, HPy obj); +HPy trace_ctx_Iter_Next(HPyContext *tctx, HPy obj); +int trace_ctx_Iter_Check(HPyContext *tctx, HPy obj); void trace_ctx_Err_SetString(HPyContext *tctx, HPy h_type, const char *utf8_message); void trace_ctx_Err_SetObject(HPyContext *tctx, HPy h_type, HPy h_value); HPy trace_ctx_Err_SetFromErrnoWithFilename(HPyContext *tctx, HPy h_type, const char *filename_fsencoded); @@ -95,13 +98,16 @@ int trace_ctx_SetAttr_s(HPyContext *tctx, HPy obj, const char *utf8_name, HPy va HPy trace_ctx_GetItem(HPyContext *tctx, HPy obj, HPy key); HPy trace_ctx_GetItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx); HPy trace_ctx_GetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key); +HPy trace_ctx_GetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); int trace_ctx_Contains(HPyContext *tctx, HPy container, HPy key); int trace_ctx_SetItem(HPyContext *tctx, HPy obj, HPy key, HPy value); int trace_ctx_SetItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx, HPy value); int trace_ctx_SetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key, HPy value); +int trace_ctx_SetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); int trace_ctx_DelItem(HPyContext *tctx, HPy obj, HPy key); int trace_ctx_DelItem_i(HPyContext *tctx, HPy obj, HPy_ssize_t idx); int trace_ctx_DelItem_s(HPyContext *tctx, HPy obj, const char *utf8_key); +int trace_ctx_DelSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); HPy trace_ctx_Type(HPyContext *tctx, HPy obj); int trace_ctx_TypeCheck(HPyContext *tctx, HPy obj, HPy type); const char *trace_ctx_Type_GetName(HPyContext *tctx, HPy type); @@ -115,6 +121,7 @@ void *trace_ctx_AsStruct_Float(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_Unicode(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_Tuple(HPyContext *tctx, HPy h); void *trace_ctx_AsStruct_List(HPyContext *tctx, HPy h); +void *trace_ctx_AsStruct_Dict(HPyContext *tctx, HPy h); HPyType_BuiltinShape trace_ctx_Type_GetBuiltinShape(HPyContext *tctx, HPy h_type); HPy trace_ctx_New(HPyContext *tctx, HPy h_type, void **data); HPy trace_ctx_Repr(HPyContext *tctx, HPy obj); @@ -149,12 +156,14 @@ HPy trace_ctx_Unicode_Substring(HPyContext *tctx, HPy str, HPy_ssize_t start, HP int trace_ctx_List_Check(HPyContext *tctx, HPy h); HPy trace_ctx_List_New(HPyContext *tctx, HPy_ssize_t len); int trace_ctx_List_Append(HPyContext *tctx, HPy h_list, HPy h_item); +int trace_ctx_List_Insert(HPyContext *tctx, HPy h_list, HPy_ssize_t index, HPy h_item); int trace_ctx_Dict_Check(HPyContext *tctx, HPy h); HPy trace_ctx_Dict_New(HPyContext *tctx); HPy trace_ctx_Dict_Keys(HPyContext *tctx, HPy h); HPy trace_ctx_Dict_Copy(HPyContext *tctx, HPy h); int trace_ctx_Tuple_Check(HPyContext *tctx, HPy h); -HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n); +HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, const HPy items[], HPy_ssize_t n); +HPy trace_ctx_Slice_New(HPyContext *tctx, HPy start, HPy stop, HPy step); int trace_ctx_Slice_Unpack(HPyContext *tctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); HPy trace_ctx_Import_ImportModule(HPyContext *tctx, const char *utf8_name); HPy trace_ctx_Capsule_New(HPyContext *tctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); @@ -193,8 +202,8 @@ static inline void trace_ctx_init_info(HPyTraceInfo *info, HPyContext *uctx) { info->magic_number = HPY_TRACE_MAGIC; info->uctx = uctx; - info->call_counts = (uint64_t *)calloc(263, sizeof(uint64_t)); - info->durations = (_HPyTime_t *)calloc(263, sizeof(_HPyTime_t)); + info->call_counts = (uint64_t *)calloc(273, sizeof(uint64_t)); + info->durations = (_HPyTime_t *)calloc(273, sizeof(_HPyTime_t)); info->on_enter_func = HPy_NULL; info->on_exit_func = HPy_NULL; } @@ -292,6 +301,7 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->h_MemoryViewType = uctx->h_MemoryViewType; tctx->h_CapsuleType = uctx->h_CapsuleType; tctx->h_SliceType = uctx->h_SliceType; + tctx->h_DictType = uctx->h_DictType; tctx->h_Builtins = uctx->h_Builtins; tctx->ctx_Dup = &trace_ctx_Dup; tctx->ctx_Close = &trace_ctx_Close; @@ -354,6 +364,9 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_CallTupleDict = &trace_ctx_CallTupleDict; tctx->ctx_Call = &trace_ctx_Call; tctx->ctx_CallMethod = &trace_ctx_CallMethod; + tctx->ctx_GetIter = &trace_ctx_GetIter; + tctx->ctx_Iter_Next = &trace_ctx_Iter_Next; + tctx->ctx_Iter_Check = &trace_ctx_Iter_Check; tctx->ctx_FatalError = uctx->ctx_FatalError; tctx->ctx_Err_SetString = &trace_ctx_Err_SetString; tctx->ctx_Err_SetObject = &trace_ctx_Err_SetObject; @@ -379,13 +392,16 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_GetItem = &trace_ctx_GetItem; tctx->ctx_GetItem_i = &trace_ctx_GetItem_i; tctx->ctx_GetItem_s = &trace_ctx_GetItem_s; + tctx->ctx_GetSlice = &trace_ctx_GetSlice; tctx->ctx_Contains = &trace_ctx_Contains; tctx->ctx_SetItem = &trace_ctx_SetItem; tctx->ctx_SetItem_i = &trace_ctx_SetItem_i; tctx->ctx_SetItem_s = &trace_ctx_SetItem_s; + tctx->ctx_SetSlice = &trace_ctx_SetSlice; tctx->ctx_DelItem = &trace_ctx_DelItem; tctx->ctx_DelItem_i = &trace_ctx_DelItem_i; tctx->ctx_DelItem_s = &trace_ctx_DelItem_s; + tctx->ctx_DelSlice = &trace_ctx_DelSlice; tctx->ctx_Type = &trace_ctx_Type; tctx->ctx_TypeCheck = &trace_ctx_TypeCheck; tctx->ctx_Type_GetName = &trace_ctx_Type_GetName; @@ -399,6 +415,7 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_AsStruct_Unicode = &trace_ctx_AsStruct_Unicode; tctx->ctx_AsStruct_Tuple = &trace_ctx_AsStruct_Tuple; tctx->ctx_AsStruct_List = &trace_ctx_AsStruct_List; + tctx->ctx_AsStruct_Dict = &trace_ctx_AsStruct_Dict; tctx->ctx_Type_GetBuiltinShape = &trace_ctx_Type_GetBuiltinShape; tctx->ctx_New = &trace_ctx_New; tctx->ctx_Repr = &trace_ctx_Repr; @@ -433,12 +450,14 @@ static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx) tctx->ctx_List_Check = &trace_ctx_List_Check; tctx->ctx_List_New = &trace_ctx_List_New; tctx->ctx_List_Append = &trace_ctx_List_Append; + tctx->ctx_List_Insert = &trace_ctx_List_Insert; tctx->ctx_Dict_Check = &trace_ctx_Dict_Check; tctx->ctx_Dict_New = &trace_ctx_Dict_New; tctx->ctx_Dict_Keys = &trace_ctx_Dict_Keys; tctx->ctx_Dict_Copy = &trace_ctx_Dict_Copy; tctx->ctx_Tuple_Check = &trace_ctx_Tuple_Check; tctx->ctx_Tuple_FromArray = &trace_ctx_Tuple_FromArray; + tctx->ctx_Slice_New = &trace_ctx_Slice_New; tctx->ctx_Slice_Unpack = &trace_ctx_Slice_Unpack; tctx->ctx_Import_ImportModule = &trace_ctx_Import_ImportModule; tctx->ctx_Capsule_New = &trace_ctx_Capsule_New; diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c index 7a1a16332f..97d8308d53 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c @@ -12,7 +12,7 @@ #include "trace_internal.h" -#define TRACE_NFUNC 180 +#define TRACE_NFUNC 189 #define NO_FUNC "" static const char *trace_func_table[] = { @@ -279,6 +279,16 @@ static const char *trace_func_table[] = { "ctx_SetCallFunction", "ctx_Call", "ctx_CallMethod", + NO_FUNC, + "ctx_AsStruct_Dict", + "ctx_List_Insert", + "ctx_GetSlice", + "ctx_SetSlice", + "ctx_DelSlice", + "ctx_GetIter", + "ctx_Iter_Next", + "ctx_Iter_Check", + "ctx_Slice_New", NULL /* sentinel */ }; @@ -289,7 +299,7 @@ int hpy_trace_get_nfunc(void) const char * hpy_trace_get_func_name(int idx) { - if (idx >= 0 && idx < 263) + if (idx >= 0 && idx < 273) return trace_func_table[idx]; return NULL; } diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c index cd8a4b1308..4940a12a9d 100644 --- a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c +++ b/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c @@ -804,6 +804,45 @@ HPy trace_ctx_CallMethod(HPyContext *tctx, HPy name, const HPy *args, size_t nar return res; } +HPy trace_ctx_GetIter(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 269); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPy_GetIter(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 269, r0, r1, &_ts_start, &_ts_end); + return res; +} + +HPy trace_ctx_Iter_Next(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 270); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPyIter_Next(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 270, r0, r1, &_ts_start, &_ts_end); + return res; +} + +int trace_ctx_Iter_Check(HPyContext *tctx, HPy obj) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 271); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPyIter_Check(uctx, obj); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 271, r0, r1, &_ts_start, &_ts_end); + return res; +} + void trace_ctx_Err_SetString(HPyContext *tctx, HPy h_type, const char *utf8_message) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 137); @@ -1110,6 +1149,19 @@ HPy trace_ctx_GetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key) return res; } +HPy trace_ctx_GetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 266); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPy_GetSlice(uctx, obj, start, end); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 266, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Contains(HPyContext *tctx, HPy container, HPy key) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 161); @@ -1162,6 +1214,19 @@ int trace_ctx_SetItem_s(HPyContext *tctx, HPy obj, const char *utf8_key, HPy val return res; } +int trace_ctx_SetSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 267); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPy_SetSlice(uctx, obj, start, end, value); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 267, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_DelItem(HPyContext *tctx, HPy obj, HPy key) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 235); @@ -1201,6 +1266,19 @@ int trace_ctx_DelItem_s(HPyContext *tctx, HPy obj, const char *utf8_key) return res; } +int trace_ctx_DelSlice(HPyContext *tctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 268); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPy_DelSlice(uctx, obj, start, end); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 268, r0, r1, &_ts_start, &_ts_end); + return res; +} + HPy trace_ctx_Type(HPyContext *tctx, HPy obj) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 165); @@ -1370,6 +1448,19 @@ void *trace_ctx_AsStruct_List(HPyContext *tctx, HPy h) return res; } +void *trace_ctx_AsStruct_Dict(HPyContext *tctx, HPy h) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 264); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + void * res = _HPy_AsStruct_Dict(uctx, h); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 264, r0, r1, &_ts_start, &_ts_end); + return res; +} + HPyType_BuiltinShape trace_ctx_Type_GetBuiltinShape(HPyContext *tctx, HPy h_type) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 234); @@ -1812,6 +1903,19 @@ int trace_ctx_List_Append(HPyContext *tctx, HPy h_list, HPy h_item) return res; } +int trace_ctx_List_Insert(HPyContext *tctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 265); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + int res = HPyList_Insert(uctx, h_list, index, h_item); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 265, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Dict_Check(HPyContext *tctx, HPy h) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 201); @@ -1877,7 +1981,7 @@ int trace_ctx_Tuple_Check(HPyContext *tctx, HPy h) return res; } -HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n) +HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, const HPy items[], HPy_ssize_t n) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 204); HPyContext *uctx = info->uctx; @@ -1890,6 +1994,19 @@ HPy trace_ctx_Tuple_FromArray(HPyContext *tctx, HPy items[], HPy_ssize_t n) return res; } +HPy trace_ctx_Slice_New(HPyContext *tctx, HPy start, HPy stop, HPy step) +{ + HPyTraceInfo *info = hpy_trace_on_enter(tctx, 272); + HPyContext *uctx = info->uctx; + _HPyTime_t _ts_start, _ts_end; + _HPyClockStatus_t r0, r1; + r0 = get_monotonic_clock(&_ts_start); + HPy res = HPySlice_New(uctx, start, stop, step); + r1 = get_monotonic_clock(&_ts_end); + hpy_trace_on_exit(info, 272, r0, r1, &_ts_start, &_ts_end); + return res; +} + int trace_ctx_Slice_Unpack(HPyContext *tctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) { HPyTraceInfo *info = hpy_trace_on_enter(tctx, 259); diff --git a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py b/graalpython/lib-graalpython/modules/hpy/devel/__init__.py index e59dd027ac..3b7f6e6d18 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py +++ b/graalpython/lib-graalpython/modules/hpy/devel/__init__.py @@ -428,7 +428,7 @@ def write_stub(self, output_dir, ext, compile=False): ext_file = os.path.basename(ext._file_name) module_name = ext_file.split(".")[0] if not self.dry_run: - with open(stub_file, 'w') as f: + with open(stub_file, 'w', encoding='utf-8') as f: f.write(_HPY_UNIVERSAL_MODULE_STUB_TEMPLATE.format( ext_file=ext_file, module_name=module_name) ) diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c index dc6ee2b2bf..9fa72a205c 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c @@ -15,6 +15,14 @@ ctx_Dump(HPyContext *ctx, HPy h) _PyObject_Dump(_h2py(h)); } +_HPy_HIDDEN HPy +ctx_Type(HPyContext *ctx, HPy obj) +{ + PyTypeObject *tp = Py_TYPE(_h2py(obj)); + Py_INCREF(tp); + return _py2h((PyObject *)tp); +} + /* NOTE: In contrast to CPython, HPy has to check that 'h_type' is a type. This is not necessary on CPython because it requires C type 'PyTypeObject *' but here we can only receive an HPy handle. Appropriate checking of the argument @@ -34,10 +42,14 @@ ctx_Is(HPyContext *ctx, HPy h_obj, HPy h_other) _HPy_HIDDEN HPy ctx_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx) { + PyObject *py_obj = _h2py(obj); + if (PySequence_Check(py_obj)) { + return _py2h(PySequence_GetItem(py_obj, idx)); + } PyObject* key = PyLong_FromSsize_t(idx); if (key == NULL) return HPy_NULL; - HPy result = _py2h(PyObject_GetItem(_h2py(obj), key)); + HPy result = _py2h(PyObject_GetItem(py_obj, key)); Py_DECREF(key); return result; } diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c index eb16a87b57..42661f8fea 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c @@ -8,7 +8,7 @@ _HPy_HIDDEN HPy -ctx_Tuple_FromArray(HPyContext *ctx, HPy items[], HPy_ssize_t n) +ctx_Tuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n) { PyObject *res = PyTuple_New(n); if (!res) diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c index 2310aa4fbe..ef89e2590d 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c +++ b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c @@ -76,6 +76,7 @@ FULLY_ALIGNED_SPACE(PyFloatObject) FULLY_ALIGNED_SPACE(PyUnicodeObject) FULLY_ALIGNED_SPACE(PyTupleObject) FULLY_ALIGNED_SPACE(PyListObject) +FULLY_ALIGNED_SPACE(PyDictObject) #define _HPy_HEAD_SIZE(HEAD) (offsetof(_HPy_FullyAlignedSpaceFor##HEAD, payload)) @@ -101,6 +102,8 @@ _HPy_GetHeaderSize(HPyType_BuiltinShape shape) return _HPy_HEAD_SIZE(PyTupleObject); case HPyType_BuiltinShape_List: return _HPy_HEAD_SIZE(PyListObject); + case HPyType_BuiltinShape_Dict: + return _HPy_HEAD_SIZE(PyDictObject); } return -1; } @@ -1504,6 +1507,12 @@ ctx_AsStruct_List(HPyContext *ctx, HPy h) return _HPy_Payload(_h2py(h), HPyType_BuiltinShape_List); } +_HPy_HIDDEN void* +ctx_AsStruct_Dict(HPyContext *ctx, HPy h) +{ + return _HPy_Payload(_h2py(h), HPyType_BuiltinShape_Dict); +} + _HPy_HIDDEN void* ctx_AsStruct_Slow(HPyContext *ctx, HPy h) { diff --git a/graalpython/lib-graalpython/modules/hpy/devel/version.py b/graalpython/lib-graalpython/modules/hpy/devel/version.py index 356bae1ebb..0cd1dbcc22 100644 --- a/graalpython/lib-graalpython/modules/hpy/devel/version.py +++ b/graalpython/lib-graalpython/modules/hpy/devel/version.py @@ -1,4 +1,4 @@ # automatically generated by setup.py:get_scm_config() -__version__ = "0.9.0" -__git_revision__ = "f6114734" +__version__ = "0.9.1.dev79+gb0fbdf73" +__git_revision__ = "b0fbdf73" diff --git a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py new file mode 100644 index 0000000000..56a68d344c --- /dev/null +++ b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py @@ -0,0 +1,43 @@ +from hpy.universal import _debug + +class HPyDebugError(Exception): + pass + +class HPyLeakError(HPyDebugError): + def __init__(self, leaks): + super().__init__() + self.leaks = leaks + + def __str__(self): + lines = [] + n = len(self.leaks) + s = 's' if n != 1 else '' + lines.append(f'{n} unclosed handle{s}:') + for dh in self.leaks: + lines.append(' %r' % dh) + return '\n'.join(lines) + + +class LeakDetector: + + def __init__(self): + self.generation = None + + def start(self): + if self.generation is not None: + raise ValueError('LeakDetector already started') + self.generation = _debug.new_generation() + + def stop(self): + if self.generation is None: + raise ValueError('LeakDetector not started yet') + leaks = _debug.get_open_handles(self.generation) + if leaks: + raise HPyLeakError(leaks) + + def __enter__(self): + self.start() + return self + + def __exit__(self, etype, evalue, tb): + self.stop() diff --git a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py new file mode 100644 index 0000000000..9a33c51343 --- /dev/null +++ b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py @@ -0,0 +1,31 @@ +# hpy.debug / pytest integration + +import pytest +from .leakdetector import LeakDetector + +# For now "hpy_debug" just does leak detection, but in the future it might +# grows extra features: that's why it's called generically "hpy_debug" instead +# of "detect_leaks". + +# NOTE: the fixture itself is currently untested :(. It turns out that testing +# that the fixture raises during the teardown is complicated and probably +# requires to write a full-fledged plugin. We might want to turn this into a +# real plugin in the future, but for now I think this is enough. + +# pypy still uses a very ancient version of pytest, 2.9.2: pytest<3.x needs to +# use @yield_fixture, which is deprecated in newer version of pytest (where +# you can just use @fixture) +if pytest.__version__ < '3': + fixture = pytest.yield_fixture +else: + fixture = pytest.fixture + +@fixture +def hpy_debug(request): + """ + pytest fixture which makes it possible to control hpy.debug from within a test. + + In particular, it automatically check that the test doesn't leak. + """ + with LeakDetector() as ld: + yield ld From e6b441c18c563e2bb328fb9b71d5067c2a0f4b34 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 8 Apr 2025 15:34:01 +0200 Subject: [PATCH 2/3] include hpy as submodule, commit b0fbdf73bb74bc96310835781bb3fe3386138253 --- .../hpy/debug_internal.h | 233 --- .../hpy/hpy_debug.h | 59 - .../modules/_hpy_debug.c | 403 ---- .../include/hpy/version.h | 4 - .../src/debug/include/hpy_debug.h | 57 - .../src/trace/include/hpy_trace.h | 42 - graalpython/hpy/.gitattributes | 1 + graalpython/hpy/.github/FUNDING.yml | 1 + graalpython/hpy/.github/workflows/ci.yml | 427 ++++ .../hpy/.github/workflows/release-pypi.yml | 136 ++ .../hpy/.github/workflows/valgrind-tests.yml | 38 + graalpython/hpy/.gitignore | 92 + graalpython/hpy/.readthedocs.yml | 10 + graalpython/hpy/AUTHORS | 38 + graalpython/hpy/CONTRIBUTING.md | 55 + graalpython/hpy/LICENSE | 21 + graalpython/hpy/MANIFEST.in | 1 + graalpython/hpy/Makefile | 86 + graalpython/hpy/README-gdb.md | 207 ++ graalpython/hpy/README.md | 81 + graalpython/hpy/c_test/Makefile | 16 + graalpython/hpy/c_test/acutest.h | 1794 +++++++++++++++++ graalpython/hpy/c_test/test_debug_handles.c | 99 + graalpython/hpy/c_test/test_stacktrace.c | 22 + graalpython/hpy/docs/Makefile | 28 + graalpython/hpy/docs/_static/README.txt | 1 + graalpython/hpy/docs/_templates/README.txt | 1 + .../docs/api-reference/argument-parsing.rst | 5 + .../hpy/docs/api-reference/build-value.rst | 5 + .../hpy/docs/api-reference/formatting.rst | 5 + .../hpy/docs/api-reference/function-index.rst | 187 ++ .../hpy/docs/api-reference/helpers.rst | 5 + .../hpy/docs/api-reference/hpy-call.rst | 5 + .../hpy/docs/api-reference/hpy-ctx.rst | 9 + .../hpy/docs/api-reference/hpy-dict.rst | 5 + .../hpy/docs/api-reference/hpy-err.rst | 5 + .../hpy/docs/api-reference/hpy-eval.rst | 8 + .../hpy/docs/api-reference/hpy-field.rst | 5 + .../hpy/docs/api-reference/hpy-gil.rst | 5 + .../hpy/docs/api-reference/hpy-global.rst | 5 + .../hpy/docs/api-reference/hpy-object.rst | 5 + .../hpy/docs/api-reference/hpy-sequence.rst | 23 + .../hpy/docs/api-reference/hpy-type.rst | 50 + graalpython/hpy/docs/api-reference/index.rst | 69 + .../hpy/docs/api-reference/inline-helpers.rst | 14 + .../hpy/docs/api-reference/public-api.rst | 9 + .../hpy/docs/api-reference/structseq.rst | 15 + graalpython/hpy/docs/api.rst | 581 ++++++ graalpython/hpy/docs/changelog.rst | 235 +++ graalpython/hpy/docs/conf.py | 134 ++ graalpython/hpy/docs/contributing/index.rst | 33 + graalpython/hpy/docs/debug-mode.rst | 169 ++ .../hpy/docs/examples/debug-example.py | 10 + .../examples/hpytype-example/builtin_type.c | 106 + .../docs/examples/hpytype-example/setup.py | 11 + .../examples/hpytype-example/simple_type.c | 102 + .../examples/hpytype-example/simple_type.rst | 8 + .../hpy/docs/examples/mixed-example/mixed.c | 49 + .../hpy/docs/examples/mixed-example/setup.py | 10 + .../hpy/docs/examples/quickstart/quickstart.c | 39 + .../hpy/docs/examples/quickstart/setup.py | 13 + .../hpy/docs/examples/simple-example/setup.py | 10 + .../hpy/docs/examples/simple-example/simple.c | 43 + .../hpy/docs/examples/snippets/hpycall.c | 196 ++ .../hpy/docs/examples/snippets/hpyinit.c | 22 + .../hpy/docs/examples/snippets/hpyvarargs.c | 45 + .../hpy/docs/examples/snippets/legacyinit.c | 20 + .../hpy/docs/examples/snippets/setup.py | 16 + .../hpy/docs/examples/snippets/snippets.c | 81 + graalpython/hpy/docs/examples/tests.py | 124 ++ .../hpy/docs/examples/trace-example.py | 9 + graalpython/hpy/docs/index.rst | 80 + .../hpy/docs/leysin-2020-design-decisions.md | 295 +++ graalpython/hpy/docs/misc/embedding.rst | 46 + graalpython/hpy/docs/misc/index.rst | 7 + graalpython/hpy/docs/module-state.txt | 44 + graalpython/hpy/docs/overview.rst | 444 ++++ .../hpy/docs/porting-example/index.rst | 356 ++++ .../hpy/docs/porting-example/steps/.gitignore | 1 + .../docs/porting-example/steps/conftest.py | 35 + .../hpy/docs/porting-example/steps/setup00.py | 12 + .../hpy/docs/porting-example/steps/setup01.py | 13 + .../hpy/docs/porting-example/steps/setup02.py | 13 + .../hpy/docs/porting-example/steps/setup03.py | 14 + .../porting-example/steps/step_00_c_api.c | 165 ++ .../porting-example/steps/step_00_c_api.rst | 8 + .../steps/step_01_hpy_legacy.c | 189 ++ .../steps/step_01_hpy_legacy.rst | 8 + .../steps/step_02_hpy_legacy.c | 169 ++ .../steps/step_02_hpy_legacy.rst | 8 + .../porting-example/steps/step_03_hpy_final.c | 153 ++ .../steps/step_03_hpy_final.rst | 8 + .../steps/test_porting_example.py | 60 + graalpython/hpy/docs/porting-guide.rst | 569 ++++++ graalpython/hpy/docs/quickstart.rst | 55 + graalpython/hpy/docs/requirements.txt | 8 + graalpython/hpy/docs/trace-mode.rst | 64 + graalpython/hpy/docs/xxx-unsorted-notes.txt | 91 + graalpython/hpy/gdb-py.test | 3 + .../modules => hpy}/hpy/debug/__init__.py | 0 .../modules => hpy}/hpy/debug/leakdetector.py | 0 .../modules => hpy}/hpy/debug/pytest.py | 0 .../debug => hpy/hpy/debug/src}/_debugmod.c | 0 .../hpy/debug/src/autogen_debug_ctx_call.i | 468 +++++ .../hpy/debug/src}/autogen_debug_ctx_init.h | 0 .../hpy/debug/src}/autogen_debug_wrappers.c | 0 .../debug => hpy/hpy/debug/src}/debug_ctx.c | 0 .../hpy/hpy/debug/src/debug_ctx_cpython.c | 299 +++ .../hpy/debug/src}/debug_ctx_not_cpython.c | 0 .../hpy/debug/src}/debug_handles.c | 0 .../hpy/debug/src}/debug_internal.h | 0 .../src/debug => hpy/hpy/debug/src}/dhqueue.c | 0 .../hpy/debug/src/include}/hpy_debug.h | 0 .../debug => hpy/hpy/debug/src}/memprotect.c | 0 .../debug => hpy/hpy/debug/src}/stacktrace.c | 0 .../modules => hpy}/hpy/devel/__init__.py | 0 .../modules => hpy}/hpy/devel/abitag.py | 0 .../hpy/devel}/include/hpy.h | 0 .../include/hpy/autogen_hpyfunc_declare.h | 0 .../hpy/devel}/include/hpy/autogen_hpyslot.h | 0 .../hpy/devel}/include/hpy/cpy_types.h | 0 .../include/hpy/cpython/autogen_api_impl.h | 0 .../devel}/include/hpy/cpython/autogen_ctx.h | 0 .../hpy/cpython/autogen_hpyfunc_trampolines.h | 0 .../include/hpy/cpython/hpyfunc_trampolines.h | 0 .../hpy/devel}/include/hpy/cpython/misc.h | 0 .../include/hpy/forbid_python_h/Python.h | 0 .../hpy/devel}/include/hpy/hpydef.h | 0 .../hpy/devel}/include/hpy/hpyexports.h | 0 .../hpy/devel}/include/hpy/hpyfunc.h | 0 .../hpy/devel}/include/hpy/hpymodule.h | 0 .../hpy/devel}/include/hpy/hpytype.h | 0 .../hpy/devel}/include/hpy/inline_helpers.h | 0 .../hpy/devel}/include/hpy/macros.h | 0 .../hpy/devel}/include/hpy/runtime/argparse.h | 0 .../devel}/include/hpy/runtime/buildvalue.h | 0 .../devel}/include/hpy/runtime/ctx_funcs.h | 0 .../devel}/include/hpy/runtime/ctx_module.h | 0 .../hpy/devel}/include/hpy/runtime/ctx_type.h | 0 .../hpy/devel}/include/hpy/runtime/format.h | 0 .../hpy/devel}/include/hpy/runtime/helpers.h | 0 .../devel}/include/hpy/runtime/structseq.h | 0 .../include/hpy/universal/autogen_ctx.h | 0 .../universal/autogen_hpyfunc_trampolines.h | 0 .../hpy/universal/autogen_trampolines.h | 0 .../hpy/universal/hpyfunc_trampolines.h | 0 .../include/hpy/universal/misc_trampolines.h | 0 .../hpy/devel/src/runtime/argparse.c | 0 .../hpy/devel/src/runtime/buildvalue.c | 0 .../hpy/devel/src/runtime/ctx_bytes.c | 0 .../hpy/devel/src/runtime/ctx_call.c | 0 .../hpy/devel/src/runtime/ctx_capsule.c | 0 .../hpy/devel/src/runtime/ctx_contextvar.c | 0 .../hpy/devel/src/runtime/ctx_err.c | 0 .../hpy/devel/src/runtime/ctx_eval.c | 0 .../hpy/devel/src/runtime/ctx_listbuilder.c | 0 .../hpy/devel/src/runtime/ctx_long.c | 0 .../hpy/devel/src/runtime/ctx_module.c | 0 .../hpy/devel/src/runtime/ctx_object.c | 0 .../hpy/devel/src/runtime}/ctx_tracker.c | 0 .../hpy/devel/src/runtime/ctx_tuple.c | 0 .../hpy/devel/src/runtime/ctx_tuplebuilder.c | 0 .../hpy/devel/src/runtime/ctx_type.c | 0 .../hpy/devel/src/runtime/format.c | 0 .../hpy/devel/src/runtime/helpers.c | 0 .../hpy/devel/src/runtime/structseq.c | 0 graalpython/hpy/hpy/tools/autogen/__init__.py | 21 + graalpython/hpy/hpy/tools/autogen/__main__.py | 66 + graalpython/hpy/hpy/tools/autogen/autogen.h | 40 + .../hpy/hpy/tools/autogen/autogenfile.py | 35 + graalpython/hpy/hpy/tools/autogen/conf.py | 211 ++ graalpython/hpy/hpy/tools/autogen/ctx.py | 106 + graalpython/hpy/hpy/tools/autogen/debug.py | 228 +++ graalpython/hpy/hpy/tools/autogen/doc.py | 173 ++ graalpython/hpy/hpy/tools/autogen/hpyfunc.py | 204 ++ graalpython/hpy/hpy/tools/autogen/hpyslot.py | 18 + graalpython/hpy/hpy/tools/autogen/parse.py | 278 +++ .../hpy/hpy/tools/autogen/public_api.h | 1557 ++++++++++++++ graalpython/hpy/hpy/tools/autogen/pypy.py | 40 + .../hpy/tools/autogen/testing}/__init__.py | 0 .../hpy/tools/autogen/testing/test_autogen.py | 224 ++ .../hpy/tools/autogen/testing/test_hpyfunc.py | 145 ++ graalpython/hpy/hpy/tools/autogen/trace.py | 182 ++ .../hpy/hpy/tools/autogen/trampolines.py | 140 ++ graalpython/hpy/hpy/tools/include_path.py | 4 + graalpython/hpy/hpy/tools/valgrind/hpy.supp | 44 + .../hpy/hpy/tools/valgrind/python.supp | 389 ++++ graalpython/hpy/hpy/trace/__init__.py | 6 + .../trace => hpy/hpy/trace/src}/_tracemod.c | 0 .../hpy/trace/src}/autogen_trace_ctx_init.h | 0 .../hpy/trace/src}/autogen_trace_func_table.c | 0 .../hpy/trace/src}/autogen_trace_wrappers.c | 0 .../hpy/trace/src/include}/hpy_trace.h | 0 .../trace => hpy/hpy/trace/src}/trace_ctx.c | 0 .../hpy/trace/src}/trace_internal.h | 0 graalpython/hpy/hpy/universal/src/api.h | 18 + .../hpy/hpy/universal/src/autogen_ctx_call.i | 180 ++ .../hpy/hpy/universal/src/autogen_ctx_def.h | 207 ++ .../hpy/hpy/universal/src/autogen_ctx_impl.h | 612 ++++++ graalpython/hpy/hpy/universal/src/ctx.c | 10 + graalpython/hpy/hpy/universal/src/ctx_meth.c | 136 ++ graalpython/hpy/hpy/universal/src/ctx_meth.h | 6 + graalpython/hpy/hpy/universal/src/ctx_misc.c | 83 + graalpython/hpy/hpy/universal/src/ctx_misc.h | 20 + graalpython/hpy/hpy/universal/src/handles.h | 58 + graalpython/hpy/hpy/universal/src/hpymodule.c | 691 +++++++ .../hpy/hpy/universal/src/misc_win32.h | 55 + graalpython/hpy/microbench/README.md | 29 + graalpython/hpy/microbench/_valgrind_build.py | 29 + graalpython/hpy/microbench/conftest.py | 124 ++ graalpython/hpy/microbench/pytest_valgrind.sh | 3 + graalpython/hpy/microbench/setup.py | 17 + graalpython/hpy/microbench/src/cpy_simple.c | 174 ++ graalpython/hpy/microbench/src/hpy_simple.c | 135 ++ graalpython/hpy/microbench/test_microbench.py | 226 +++ graalpython/hpy/proof-of-concept/pof.c | 109 + graalpython/hpy/proof-of-concept/pofcpp.cpp | 117 ++ .../hpy/proof-of-concept/pofpackage/bar.cpp | 45 + .../hpy/proof-of-concept/pofpackage/foo.c | 20 + .../hpy/proof-of-concept/requirements.txt | 1 + graalpython/hpy/proof-of-concept/setup.py | 36 + graalpython/hpy/proof-of-concept/test_pof.py | 44 + graalpython/hpy/proof-of-concept/test_pof.sh | 177 ++ graalpython/hpy/pyproject.toml | 3 + graalpython/hpy/requirements-autogen.txt | 4 + graalpython/hpy/setup.py | 269 +++ .../hpytest/debug => hpy/test}/__init__.py | 0 .../hpytest => hpy/test}/check_py27_compat.py | 0 .../src/hpytest => hpy/test}/conftest.py | 0 .../hpy_devel => hpy/test/debug}/__init__.py | 0 .../test}/debug/test_builder_invalid.py | 0 .../test}/debug/test_charptr.py | 0 .../test}/debug/test_context_reuse.py | 0 .../test}/debug/test_handles_invalid.py | 0 .../test}/debug/test_handles_leak.py | 0 .../hpytest => hpy/test}/debug/test_misc.py | 0 graalpython/hpy/test/hpy_devel/__init__.py | 0 .../test}/hpy_devel/test_abitag.py | 0 .../test}/hpy_devel/test_distutils.py | 0 .../src/hpytest => hpy/test}/support.py | 0 .../src/hpytest => hpy/test}/test_00_basic.py | 0 .../src/hpytest => hpy/test}/test_argparse.py | 0 .../src/hpytest => hpy/test}/test_call.py | 0 .../src/hpytest => hpy/test}/test_capsule.py | 0 .../test}/test_capsule_legacy.py | 0 .../hpytest => hpy/test}/test_contextvar.py | 0 .../hpytest => hpy/test}/test_cpy_compat.py | 0 .../src/hpytest => hpy/test}/test_eval.py | 0 .../src/hpytest => hpy/test}/test_helpers.py | 0 .../test}/test_hpybuildvalue.py | 0 .../src/hpytest => hpy/test}/test_hpybytes.py | 0 .../src/hpytest => hpy/test}/test_hpydict.py | 0 .../src/hpytest => hpy/test}/test_hpyerr.py | 0 .../src/hpytest => hpy/test}/test_hpyfield.py | 0 .../hpytest => hpy/test}/test_hpyglobal.py | 0 .../hpytest => hpy/test}/test_hpyimport.py | 0 .../src/hpytest => hpy/test}/test_hpyiter.py | 0 .../src/hpytest => hpy/test}/test_hpylist.py | 0 .../src/hpytest => hpy/test}/test_hpylong.py | 0 .../hpytest => hpy/test}/test_hpymodule.py | 0 .../src/hpytest => hpy/test}/test_hpyslice.py | 0 .../hpytest => hpy/test}/test_hpystructseq.py | 0 .../src/hpytest => hpy/test}/test_hpytuple.py | 0 .../src/hpytest => hpy/test}/test_hpytype.py | 0 .../test}/test_hpytype_legacy.py | 0 .../hpytest => hpy/test}/test_hpyunicode.py | 0 .../hpytest => hpy/test}/test_importing.py | 0 .../test}/test_legacy_forbidden.py | 0 .../src/hpytest => hpy/test}/test_number.py | 0 .../src/hpytest => hpy/test}/test_object.py | 0 .../src/hpytest => hpy/test}/test_slots.py | 0 .../hpytest => hpy/test}/test_slots_legacy.py | 0 .../src/hpytest => hpy/test}/test_support.py | 0 .../src/hpytest => hpy/test}/test_tracker.py | 0 .../hpytest => hpy/test}/trace/test_trace.py | 0 .../hpy/devel/src/runtime/ctx_tracker.c | 155 -- .../modules/hpy/devel/version.py | 4 - .../modules/hpy/trace/__init__.py | 11 - .../modules/hpy/trace/leakdetector.py | 43 - .../modules/hpy/trace/pytest.py | 31 - 280 files changed, 17094 insertions(+), 1042 deletions(-) delete mode 100644 graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h delete mode 100644 graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h delete mode 100644 graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c delete mode 100644 graalpython/com.oracle.graal.python.hpy/include/hpy/version.h delete mode 100644 graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h delete mode 100644 graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h create mode 100644 graalpython/hpy/.gitattributes create mode 100644 graalpython/hpy/.github/FUNDING.yml create mode 100644 graalpython/hpy/.github/workflows/ci.yml create mode 100644 graalpython/hpy/.github/workflows/release-pypi.yml create mode 100644 graalpython/hpy/.github/workflows/valgrind-tests.yml create mode 100644 graalpython/hpy/.gitignore create mode 100644 graalpython/hpy/.readthedocs.yml create mode 100644 graalpython/hpy/AUTHORS create mode 100644 graalpython/hpy/CONTRIBUTING.md create mode 100644 graalpython/hpy/LICENSE create mode 100644 graalpython/hpy/MANIFEST.in create mode 100644 graalpython/hpy/Makefile create mode 100644 graalpython/hpy/README-gdb.md create mode 100644 graalpython/hpy/README.md create mode 100644 graalpython/hpy/c_test/Makefile create mode 100644 graalpython/hpy/c_test/acutest.h create mode 100644 graalpython/hpy/c_test/test_debug_handles.c create mode 100644 graalpython/hpy/c_test/test_stacktrace.c create mode 100644 graalpython/hpy/docs/Makefile create mode 100644 graalpython/hpy/docs/_static/README.txt create mode 100644 graalpython/hpy/docs/_templates/README.txt create mode 100644 graalpython/hpy/docs/api-reference/argument-parsing.rst create mode 100644 graalpython/hpy/docs/api-reference/build-value.rst create mode 100644 graalpython/hpy/docs/api-reference/formatting.rst create mode 100644 graalpython/hpy/docs/api-reference/function-index.rst create mode 100644 graalpython/hpy/docs/api-reference/helpers.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-call.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-ctx.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-dict.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-err.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-eval.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-field.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-gil.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-global.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-object.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-sequence.rst create mode 100644 graalpython/hpy/docs/api-reference/hpy-type.rst create mode 100644 graalpython/hpy/docs/api-reference/index.rst create mode 100644 graalpython/hpy/docs/api-reference/inline-helpers.rst create mode 100644 graalpython/hpy/docs/api-reference/public-api.rst create mode 100644 graalpython/hpy/docs/api-reference/structseq.rst create mode 100644 graalpython/hpy/docs/api.rst create mode 100644 graalpython/hpy/docs/changelog.rst create mode 100644 graalpython/hpy/docs/conf.py create mode 100644 graalpython/hpy/docs/contributing/index.rst create mode 100644 graalpython/hpy/docs/debug-mode.rst create mode 100644 graalpython/hpy/docs/examples/debug-example.py create mode 100644 graalpython/hpy/docs/examples/hpytype-example/builtin_type.c create mode 100644 graalpython/hpy/docs/examples/hpytype-example/setup.py create mode 100644 graalpython/hpy/docs/examples/hpytype-example/simple_type.c create mode 100644 graalpython/hpy/docs/examples/hpytype-example/simple_type.rst create mode 100644 graalpython/hpy/docs/examples/mixed-example/mixed.c create mode 100644 graalpython/hpy/docs/examples/mixed-example/setup.py create mode 100644 graalpython/hpy/docs/examples/quickstart/quickstart.c create mode 100644 graalpython/hpy/docs/examples/quickstart/setup.py create mode 100644 graalpython/hpy/docs/examples/simple-example/setup.py create mode 100644 graalpython/hpy/docs/examples/simple-example/simple.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpycall.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpyinit.c create mode 100644 graalpython/hpy/docs/examples/snippets/hpyvarargs.c create mode 100644 graalpython/hpy/docs/examples/snippets/legacyinit.c create mode 100644 graalpython/hpy/docs/examples/snippets/setup.py create mode 100644 graalpython/hpy/docs/examples/snippets/snippets.c create mode 100644 graalpython/hpy/docs/examples/tests.py create mode 100644 graalpython/hpy/docs/examples/trace-example.py create mode 100644 graalpython/hpy/docs/index.rst create mode 100644 graalpython/hpy/docs/leysin-2020-design-decisions.md create mode 100644 graalpython/hpy/docs/misc/embedding.rst create mode 100644 graalpython/hpy/docs/misc/index.rst create mode 100644 graalpython/hpy/docs/module-state.txt create mode 100644 graalpython/hpy/docs/overview.rst create mode 100644 graalpython/hpy/docs/porting-example/index.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/.gitignore create mode 100644 graalpython/hpy/docs/porting-example/steps/conftest.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup00.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup01.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup02.py create mode 100644 graalpython/hpy/docs/porting-example/steps/setup03.py create mode 100644 graalpython/hpy/docs/porting-example/steps/step_00_c_api.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c create mode 100644 graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst create mode 100644 graalpython/hpy/docs/porting-example/steps/test_porting_example.py create mode 100644 graalpython/hpy/docs/porting-guide.rst create mode 100644 graalpython/hpy/docs/quickstart.rst create mode 100644 graalpython/hpy/docs/requirements.txt create mode 100644 graalpython/hpy/docs/trace-mode.rst create mode 100644 graalpython/hpy/docs/xxx-unsorted-notes.txt create mode 100755 graalpython/hpy/gdb-py.test rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/__init__.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/leakdetector.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/debug/pytest.py (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/_debugmod.c (100%) create mode 100644 graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/autogen_debug_ctx_init.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/autogen_debug_wrappers.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_ctx.c (100%) create mode 100644 graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_ctx_not_cpython.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_handles.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/debug_internal.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/dhqueue.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src/include}/hpy_debug.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/memprotect.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/debug => hpy/hpy/debug/src}/stacktrace.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/__init__.py (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/abitag.py (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/autogen_hpyfunc_declare.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/autogen_hpyslot.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpy_types.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_api_impl.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_ctx.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/cpython/misc.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/forbid_python_h/Python.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpydef.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpyexports.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpyfunc.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpymodule.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/hpytype.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/inline_helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/macros.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/argparse.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/buildvalue.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_funcs.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_module.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/ctx_type.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/format.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/helpers.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/runtime/structseq.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_ctx.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/autogen_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/hpyfunc_trampolines.h (100%) rename graalpython/{com.oracle.graal.python.hpy => hpy/hpy/devel}/include/hpy/universal/misc_trampolines.h (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/argparse.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/buildvalue.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_bytes.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_call.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_capsule.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_contextvar.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_err.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_eval.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_listbuilder.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_long.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_module.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_object.c (100%) rename graalpython/{com.oracle.graal.python.jni/src => hpy/hpy/devel/src/runtime}/ctx_tracker.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_tuple.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_tuplebuilder.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/ctx_type.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/format.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/helpers.c (100%) rename graalpython/{lib-graalpython/modules => hpy}/hpy/devel/src/runtime/structseq.c (100%) create mode 100644 graalpython/hpy/hpy/tools/autogen/__init__.py create mode 100644 graalpython/hpy/hpy/tools/autogen/__main__.py create mode 100644 graalpython/hpy/hpy/tools/autogen/autogen.h create mode 100644 graalpython/hpy/hpy/tools/autogen/autogenfile.py create mode 100644 graalpython/hpy/hpy/tools/autogen/conf.py create mode 100644 graalpython/hpy/hpy/tools/autogen/ctx.py create mode 100644 graalpython/hpy/hpy/tools/autogen/debug.py create mode 100644 graalpython/hpy/hpy/tools/autogen/doc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/hpyfunc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/hpyslot.py create mode 100644 graalpython/hpy/hpy/tools/autogen/parse.py create mode 100644 graalpython/hpy/hpy/tools/autogen/public_api.h create mode 100644 graalpython/hpy/hpy/tools/autogen/pypy.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/hpy/tools/autogen/testing}/__init__.py (100%) create mode 100644 graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py create mode 100644 graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py create mode 100644 graalpython/hpy/hpy/tools/autogen/trace.py create mode 100644 graalpython/hpy/hpy/tools/autogen/trampolines.py create mode 100644 graalpython/hpy/hpy/tools/include_path.py create mode 100644 graalpython/hpy/hpy/tools/valgrind/hpy.supp create mode 100644 graalpython/hpy/hpy/tools/valgrind/python.supp create mode 100644 graalpython/hpy/hpy/trace/__init__.py rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/_tracemod.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_ctx_init.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_func_table.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/autogen_trace_wrappers.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src/include}/hpy_trace.h (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/trace_ctx.c (100%) rename graalpython/{com.oracle.graal.python.jni/src/trace => hpy/hpy/trace/src}/trace_internal.h (100%) create mode 100644 graalpython/hpy/hpy/universal/src/api.h create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_call.i create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_def.h create mode 100644 graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h create mode 100644 graalpython/hpy/hpy/universal/src/ctx.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_meth.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_meth.h create mode 100644 graalpython/hpy/hpy/universal/src/ctx_misc.c create mode 100644 graalpython/hpy/hpy/universal/src/ctx_misc.h create mode 100644 graalpython/hpy/hpy/universal/src/handles.h create mode 100644 graalpython/hpy/hpy/universal/src/hpymodule.c create mode 100644 graalpython/hpy/hpy/universal/src/misc_win32.h create mode 100644 graalpython/hpy/microbench/README.md create mode 100644 graalpython/hpy/microbench/_valgrind_build.py create mode 100644 graalpython/hpy/microbench/conftest.py create mode 100755 graalpython/hpy/microbench/pytest_valgrind.sh create mode 100644 graalpython/hpy/microbench/setup.py create mode 100644 graalpython/hpy/microbench/src/cpy_simple.c create mode 100644 graalpython/hpy/microbench/src/hpy_simple.c create mode 100644 graalpython/hpy/microbench/test_microbench.py create mode 100644 graalpython/hpy/proof-of-concept/pof.c create mode 100644 graalpython/hpy/proof-of-concept/pofcpp.cpp create mode 100644 graalpython/hpy/proof-of-concept/pofpackage/bar.cpp create mode 100644 graalpython/hpy/proof-of-concept/pofpackage/foo.c create mode 100644 graalpython/hpy/proof-of-concept/requirements.txt create mode 100644 graalpython/hpy/proof-of-concept/setup.py create mode 100644 graalpython/hpy/proof-of-concept/test_pof.py create mode 100755 graalpython/hpy/proof-of-concept/test_pof.sh create mode 100644 graalpython/hpy/pyproject.toml create mode 100644 graalpython/hpy/requirements-autogen.txt create mode 100644 graalpython/hpy/setup.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest/debug => hpy/test}/__init__.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/check_py27_compat.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/conftest.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel => hpy/test/debug}/__init__.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_builder_invalid.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_charptr.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_context_reuse.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_handles_invalid.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_handles_leak.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/debug/test_misc.py (100%) create mode 100644 graalpython/hpy/test/hpy_devel/__init__.py rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/hpy_devel/test_abitag.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/hpy_devel/test_distutils.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/support.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_00_basic.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_argparse.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_call.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_capsule.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_capsule_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_contextvar.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_cpy_compat.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_eval.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_helpers.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpybuildvalue.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpybytes.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpydict.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyerr.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyfield.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyglobal.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyimport.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyiter.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpylist.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpylong.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpymodule.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyslice.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpystructseq.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytuple.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytype.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpytype_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_hpyunicode.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_importing.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_legacy_forbidden.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_number.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_object.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_slots.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_slots_legacy.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_support.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/test_tracker.py (100%) rename graalpython/{com.oracle.graal.python.hpy.test/src/hpytest => hpy/test}/trace/test_trace.py (100%) delete mode 100644 graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c delete mode 100644 graalpython/lib-graalpython/modules/hpy/devel/version.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/__init__.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py delete mode 100644 graalpython/lib-graalpython/modules/hpy/trace/pytest.py diff --git a/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h b/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h deleted file mode 100644 index e5bda5199b..0000000000 --- a/graalpython/com.oracle.graal.python.cext/hpy/debug_internal.h +++ /dev/null @@ -1,233 +0,0 @@ -/* Internal header for all the files in hpy/debug/src. The public API is in - include/hpy_debug.h -*/ -#ifndef HPY_DEBUG_INTERNAL_H -#define HPY_DEBUG_INTERNAL_H - -#include -#include "hpy.h" -#include "hpy_debug.h" - -#define HPY_DEBUG_MAGIC 0xDEB00FF - -/* The Debug context is a wrapper around an underlying context, which we will - call Universal. Inside the debug mode we manipulate handles which belongs - to both contexts, so to make things easier we create two typedefs to make - it clear what kind of handle we expect: UHPy and DHPy: - - * UHPy are opaque from our point of view. - - * DHPy are actually DebugHandle* in disguise. DebugHandles are wrappers - around a UHPy, with a bunch of extra info. - - To cast between DHPy and DebugHandle*, use as_DebugHandle and as_DHPy: - these are just no-op casts. - - Each DHPy wraps a corresponding UHPy: DHPys are created by calling - DHPy_open, and they must be eventually closed by DHPy_close. Note that if - you call DHPy_open twice on the same UHPy, you get two different DHPy. - - To unwrap a DHPy and get the underlying UHPy, call DHPy_unwrap. If you call - DHPy_unwrap multiple times on the same DHPy, you always get the same UHPy. - - WARNING: both UHPy and DHPy are alias of HPy, so we need to take care of - not mixing them, because the compiler cannot help. - - Each DebugHandle has a "generation", which is just an int to be able to get - only the handles which were created after a certain point. - - DHPys/DebugHandles are memory-managed by using a free list: - - - info->open_handles is a list of all DHPys which are currently open - - - DHPy_close() moves a DHPy from info->open_handles to info->closed_handles - - - if closed_handles is too big, the oldest DHPy is freed by DHPy_free() - - - to allocate memory for a new DHPy, DHPy_open() does the following: - - * if closed_handles is full, it reuses the memory of the oldest DHPy - in the queue - - * else, it malloc()s memory for a new DHPy - - - Each DebugHandle can have some "raw" data associated with it. It is a - generic pointer to any data. The validity, or life-time, of such pointer - is supposed to be the same as the that of the handle and the debug mode - enforces it. Additionally, the data can be also marked as write protected. - - Example is the `const char*` handed out by `HPyUnicode_AsUTF8AndSize`. It - must not be written by the user (users may discard the const modifier), and - the pointer is considered invalid once the handle is closed, so it must not - be accessed even for reading. Most Python implementations, will choose to - hand out pointer to the actual internal data, which happen to stay valid and - accessible and this may lead the users to a wrong conclusion that they can - use the pointer after the handle is closed. - - The memory protection mechanism is abstracted by several functions that - may have different implementations depending on the compile-time - configuration. Those are: - - * `raw_data_copy`: makes a copy of some data, optionally the copy can be - made read-only. - * `raw_data_protect`: protects the result of `raw_data_copy` from reading - * `raw_data_free`: if `raw_data_protect` retained any actual memory or other - resources, this indicates that those can be freed - - Any HPy context function that wishes to attach raw data to a handle should - make a copy of the actual data by using `raw_data_copy`. This copy should be - then set as the value of the associated_data field. Once the handle is - closed, the raw data pointer is passed to raw_data_protect and once the handle - is reused the raw data pointer is passed to raw_data_free. - - This means that if the implementation of `raw_data_protect` retains some - resources, we are leaking them. To mitigate this a bit, we have a limit on the - overall size of data that can be leaked and once it is reached, we use - raw_data_free immediately once the associated handle is closed. - - Note that, for example, the mmap based implementation of `raw_data_copy` - never allocates less than a page, so it actually takes more memory than - what is the size of the raw data. This is, however, mostly covered by the - limit on closed handles. For the default configuration we have: - - DEFAULT_CLOSED_HANDLES_QUEUE_MAX_SIZE = 1024 - DEFAULT_PROTECTED_RAW_DATA_MAX_SIZE = 1024 * 1024 * 10 - - the total leaked raw data size limit of 10MB is larger than if we created - and leaked 1024 handles with only a small raw data attached to them (4MB - for 1024 pages of 4KB). This ratio may be different for larger pages or for - different configuration of the limits. For the sake of keeping the - implementation reasonably simple and portable, we choose to ignore this - for the time being. -*/ - -typedef HPy UHPy; -typedef HPy DHPy; - -/* Under CPython: - - UHPy always end with 1 (see hpy.universal's _py2h and _h2py) - - DHPy are pointers, so they always end with 0 - - DHPy_sanity_check is a minimal check to ensure that we are not treating a - UHPy as a DHPy. Note that DHPy_sanity_check works fine also on HPy_NULL. - - NOTE: UHPy_sanity_check works ONLY with CPython's hpy.universal, because - UHPys are computed in such a way that the last bit it's always 1. On other - implementations this assumption might not hold. By default, - UHPy_sanity_check does nothing, unless you #define - HPY_DEBUG_ENABLE_UHPY_SANITY_CHECK, which for CPython is done by setup.py -*/ -static inline void DHPy_sanity_check(DHPy dh) { - assert( (dh._i & 1) == 0 ); -} - -static inline void UHPy_sanity_check(UHPy uh) { -#ifdef HPY_DEBUG_ENABLE_UHPY_SANITY_CHECK - if (!HPy_IsNull(uh)) - assert( (uh._i & 1) == 1 ); -#endif -} - -// NOTE: having a "generation" field is the easiest way to know when a handle -// was created, but we waste 8 bytes per handle. Since all handles of the same -// generation are stored sequentially in the open_handles list, a possible -// alternative implementation is to put special placeholders inside the list -// to mark the creation of a new generation -typedef struct DebugHandle { - UHPy uh; - long generation; - bool is_closed; - // pointer to and size of any raw data associated with - // the lifetime of the handle: - void *associated_data; - // allocation_stacktrace information if available - char *allocation_stacktrace; - HPy_ssize_t associated_data_size; - struct DebugHandle *prev; - struct DebugHandle *next; -} DebugHandle; - -static inline DebugHandle * as_DebugHandle(DHPy dh) { - DHPy_sanity_check(dh); - return (DebugHandle *)dh._i; -} - -static inline DHPy as_DHPy(DebugHandle *handle) { - return (DHPy){(HPy_ssize_t)handle}; -} - -DHPy DHPy_open(HPyContext *dctx, UHPy uh); -void DHPy_close(HPyContext *dctx, DHPy dh); -void DHPy_close_and_check(HPyContext *dctx, DHPy dh); -void DHPy_free(HPyContext *dctx, DHPy dh); -void DHPy_invalid_handle(HPyContext *dctx, DHPy dh); - -static inline UHPy DHPy_unwrap(HPyContext *dctx, DHPy dh) -{ - if (HPy_IsNull(dh)) - return HPy_NULL; - DebugHandle *handle = as_DebugHandle(dh); - if (handle->is_closed) - DHPy_invalid_handle(dctx, dh); - return handle->uh; -} - -/* === DHQueue === */ - -typedef struct { - DebugHandle *head; - DebugHandle *tail; - HPy_ssize_t size; -} DHQueue; - -void DHQueue_init(DHQueue *q); -void DHQueue_append(DHQueue *q, DebugHandle *h); -DebugHandle *DHQueue_popfront(DHQueue *q); -void DHQueue_remove(DHQueue *q, DebugHandle *h); -void DHQueue_sanity_check(DHQueue *q); - -/* === HPyDebugInfo === */ - -static const HPy_ssize_t DEFAULT_CLOSED_HANDLES_QUEUE_MAX_SIZE = 1024; -static const HPy_ssize_t DEFAULT_PROTECTED_RAW_DATA_MAX_SIZE = 1024 * 1024 * 10; - -typedef struct { - long magic_number; // used just for sanity checks - HPyContext *uctx; - long current_generation; - - // the following should be an HPyField, but it's complicate: - // HPyFields should be used only on memory which is known by the GC, which - // happens automatically if you use e.g. HPy_New, but currently - // HPy_DebugInfo is malloced(). We need either: - // 1. a generic HPy_GcMalloc() OR - // 2. HPy_{Un}TrackMemory(), so that we can add manually allocated - // memory as a GC root - UHPy uh_on_invalid_handle; - HPy_ssize_t closed_handles_queue_max_size; // configurable by the user - HPy_ssize_t protected_raw_data_max_size; - HPy_ssize_t protected_raw_data_size; - // Limit for the stack traces captured for allocated handles - // Value 0 implies that stack traces should not be captured - HPy_ssize_t handle_alloc_stacktrace_limit; - DHQueue open_handles; - DHQueue closed_handles; -} HPyDebugInfo; - -static inline HPyDebugInfo *get_info(HPyContext *dctx) -{ - HPyDebugInfo *info = (HPyDebugInfo*)dctx->_private; - assert(info->magic_number == HPY_DEBUG_MAGIC); // sanity check - return info; -} - - -void *raw_data_copy(const void* data, HPy_ssize_t size, bool write_protect); -void raw_data_protect(void* data, HPy_ssize_t size); -/* Return value: 0 indicates success, any different value indicates an error */ -int raw_data_free(void *data, HPy_ssize_t size); - -void create_stacktrace(char **target, HPy_ssize_t max_frames_count); - -#endif /* HPY_DEBUG_INTERNAL_H */ diff --git a/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h b/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h deleted file mode 100644 index c68abbe677..0000000000 --- a/graalpython/com.oracle.graal.python.cext/hpy/hpy_debug.h +++ /dev/null @@ -1,59 +0,0 @@ -#ifndef HPY_DEBUG_H -#define HPY_DEBUG_H - -#include "hpy.h" - -/* - This is the main public API for the debug mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique dctx which - wraps it. - - If you call hpy_debug_get_ctx twice on the same uctx, you get the same - result. - - IMPLEMENTATION NOTE: at the moment of writing, the only known user of the - debug mode is CPython's hpy.universal: in that module, the uctx is a - statically allocated singleton, so for simplicity of implementation - currently we do the same inside debug_ctx.c, with a sanity check to ensure - that we don't call hpy_debug_get_ctx with different uctxs. But this is a - limitation of the current implementation and users should not rely on it. It - is likely that we will need to change it in the future, e.g. if we want to - have per-subinterpreter uctxs. -*/ - -HPyContext * hpy_debug_get_ctx(HPyContext *uctx); -int hpy_debug_ctx_init(HPyContext *dctx, HPyContext *uctx); -void hpy_debug_set_ctx(HPyContext *dctx); - -// convert between debug and universal handles. These are basically -// the same as DHPy_open and DHPy_unwrap but with a different name -// because this is the public-facing API and DHPy/UHPy are only internal -// implementation details. -HPy hpy_debug_open_handle(HPyContext *dctx, HPy uh); -HPy hpy_debug_unwrap_handle(HPyContext *dctx, HPy dh); -void hpy_debug_close_handle(HPyContext *dctx, HPy dh); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -// Copied from Python's exports.h, pyport.h -#ifndef Py_EXPORTED_SYMBOL - #if defined(_WIN32) || defined(__CYGWIN__) - #define Py_EXPORTED_SYMBOL __declspec(dllexport) - #else - #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default"))) - #endif -#endif -#ifdef ___cplusplus -extern "C" -#endif -Py_EXPORTED_SYMBOL -HPy HPyInit__debug(HPyContext *uctx); - -#endif /* HPY_DEBUG_H */ diff --git a/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c b/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c deleted file mode 100644 index 9d0573f36c..0000000000 --- a/graalpython/com.oracle.graal.python.cext/modules/_hpy_debug.c +++ /dev/null @@ -1,403 +0,0 @@ -// Python-level interface for the _debug module. Written in HPy itself, the -// idea is that it should be reusable by other implementations - -// NOTE: hpy.debug._debug is loaded using the UNIVERSAL ctx. To make it -// clearer, we will use "uctx" and "dctx" to distinguish them. - -#include "hpy.h" -#include "debug_internal.h" - -static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, - DebugHandle *handle); - - -HPyDef_METH(new_generation, "new_generation", new_generation_impl, HPyFunc_NOARGS) -static UHPy new_generation_impl(HPyContext *uctx, UHPy self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - info->current_generation++; - return HPyLong_FromLong(uctx, info->current_generation); -} - -static UHPy build_list_of_handles(HPyContext *uctx, UHPy u_self, DHQueue *q, - long gen) -{ - UHPy u_DebugHandleType = HPy_NULL; - UHPy u_result = HPy_NULL; - UHPy u_item = HPy_NULL; - - u_DebugHandleType = HPy_GetAttr_s(uctx, u_self, "DebugHandle"); - if (HPy_IsNull(u_DebugHandleType)) - goto error; - - u_result = HPyList_New(uctx, 0); - if (HPy_IsNull(u_result)) - goto error; - - DebugHandle *dh = q->head; - while(dh != NULL) { - if (dh->generation >= gen) { - UHPy u_item = new_DebugHandleObj(uctx, u_DebugHandleType, dh); - if (HPy_IsNull(u_item)) - goto error; - if (HPyList_Append(uctx, u_result, u_item) == -1) - goto error; - HPy_Close(uctx, u_item); - } - dh = dh->next; - } - - HPy_Close(uctx, u_DebugHandleType); - return u_result; - - error: - HPy_Close(uctx, u_DebugHandleType); - HPy_Close(uctx, u_result); - HPy_Close(uctx, u_item); - return HPy_NULL; -} - - -HPyDef_METH(get_open_handles, "get_open_handles", get_open_handles_impl, HPyFunc_O, .doc= - "Return a list containing all the open handles whose generation is >= " - "of the given arg") -static UHPy get_open_handles_impl(HPyContext *uctx, UHPy u_self, UHPy u_gen) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - - long gen = HPyLong_AsLong(uctx, u_gen); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - - return build_list_of_handles(uctx, u_self, &info->open_handles, gen); -} - -HPyDef_METH(get_closed_handles, "get_closed_handles", get_closed_handles_impl, - HPyFunc_VARARGS, .doc= - "Return a list of all the closed handle in the cache") -static UHPy get_closed_handles_impl(HPyContext *uctx, UHPy u_self, HPy *args, HPy_ssize_t nargs) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - long gen = 0; - if (nargs > 0) { - if (nargs != 1) { - HPyErr_SetString(uctx, uctx->h_TypeError, - "get_closed_handles expects no arguments or exactly one argument"); - return HPy_NULL; - } - gen = HPyLong_AsLong(uctx, args[0]); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - } - return build_list_of_handles(uctx, u_self, &info->closed_handles, gen); -} - -HPyDef_METH(get_closed_handles_queue_max_size, "get_closed_handles_queue_max_size", - get_closed_handles_queue_max_size_impl, HPyFunc_NOARGS, .doc= - "Return the maximum size of the closed handles queue") -static UHPy get_closed_handles_queue_max_size_impl(HPyContext *uctx, UHPy u_self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - return HPyLong_FromSsize_t(uctx, info->closed_handles_queue_max_size); -} - -HPyDef_METH(set_closed_handles_queue_max_size, "set_closed_handles_queue_max_size", - set_closed_handles_queue_max_size_impl, HPyFunc_O, .doc= - "Set the maximum size of the closed handles queue") -static UHPy set_closed_handles_queue_max_size_impl(HPyContext *uctx, UHPy u_self, UHPy u_size) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - HPy_ssize_t size = HPyLong_AsSize_t(uctx, u_size); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - info->closed_handles_queue_max_size = size; - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(get_protected_raw_data_max_size, "get_protected_raw_data_max_size", -get_protected_raw_data_max_size_impl, HPyFunc_NOARGS, .doc= -"Return the maximum size of the retained raw memory associated with closed handles") -static UHPy get_protected_raw_data_max_size_impl(HPyContext *uctx, UHPy u_self) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - return HPyLong_FromSsize_t(uctx, info->protected_raw_data_max_size); -} - -HPyDef_METH(set_protected_raw_data_max_size, "set_protected_raw_data_max_size", -set_protected_raw_data_max_size_impl, HPyFunc_O, .doc= -"Set the maximum size of the retained raw memory associated with closed handles") -static UHPy set_protected_raw_data_max_size_impl(HPyContext *uctx, UHPy u_self, UHPy u_size) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - HPy_ssize_t size = HPyLong_AsSize_t(uctx, u_size); - if (HPyErr_Occurred(uctx)) - return HPy_NULL; - info->protected_raw_data_max_size = size; - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(set_on_invalid_handle, "set_on_invalid_handle", set_on_invalid_handle_impl, - HPyFunc_O, .doc= - "Set the function to call when we detect the usage of an invalid handle") -static UHPy set_on_invalid_handle_impl(HPyContext *uctx, UHPy u_self, UHPy u_arg) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - if (HPy_Is(uctx, u_arg, uctx->h_None)) { - info->uh_on_invalid_handle = HPy_NULL; - } else if (!HPyCallable_Check(uctx, u_arg)) { - HPyErr_SetString(uctx, uctx->h_TypeError, "Expected a callable object"); - return HPy_NULL; - } else { - info->uh_on_invalid_handle = HPy_Dup(uctx, u_arg); - } - return HPy_Dup(uctx, uctx->h_None); -} - -HPyDef_METH(set_handle_stack_trace_limit, "set_handle_stack_trace_limit", - set_handle_stack_trace_limit_impl, HPyFunc_O, .doc= - "Set the limit to captured HPy handles allocations stack traces. " - "None means do not capture the stack traces.") -static UHPy set_handle_stack_trace_limit_impl(HPyContext *uctx, UHPy u_self, UHPy u_arg) -{ - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPyDebugInfo *info = get_info(dctx); - if (HPy_Is(uctx, u_arg, uctx->h_None)) { - info->handle_alloc_stacktrace_limit = 0; - } else { - assert(!HPyErr_Occurred(uctx)); - HPy_ssize_t newlimit = HPyLong_AsSsize_t(uctx, u_arg); - if (newlimit == -1 && HPyErr_Occurred(uctx)) { - return HPy_NULL; - } - info->handle_alloc_stacktrace_limit = newlimit; - } - return HPy_Dup(uctx, uctx->h_None); -} - - -/* ~~~~~~ DebugHandleType and DebugHandleObject ~~~~~~~~ - - This is the applevel view of a DebugHandle/DHPy. - - Note that there are two different ways to expose DebugHandle to applevel: - - 1. make DebugHandle itself a Python object: this is simple but means that - you have to pay the PyObject_HEAD overhead (16 bytes) for all of them - - 2. make DebugHandle a plain C struct, and expose them through a - Python-level wrapper. - - We choose to implement solution 2 because we expect to have many - DebugHandle around, but to expose only few of them to applevel, when you - call get_open_handles. This way, we save 16 bytes per DebugHandle. - - This means that you can have different DebugHandleObjects wrapping the same - DebugHandle. To make it easier to compare them, they expose the .id - attribute, which is the address of the wrapped DebugHandle. Also, - DebugHandleObjects compare equal if their .id is equal. -*/ - -typedef struct { - DebugHandle *handle; -} DebugHandleObject; - -HPyType_HELPERS(DebugHandleObject) - -HPyDef_GET(DebugHandle_obj, "obj", DebugHandle_obj_get, - .doc="The object which the handle points to") -static UHPy DebugHandle_obj_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPy_Dup(uctx, dh->handle->uh); -} - -HPyDef_GET(DebugHandle_id, "id", DebugHandle_id_get, - .doc="A numeric identifier representing the underlying universal handle") -static UHPy DebugHandle_id_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPyLong_FromSsize_t(uctx, (HPy_ssize_t)dh->handle); -} - -HPyDef_GET(DebugHandle_is_closed, "is_closed", DebugHandle_is_closed_get, - .doc="Self-explanatory") -static UHPy DebugHandle_is_closed_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - return HPyBool_FromLong(uctx, dh->handle->is_closed); -} - -HPyDef_GET(DebugHandle_raw_data_size, "raw_data_size", DebugHandle_raw_data_size_get, -.doc="Size of retained raw memory. FOR TESTS ONLY.") -static UHPy DebugHandle_raw_data_size_get(HPyContext *uctx, UHPy self, void *closure) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - if (dh->handle->associated_data) { - return HPyLong_FromSsize_t(uctx, dh->handle->associated_data_size); - } else { - return HPyLong_FromLong(uctx, -1); - } -} - -HPyDef_SLOT(DebugHandle_cmp, DebugHandle_cmp_impl, HPy_tp_richcompare) -static UHPy DebugHandle_cmp_impl(HPyContext *uctx, UHPy self, UHPy o, HPy_RichCmpOp op) -{ - UHPy T = HPy_Type(uctx, self); - if (!HPy_TypeCheck(uctx, o, T)) - return HPy_Dup(uctx, uctx->h_NotImplemented); - DebugHandleObject *dh_self = DebugHandleObject_AsStruct(uctx, self); - DebugHandleObject *dh_o = DebugHandleObject_AsStruct(uctx, o); - - switch(op) { - case HPy_EQ: - return HPyBool_FromLong(uctx, dh_self->handle == dh_o->handle); - case HPy_NE: - return HPyBool_FromLong(uctx, dh_self->handle != dh_o->handle); - default: - return HPy_Dup(uctx, uctx->h_NotImplemented); - } -} - -HPyDef_SLOT(DebugHandle_repr, DebugHandle_repr_impl, HPy_tp_repr) -static UHPy DebugHandle_repr_impl(HPyContext *uctx, UHPy self) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - UHPy uh_fmt = HPy_NULL; - UHPy uh_id = HPy_NULL; - UHPy uh_args = HPy_NULL; - UHPy uh_result = HPy_NULL; - UHPy h_trace_header = HPy_NULL; - UHPy h_trace = HPy_NULL; - - const char *fmt = NULL; - if (dh->handle->is_closed) - fmt = "\n%s%s"; - else - fmt = "\n%s%s"; - - // XXX: switch to HPyUnicode_FromFormat when we have it - uh_fmt = HPyUnicode_FromString(uctx, fmt); - if (HPy_IsNull(uh_fmt)) - goto exit; - - uh_id = HPyLong_FromSsize_t(uctx, (HPy_ssize_t)dh->handle); - if (HPy_IsNull(uh_id)) - goto exit; - - const char *trace_header; - const char *trace; - if (dh->handle->allocation_stacktrace) { - trace_header = "Allocation stacktrace:\n"; - trace = dh->handle->allocation_stacktrace; - } else { - trace_header = "To get the stack trace of where it was allocated use:\nhpy.debug."; - trace = set_handle_stack_trace_limit.meth.name; - } - h_trace_header = HPyUnicode_FromString(uctx, trace_header); - h_trace = HPyUnicode_FromString(uctx, trace); - - if (dh->handle->is_closed) - uh_args = HPyTuple_FromArray(uctx, (UHPy[]){uh_id, - h_trace_header, h_trace}, 3); - else - uh_args = HPyTuple_FromArray(uctx, (UHPy[]){uh_id, dh->handle->uh, - h_trace_header, h_trace}, 4); - if (HPy_IsNull(uh_args)) - goto exit; - - uh_result = HPy_Remainder(uctx, uh_fmt, uh_args); - - exit: - HPy_Close(uctx, uh_fmt); - HPy_Close(uctx, uh_id); - HPy_Close(uctx, uh_args); - HPy_Close(uctx, h_trace); - HPy_Close(uctx, h_trace_header); - return uh_result; -} - - -HPyDef_METH(DebugHandle__force_close, "_force_close", DebugHandle__force_close_impl, - HPyFunc_NOARGS, .doc="Close the underyling handle. FOR TESTS ONLY.") -static UHPy DebugHandle__force_close_impl(HPyContext *uctx, UHPy self) -{ - DebugHandleObject *dh = DebugHandleObject_AsStruct(uctx, self); - HPyContext *dctx = hpy_debug_get_ctx(uctx); - HPy_Close(dctx, as_DHPy(dh->handle)); - return HPy_Dup(uctx, uctx->h_None); -} - -static HPyDef *DebugHandleType_defs[] = { - &DebugHandle_obj, - &DebugHandle_id, - &DebugHandle_is_closed, - &DebugHandle_raw_data_size, - &DebugHandle_cmp, - &DebugHandle_repr, - &DebugHandle__force_close, - NULL -}; - -static HPyType_Spec DebugHandleType_spec = { - .name = "hpy.debug._debug.DebugHandle", - .basicsize = sizeof(DebugHandleObject), - .flags = HPy_TPFLAGS_DEFAULT, - .defines = DebugHandleType_defs, -}; - - -static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType, - DebugHandle *handle) -{ - DebugHandleObject *dhobj; - UHPy u_result = HPy_New(uctx, u_DebugHandleType, &dhobj); - dhobj->handle = handle; - return u_result; -} - - -/* ~~~~~~ definition of the module hpy.debug._debug ~~~~~~~ */ - -static HPyDef *module_defines[] = { - &new_generation, - &get_open_handles, - &get_closed_handles, - &get_closed_handles_queue_max_size, - &set_closed_handles_queue_max_size, - &get_protected_raw_data_max_size, - &set_protected_raw_data_max_size, - &set_on_invalid_handle, - &set_handle_stack_trace_limit, - NULL -}; - -static HPyModuleDef moduledef = { - .name = "hpy.debug._debug", - .doc = "HPy debug mode", - .size = -1, - .defines = module_defines -}; - - -HPy_MODINIT(_debug) -static UHPy init__debug_impl(HPyContext *uctx) -{ - UHPy m = HPyModule_Create(uctx, &moduledef); - if (HPy_IsNull(m)) - return HPy_NULL; - - UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL); - if (HPy_IsNull(h_DebugHandleType)) - return HPy_NULL; - HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType); - HPy_Close(uctx, h_DebugHandleType); - return m; -} diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h b/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h deleted file mode 100644 index 81d60ed7d5..0000000000 --- a/graalpython/com.oracle.graal.python.hpy/include/hpy/version.h +++ /dev/null @@ -1,4 +0,0 @@ - -// automatically generated by setup.py:get_scm_config() -#define HPY_VERSION "0.9.1.dev79+gb0fbdf73" -#define HPY_GIT_REVISION "b0fbdf73" diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h b/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h deleted file mode 100644 index f2fb9471b3..0000000000 --- a/graalpython/com.oracle.graal.python.jni/src/debug/include/hpy_debug.h +++ /dev/null @@ -1,57 +0,0 @@ -#ifndef HPY_DEBUG_H -#define HPY_DEBUG_H - -#include "hpy.h" - -/* - This is the main public API for the debug mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique dctx which - wraps it. - - If you call hpy_debug_get_ctx twice on the same uctx, you get the same - result. - - IMPLEMENTATION NOTE: at the moment of writing, the only known user of the - debug mode is CPython's hpy.universal: in that module, the uctx is a - statically allocated singleton, so for simplicity of implementation - currently we do the same inside debug_ctx.c, with a sanity check to ensure - that we don't call hpy_debug_get_ctx with different uctxs. But this is a - limitation of the current implementation and users should not rely on it. It - is likely that we will need to change it in the future, e.g. if we want to - have per-subinterpreter uctxs. -*/ - -HPyContext * hpy_debug_get_ctx(HPyContext *uctx); -int hpy_debug_ctx_init(HPyContext *dctx, HPyContext *uctx); -void hpy_debug_set_ctx(HPyContext *dctx); - -// convert between debug and universal handles. These are basically -// the same as DHPy_open and DHPy_unwrap but with a different name -// because this is the public-facing API and DHPy/UHPy are only internal -// implementation details. -HPy hpy_debug_open_handle(HPyContext *dctx, HPy uh); -HPy hpy_debug_unwrap_handle(HPyContext *dctx, HPy dh); -void hpy_debug_close_handle(HPyContext *dctx, HPy dh); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -HPyModuleDef* HPyInit__debug(); - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -void HPyInitGlobalContext__debug(HPyContext *ctx); - -#endif /* HPY_DEBUG_H */ diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h b/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h deleted file mode 100644 index e75b76186e..0000000000 --- a/graalpython/com.oracle.graal.python.jni/src/trace/include/hpy_trace.h +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef HPY_TRACE_H -#define HPY_TRACE_H - -#include "hpy.h" - -/* - This is the main public API for the trace mode, and it's meant to be used - by hpy.universal implementations (including but not limited to the - CPython's version of hpy.universal which is included in this repo). - - The idea is that for every uctx there is a corresponding unique tctx which - wraps it. - - If you call hpy_trace_get_ctx twice on the same uctx, you get the same - result. -*/ - -HPyContext * hpy_trace_get_ctx(HPyContext *uctx); -int hpy_trace_ctx_init(HPyContext *tctx, HPyContext *uctx); -int hpy_trace_ctx_free(HPyContext *tctx); -int hpy_trace_get_nfunc(void); -const char * hpy_trace_get_func_name(int idx); - -// this is the HPy init function created by HPy_MODINIT. In CPython's version -// of hpy.universal the code is embedded inside the extension, so we can call -// this function directly instead of dlopen it. This is similar to what -// CPython does for its own built-in modules. But we must use the same -// signature as HPy_MODINIT - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -HPyModuleDef* HPyInit__trace(); - -#ifdef ___cplusplus -extern "C" -#endif -HPy_EXPORTED_SYMBOL -void HPyInitGlobalContext__trace(HPyContext *ctx); - -#endif /* HPY_TRACE_H */ diff --git a/graalpython/hpy/.gitattributes b/graalpython/hpy/.gitattributes new file mode 100644 index 0000000000..1c5f632434 --- /dev/null +++ b/graalpython/hpy/.gitattributes @@ -0,0 +1 @@ +**/autogen_* linguist-generated=true diff --git a/graalpython/hpy/.github/FUNDING.yml b/graalpython/hpy/.github/FUNDING.yml new file mode 100644 index 0000000000..950b3021c6 --- /dev/null +++ b/graalpython/hpy/.github/FUNDING.yml @@ -0,0 +1 @@ +open_collective: hpy diff --git a/graalpython/hpy/.github/workflows/ci.yml b/graalpython/hpy/.github/workflows/ci.yml new file mode 100644 index 0000000000..0ec548c068 --- /dev/null +++ b/graalpython/hpy/.github/workflows/ci.yml @@ -0,0 +1,427 @@ +--- +name: tests + +on: [pull_request] + +jobs: + main_tests: + name: Main tests ${{ matrix.os }} ${{ matrix.python-version }} ${{ matrix.compiler }} + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + # Duplicate changes to this matrix to 'poc_tests' + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.9', '3.10'] + compiler: [""] + include: + - os: ubuntu-latest + python-version: '3.8' + - os: ubuntu-latest + python-version: '3.10' + compiler: 'g++' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/ccache.yml + # parameters: + # pythonVersion: $(python.version) + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: $(python.version) + + - name: Set up Python + uses: actions/setup-python@v5.4 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel 'setuptools>=60.2' + + - name: Build + run: | + make + python -m pip install . + + - if: ${{ matrix.compiler }} + # Only set the compiler for the tests, not for the build + run: echo "CC=${{ matrix.compiler }}" >> $GITHUB_ENV + + - name: Run tests + run: | + python -m pip install pytest pytest-xdist filelock + python -m pytest --basetemp=.tmpdir --durations=16 -n auto test/ + + main_tests_debug: + name: Main tests on CPython debug builds + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + # Cannot install 3.12 on ubuntu-latest + python-version: ['3.11', '3.10', '3.9', '3.8'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python from deadsnakes + uses: deadsnakes/action@v3.2.0 + with: + python-version: ${{ matrix.python-version }} + debug: true + + - name: Check Python debug build + run: python -c "import sys; print(hasattr(sys, 'gettotalrefcount'))" + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Build + run: | + make + python -m pip install . + + - name: Run tests + run: | + python -m pip install pytest pytest-xdist filelock + python -m pytest --durations=16 -n auto test/ + + poc_tests: + name: Proof of concept tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.10'] + include: + - os: ubuntu-latest + python-version: '3.8' + - os: ubuntu-latest + python-version: '3.9' + - os: ubuntu-latest + python-version: '3.11' + - os: ubuntu-latest + python-version: '3.12' + + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/ccache.yml + # parameters: + # pythonVersion: $(python.version) + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: $(python.version) + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + allow-prereleases: true + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython bdist_wheel' + run: proof-of-concept/test_pof.sh wheel cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal bdist_wheel' + run: proof-of-concept/test_pof.sh wheel universal + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython install' + run: proof-of-concept/test_pof.sh setup_py_install cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal install' + run: proof-of-concept/test_pof.sh setup_py_install universal + shell: bash + + - name: 'Test setup.py --hpy-abi=cpython build_ext --inplace' + run: proof-of-concept/test_pof.sh setup_py_build_ext_inplace cpython + shell: bash + + - name: 'Test setup.py --hpy-abi=universal build_ext --inplace' + run: proof-of-concept/test_pof.sh setup_py_build_ext_inplace universal + shell: bash + + + porting_example_tests: + name: Porting example tests + runs-on: ${{ matrix.os }} + continue-on-error: true + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.9'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: Install HPy + run: python -m pip install . + + - name: Install pytest + run: | + python -m pip install pytest + + - name: Run tests + run: make porting-example-tests + shell: bash + + - name: Run tests of completed port in debug mode + env: + HPY_DEBUG: "1" + TEST_ARGS: "-s -k hpy_final" + run: make porting-example-tests + shell: bash + + + valgrind_tests_1: + name: 'Valgrind tests (1/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '1/3' + + + valgrind_tests_2: + name: 'Valgrind tests (2/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '2/3' + + + valgrind_tests_3: + name: 'Valgrind tests (3/3)' + uses: ./.github/workflows/valgrind-tests.yml + with: + portion: '3/3' + + + docs_examples_tests: + name: Documentation examples tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-13, windows-latest] + python-version: ['3.10'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + shell: bash + + - name: Install HPy + run: python -m pip install . + + - name: Install pytest + run: | + python -m pip install pytest pytest-xdist filelock + - name: Run tests + run: make docs-examples-tests + shell: bash + + build_docs: + name: Build documentation + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Install / Upgrade system requirements + run: sudo apt update && sudo apt install -y libclang-17-dev + + - name: Install / Upgrade Python requirements + run: | + python -m pip install --upgrade pip + python -m pip install -r docs/requirements.txt + + - name: Build docs + run: | + cd docs; + python -m sphinx -T -W -E -b html -d _build/doctrees -D language=en . _build/html + + - name: Upload built HTML files + uses: actions/upload-artifact@v4 + with: + name: hpy_html_docs + path: docs/_build/html/* + if-no-files-found: error + retention-days: 5 + + c_tests: + name: C tests + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + - run: make -C c_test + + + check_autogen: + name: Check autogen + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Set up Python + uses: actions/setup-python@v5 + with: + # autogen needs distutils + python-version: '3.11' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Install autogen dependencies + run: pip install -r requirements-autogen.txt + + - name: make autogen + run: | + make autogen + if [ -z "$(git status --porcelain)" ]; then + # clean working copy + echo "Working copy is clean, everything ok" + else + # Uncommitted changes + echo "ERROR: uncommitted changes after running make autogen" + echo "git status" + git status + echo + echo "git diff" + git diff + exit 1 + fi + + + check_py27_compat: + name: Check Python 2.7 compatibility + runs-on: 'ubuntu-20.04' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + # parameters: + # pythonVersion: "2.7" + + - name: Set up Python2 + # Copied from cython's ci.yml + run: | + sudo ln -fs python2 /usr/bin/python + sudo apt-get update + sudo apt-get install python-setuptools python2.7 python2.7-dev + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py + sudo python2 get-pip.py + ls -l /usr/bin/pip* /usr/local/bin/pip* + which pip + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: check_py27_compat.py + run: | + python -m pip install pytest pytest-xdist filelock pathlib + python test/check_py27_compat.py + + + cpp_check: + name: Cppcheck static analysis + runs-on: 'ubuntu-22.04' + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install Cppcheck + run: sudo apt-get -qq -y install cppcheck=2.7-1 + + - name: Run Cppcheck + run: make cppcheck + + + infer: + name: Infer static analysis + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + # - template: azure-templates/python.yml + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Install Infer + run: | + python -m pip install compiledb wheel; + VERSION=1.1.0; \ + curl -sSL "https://github.com/facebook/infer/releases/download/v$VERSION/infer-linux64-v$VERSION.tar.xz" \ + | sudo tar -C /opt -xJ && \ + echo "/opt/infer-linux64-v$VERSION/bin" >> $GITHUB_PATH + + - name: Run Infer + run: make infer + + check_microbench: + name: Check micro benchmarks + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install / Upgrade system dependencies + run: sudo apt update && sudo apt install -y valgrind + + - name: Install / Upgrade Python requirements + run: | + python -m pip install --upgrade pip wheel 'setuptools>=60.2' + python -m pip install pytest cffi + + - name: Build and install HPy + run: | + make + python -m pip install . + + - name: Run microbenchmarks + run: | + cd microbench + python setup.py build_ext -i + python -m pytest -v diff --git a/graalpython/hpy/.github/workflows/release-pypi.yml b/graalpython/hpy/.github/workflows/release-pypi.yml new file mode 100644 index 0000000000..959a779170 --- /dev/null +++ b/graalpython/hpy/.github/workflows/release-pypi.yml @@ -0,0 +1,136 @@ +--- +name: Publish to PyPI + +# manually trigger this workflow +on: + workflow_dispatch: + inputs: + tag: + description: 'The Git tag to create or test a release for. Official releases should always be made from an existing tag.' + required: true + type: string + official: + description: 'If true, publish to official PyPI and create GitHub release. Otherwise publish only to test PyPI.' + default: false + type: boolean + +jobs: + build_sdist: + name: Build source package + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + + - name: Set up Python + uses: actions/setup-python@v4 + with: + # HPy requires at least Python 3.8; we mostly work with >=3.10 + python-version: '>=3.10' + + - name: Install/Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel 'setuptools>=60.2' + + - name: Build and install Python source package + run: | + python setup.py sdist + python -m pip install dist/*.tar.gz + + - name: Run tests + run: | + make + pip install pytest pytest-xdist filelock + pytest -n auto test/ + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + retention-days: 5 + + + build_bdist: + name: Build binary wheels + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Windows tests fail when built as a binary wheel for some reason + # 'macos-12' doesn't pass the tests + os: [ubuntu-latest, macos-11] # windows-latest + + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.tag }} + + # setup Python for cibuildwheel + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + # for other architectures, see: https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation + - name: Build wheels for CPython + uses: pypa/cibuildwheel@v2.12.3 + env: + # cibuildwheel automatically reads 'python_requires' from 'setup.py' + CIBW_BUILD: 'cp*' # no PyPy builds + CIBW_ARCHS_LINUX: "x86_64" # only Intel 64-bit + CIBW_ARCHS_MACOS: "x86_64" # only Intel 64-bit + CIBW_TEST_REQUIRES: pytest pytest-xdist filelock setuptools>=60.2 + # only copy test dir to current working dir + CIBW_TEST_COMMAND: cp -R {project}/test ./ && pytest --basetemp=.tmpdir --ignore=test/hpy_devel -n auto ./test/ + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + retention-days: 5 + + upload_pypi: + name: Publish packages to (Test) PyPI + needs: [build_sdist, build_bdist] + runs-on: ubuntu-latest + # don't do this action on forks by default + if: github.repository == 'hpyproject/hpy' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Upload to Test PyPI + uses: pypa/gh-action-pypi-publish@v1.8.5 + if: ${{ !inputs.official }} + with: + verbose: true + user: __token__ + password: ${{ secrets.TEST_PYPI_API_TOKEN }} + repository-url: https://test.pypi.org/legacy/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@v1.8.5 + if: ${{ inputs.official }} + with: + verbose: true + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + + create_gh_release: + needs: [upload_pypi] + runs-on: ubuntu-latest + # don't do this action on forks by default + if: github.repository == 'hpyproject/hpy' + steps: + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: dist/* + # consider tags in form '*.*.*rc*' to indicate a pre-release + prerelease: ${{ contains(inputs.tag, 'rc') }} + draft: ${{ !inputs.official }} + tag_name: ${{ inputs.tag }} diff --git a/graalpython/hpy/.github/workflows/valgrind-tests.yml b/graalpython/hpy/.github/workflows/valgrind-tests.yml new file mode 100644 index 0000000000..badbc9da6e --- /dev/null +++ b/graalpython/hpy/.github/workflows/valgrind-tests.yml @@ -0,0 +1,38 @@ +on: + workflow_call: + inputs: + portion: + description: 'Select portion of tests to run under Valgrind (default runs all)' + default: '' + type: string + required: false + +jobs: + valgrind_tests: + # name: Valgrind tests + runs-on: 'ubuntu-latest' + steps: + - uses: actions/checkout@v4 + + - name: Install / Upgrade system dependencies + run: sudo apt update && sudo apt install -y valgrind + + - name: Set up Python + uses: actions/setup-python@v5.4 + with: + python-version: 3.9 + + - name: Install / Upgrade Python dependencies + run: python -m pip install --upgrade pip wheel + + - name: Build + run: | + make + python -m pip install . + + - name: Run tests + env: + HPY_TEST_PORTION: ${{ inputs.portion }} + run: | + python -m pip install pytest pytest-valgrind pytest-portion filelock + make valgrind diff --git a/graalpython/hpy/.gitignore b/graalpython/hpy/.gitignore new file mode 100644 index 0000000000..2877066982 --- /dev/null +++ b/graalpython/hpy/.gitignore @@ -0,0 +1,92 @@ +# HPy autogen +hpy/tools/autogen/autogen_pypy.txt + +# generated by setup.py:get_scm_config() +hpy/devel/include/hpy/version.h +hpy/devel/version.py + +# stubs created by setup.py when doing build_ext --inplace +proof-of-concept/pof.py +proof-of-concept/pofpackage/foo.py + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions and static libs +*.so +*.o +*.a +*.lib +vc140.pdb +c_test/test_debug_handles + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +test-output.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +.hpy.lock + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# cppcheck +.cppcheck + +# Sphinx +docs/_build/ + +# vscode +.vscode diff --git a/graalpython/hpy/.readthedocs.yml b/graalpython/hpy/.readthedocs.yml new file mode 100644 index 0000000000..7ee9eedb2e --- /dev/null +++ b/graalpython/hpy/.readthedocs.yml @@ -0,0 +1,10 @@ +# readthedocs.org configuration + +version: 2 +sphinx: + configuration: docs/conf.py +formats: all +python: + version: 3.7 + install: + - requirements: docs/requirements.txt diff --git a/graalpython/hpy/AUTHORS b/graalpython/hpy/AUTHORS new file mode 100644 index 0000000000..707ed38f94 --- /dev/null +++ b/graalpython/hpy/AUTHORS @@ -0,0 +1,38 @@ +# This is the list of HPy's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Alexander Schremmer +Ammar Askar +Ankith (ankith26) +Antonio Cuni +Armin Rigo +Charlie Hayden +Christian Klein +Daniel Alley +Dominic Davis-Foster +Jaime Resano Aísa +Joachim Trouverie +Julian Berman +Krish Patel +Kyle Lawlor-Bagcal +Mathieu Dupuy +Matti Picus +Max Horn +Maxwell Bernstein +Mohamed Koubaa +Nico Rittinghaus +Omer Katz +Oracle and/or its affiliates +Paul Prescod +Pierre Augier +Robert O'Shea +Ronan Lamy +Simon Cross +Stefan Behnel +Stefano Rivera +Victor Stinner +Vytautas Liuolia +Łukasz Langa diff --git a/graalpython/hpy/CONTRIBUTING.md b/graalpython/hpy/CONTRIBUTING.md new file mode 100644 index 0000000000..7178db511c --- /dev/null +++ b/graalpython/hpy/CONTRIBUTING.md @@ -0,0 +1,55 @@ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +### Report Bugs + +Report bugs at . + +When reporting a bug, please include: + +- Your operating system name and version. +- Any details about your local setup that might be helpful in + troubleshooting. +- Detailed steps to reproduce the bug. + +### Donate to HPy + +Become a financial contributor and help us sustain the HPy community: [Contribute to HPy](https://opencollective.com/hpy/contribute). + +### Fix Bugs or Implement Features + +Look through the GitHub issues for bugs or proposals being discussed +or ready for implementation. + +### Write Documentation + +HPy could always use more documentation, whether as part of the +official HPy docs, in docstrings, or even on the web in blog +posts, articles, and such. + +Get Started! +------------ + +You can test your environment by running + +```bash +proof-of-concept/test_pof.sh wheel universal +``` + +This should build HPy and a proof of concept (demo) extension and +then run a pytest that validates both. + +Until we get around to more comprehensive documentation, you +can reverse engineer how the system works by looking at these +files: + +- `proof-of-concept/test_pof.sh` +- `proof-of-concept/setup.py` diff --git a/graalpython/hpy/LICENSE b/graalpython/hpy/LICENSE new file mode 100644 index 0000000000..a5b403aa3f --- /dev/null +++ b/graalpython/hpy/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 The HPy Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/graalpython/hpy/MANIFEST.in b/graalpython/hpy/MANIFEST.in new file mode 100644 index 0000000000..7a4b248c98 --- /dev/null +++ b/graalpython/hpy/MANIFEST.in @@ -0,0 +1 @@ +recursive-include hpy/devel/include *.h diff --git a/graalpython/hpy/Makefile b/graalpython/hpy/Makefile new file mode 100644 index 0000000000..6d0f1c0db8 --- /dev/null +++ b/graalpython/hpy/Makefile @@ -0,0 +1,86 @@ +.PHONY: all +all: hpy.universal + +.PHONY: hpy.universal +hpy.universal: + python3 setup.py build_clib -f build_ext -if + +.PHONY: dist-info +dist-info: + python3 setup.py dist_info + +debug: + HPY_DEBUG_BUILD=1 make all + +autogen: + python3 -m hpy.tools.autogen . + +cppcheck-build-dir: + mkdir -p $(or ${CPPCHECK_BUILD_DIR}, .cppcheck) + +.PHONY: cppcheck +cppcheck: cppcheck-build-dir + # azure pipelines doesn't show stderr, so we write the errors to a file and cat it later :( + $(eval PYTHON_INC = $(shell python3 -q -c "from sysconfig import get_paths as gp; print(gp()['include'])")) + $(eval PYTHON_PLATINC = $(shell python3 -q -c "from sysconfig import get_paths as gp; print(gp()['platinclude'])")) + cppcheck --version + cppcheck \ + -v \ + --error-exitcode=1 \ + --cppcheck-build-dir=$(or ${CPPCHECK_BUILD_DIR}, .cppcheck) \ + --enable=warning,performance,portability,information,missingInclude \ + --inline-suppr \ + --suppress=syntaxError \ + -I /usr/local/include/ \ + -I /usr/include/ \ + -I ${PYTHON_INC} \ + -I ${PYTHON_PLATINC} \ + -I . \ + -I hpy/devel/include/ \ + -I hpy/devel/include/hpy/ \ + -I hpy/devel/include/hpy/cpython/ \ + -I hpy/devel/include/hpy/universal/ \ + -I hpy/devel/include/hpy/runtime/ \ + -I hpy/universal/src/ \ + -I hpy/debug/src/ \ + -I hpy/debug/src/include \ + -I hpy/trace/src/ \ + -I hpy/trace/src/include \ + --force \ + -D NULL=0 \ + -D HPY_ABI_CPYTHON \ + -D __linux__=1 \ + -D __x86_64__=1 \ + -D __LP64__=1 \ + . + +infer: + python3 setup.py build_ext -if -U NDEBUG | compiledb + # see commit cd8cd6e for why we need to ignore debug_ctx.c + @infer --fail-on-issue --compilation-database compile_commands.json --report-blacklist-path-regex "hpy/debug/src/debug_ctx.c" + +valgrind_args = --suppressions=hpy/tools/valgrind/python.supp --suppressions=hpy/tools/valgrind/hpy.supp --leak-check=full --show-leak-kinds=definite,indirect --log-file=/tmp/valgrind-output +python_args = -m pytest --valgrind --valgrind-log=/tmp/valgrind-output + +.PHONY: valgrind +valgrind: +ifeq ($(HPY_TEST_PORTION),) + PYTHONMALLOC=malloc valgrind $(valgrind_args) python3 $(python_args) test/ +else + PYTHONMALLOC=malloc valgrind $(valgrind_args) python3 $(python_args) --portion $(HPY_TEST_PORTION) test/ +endif + +porting-example-tests: + cd docs/porting-example/steps && python3 setup00.py build_ext -i + cd docs/porting-example/steps && python3 setup01.py build_ext -i + cd docs/porting-example/steps && python3 setup02.py build_ext -i + cd docs/porting-example/steps && python3 setup03.py --hpy-abi=universal build_ext -i + python3 -m pytest docs/porting-example/steps/ ${TEST_ARGS} + +docs-examples-tests: + cd docs/examples/simple-example && python3 setup.py --hpy-abi=universal install + cd docs/examples/mixed-example && python3 setup.py install + cd docs/examples/snippets && python3 setup.py --hpy-abi=universal install + cd docs/examples/quickstart && python3 setup.py --hpy-abi=universal install + cd docs/examples/hpytype-example && python3 setup.py --hpy-abi=universal install + python3 -m pytest docs/examples/tests.py ${TEST_ARGS} diff --git a/graalpython/hpy/README-gdb.md b/graalpython/hpy/README-gdb.md new file mode 100644 index 0000000000..c1bacca196 --- /dev/null +++ b/graalpython/hpy/README-gdb.md @@ -0,0 +1,207 @@ +How to debug HPy on CPython +============================ + +This document describes how to make debugging easier when running HPy on +CPython. At the moment it is structured as a collection of notes, PRs to make +it more structured are welcome. + +**NOTE**: this document is **not** about the HPy debug mode, but it's about +debugging HPy itself. + + +Build a debug version of CPython +--------------------------------- + +This is highly recommended. It takes only few minutes and helps a lot during +debugging for two reasons: + +1. You can easily view CPython's source code inside GDB + +2. CPython is compiled with `-Og`, which means it will be easier to follow the + code step-by-step and to inspect variables + +3. The `py-*` GDB commands work out of the box. + +``` +$ cd /path/to/cpython +$ git checkout v3.8.2 +$ ./configure --with-pydebug --prefix=/opt/python-debug/ +$ make install +``` + +Enable `python-gdb.py` +---------------------- + +`python-gdb.py` is a GDB script to make it easier to inspect the state of a +Python process from whitin GDB. It is documented +[in the CPython's dev guide](https://devguide.python.org/gdb/). + +The script add a series of GBD commands such as: + + - `py-bt`: prints the Python-level traceback of the current function + + - `py-up`, `py-down`: navigate up and down the Python function stack by + going to the previous/next occurrence of `_PyEval_EvalFrameDefault`. They + are more or less equivalent to `up` and `down` inside `pdb`. + + - `py-print`: print the value of a Python-level variable + +**WARNING**: the CPython's dev guide suggests to add `add-auto-load-safe-path` +to your `~/.gdbinit`, but it doesn't work for me. What works for me is the +following: + +``` +# add this to your ~/.gdbinit +source /path/to/cpython/python-gdb.py +``` + +To check that the `py-*` commands work as expected, you can do the following: + +``` +$ gdb --args /opt/python-debug/bin/python3 -c 'for i in range(10000000): pass' +GNU gdb (Ubuntu 9.2-0ubuntu1~20.04) 9.2 +... +(gdb) run # start the python process +... + +^C +... +(gdb) py-bt +Traceback (most recent call first): + File "", line 1, in +(gdb) py-print i +global 'i' = 9657683 +``` + +Inspect `PyObject *` and `HPy` inside GDB +------------------------------------------ + +**WARNING**: `py-dump` and `hpy-dump` prints to stderr, and this interacts +badly with pytest's capturing. Make sure to run `py.test -s`, else you might +not see the output. The included script `gdb-py.test` automatically pass `-s`. + +`python-gdb.py` installs a GDB pretty-printer for `PyObject *` variables, +which sometimes can be confusing: often, it prints the Python repr of the +variable: + +``` +(gdb) set $x = PyLong_FromLong(42) +(gdb) p $x +$5 = 42 +(gdb) p (void*)$x +$7 = (void *) 0x555555903ca0 +(gdb) p *$x +$6 = {ob_refcnt = 10, ob_type = 0x5555558d5560 } +``` + +For some reason which is unknown to me, sometimes it prints a rather obscure +string: here, `type` is the Python type of the object, and `0x5555558ce88` is +its address: + +``` +(gdb) p PyExc_ValueError +$12 = +(gdb) p (void*)PyExc_ValueError +$13 = (void *) 0x5555558ce880 <_PyExc_ValueError> +``` + +Another useful trick is to define these two custom GDB commands in your +`~/.gdbinit`: + +``` +# put this in your ~/.gdbinit +define py-dump +call _PyObject_Dump($arg0) +end + +# NOTE: +# 1. this assumes that you have a variable called "ctx" available +# 2. this assumes that it's a debug version of HPy, which was compiled with +# -fkeep-inline-functions +# 3. if you don't have _HPy_Dump available, you can call manually +# ctx->ctx_Dump (but only in universal mode) +define hpy-dump +call _HPy_Dump(ctx, $arg0) +end +``` + +Example of usage: + +``` +(gdb) set $x = PyLong_FromLong(42) +(gdb) py-dump $x +object address : 0x555555903ca0 +object refcount : 11 +object type : 0x5555558d5560 +object type name: int +object repr : 42 + +(gdb) py-dump PyExc_ValueError +object address : 0x5555558ce880 +object refcount : 14 +object type : 0x5555558dd8e0 +object type name: type +object repr : + +(gdb) set $h = ctx->ctx_Long_FromLong(ctx, 1234) +(gdb) p $h +$2 = {_i = 77} +(gdb) hpy-dump $h +object address : 0x7ffff64b0580 +object refcount : 1 +object type : 0x5555558d5560 +object type name: int +object repr : 1234 +``` + + + +Create a venv for python-debug and install hpy +----------------------------------------------- + +The following commands create a `venv` based on the newly built +`python-debug`, and installs `hpy` in "editable mode". + +``` +$ cd /path/to/hpy +$ /opt/python-debug/bin/python3 -m venv venv/hpy-debug +$ . venv/hpy-debug/bin/activate +$ python setup.py develop +``` + +Once hpy is installed this way, you can edit the C files and rebuild it easily +by using `make`: + +``` +$ make # build normally +$ make debug # build HPY with -O0 -g +``` + +Run a specific HPy test under GDB +--------------------------------- + +To run `py.test` under GDB, use the script `./gdb-py.test`. + +**Tip:** most HPy tests export a Python function called `f`. You can easily +put a breakpoint into it by using `b f_impl`: + +``` +$ ./gdb-py.test -s test/test_00_basic.py -k 'test_float_asdouble and universal' +... +(gdb) b f_impl +Breakpoint 1 at 0x7ffff63a0284: file /tmp/pytest-of-antocuni/pytest-219/test_float_asdouble_universal_0/mytest.c, line 5. +(gdb) run +... +Breakpoint 1, f_impl (ctx=0x7ffff64191d0, self=..., arg=...) at /tmp/pytest-of-antocuni/pytest-220/test_float_asdouble_universal_0/mytest.c:5 +5 { +(gdb) next +6 double a = HPyFloat_AsDouble(ctx, arg); +(gdb) p arg +$1 = {_i = 75} +(gdb) hpy-dump arg +object address : 0x7ffff7017700 +object refcount : 3 +object type : 0x5555558d3400 +object type name: float +object repr : 1.0 +``` diff --git a/graalpython/hpy/README.md b/graalpython/hpy/README.md new file mode 100644 index 0000000000..311199e6bc --- /dev/null +++ b/graalpython/hpy/README.md @@ -0,0 +1,81 @@ +# HPy: a better API for Python + +[![Build](https://github.com/hpyproject/hpy/actions/workflows/ci.yml/badge.svg)](https://github.com/hpyproject/hpy/actions/workflows/ci.yml) +[![Documentation](https://readthedocs.org/projects/hpy/badge/)](https://hpy.readthedocs.io/) +[![Join the discord server at https://discord.gg/xSzxUbPkTQ](https://img.shields.io/discord/1077164940906995813.svg?color=7389D8&labelColor=6A7EC2&logo=discord&logoColor=ffffff&style=flat-square)](https://discord.gg/xSzxUbPkTQ) + +**Website**: [hpyproject.org](https://hpyproject.org/) \ +**Community**: [HPy Discord server](https://discord.gg/xSzxUbPkTQ) \ +**Mailing list**: [hpy-dev@python.org](https://mail.python.org/mailman3/lists/hpy-dev.python.org/) + +## Summary + +HPy is a better API for extending Python +in C. The old C API is specific to the current implementation of CPython. +It exposes a lot of internal details which makes it hard to: + + - implement it for other Python implementations (e.g. PyPy, GraalPy, + Jython, IronPython, etc.). + - experiment with new things inside CPython itself: e.g. using a GC + instead of refcounting, or to remove the GIL + - guarantee binary stability + +HPy is a specification of a new API and ABI for extending Python that is +Python implementation agnostic and designed to hide and abstract internal +details such that it: + + - can stay binary compatible even if the underlying Python internals change significantly + - does not hinder internal progress of CPython and other Pythons + +Please read the [documentation](https://docs.hpyproject.org/en/latest/overview.html) +for more details on HPy motivation, goals, and features, for example: + + - debug mode for better developer experience + - support for incremental porting from CPython API to HPy + - CPython ABI for raw performance on CPython + - and others + +Do you want to see how HPy API looks in code? Check out +our [quickstart example](https://docs.hpyproject.org/en/latest/quickstart.html). + +You may also be interested in HPy's +[API reference](https://docs.hpyproject.org/en/latest/api-reference/index.html). + +This repository contains the API and ABI specification and implementation +for the CPython interpreter. Other interpreters that support HPy natively: GraalPy +and PyPy, provide their own builtin HPy implementations. + + +## Why should I care about this stuff? + + - the existing C API is becoming a problem for CPython and for the + evolution of the language itself: this project makes it possible to make + experiments which might be "officially" adopted in the future + + - for PyPy, it will give obvious speed benefits: for example, data + scientists will be able to get the benefit of fast C libraries *and* fast + Python code at the same time, something which is hard to achieve now + + - the current implementation is too tied to CPython and proved to be a + problem for almost all the other alternative implementations. Having an + API which is designed to work well on two different implementations will + make the job much easier for future ones: going from 2 to N is much easier + than going from 1 to 2 + + - arguably, it will be easier to learn and understand than the massive + CPython C API + +See also [Python Performance: Past, Present, +Future](https://github.com/vstinner/talks/raw/main/2019-EuroPython/python_performance.pdf) +by Victor Stinner. + + +## What does `HPy` mean? + +The "H" in `HPy` stands for "handle": one of the key idea of the new API is to +use fully opaque handles to represent and pass around Python objects. + + +## Donate to HPy + +Become a financial contributor and help us sustain the HPy community: [Contribute to HPy](https://opencollective.com/hpy/contribute). diff --git a/graalpython/hpy/c_test/Makefile b/graalpython/hpy/c_test/Makefile new file mode 100644 index 0000000000..2e93170d4c --- /dev/null +++ b/graalpython/hpy/c_test/Makefile @@ -0,0 +1,16 @@ +CC ?= gcc +INCLUDE=-I.. -I../hpy/devel/include -I../hpy/debug/src/include +CFLAGS = -O0 -UNDEBUG -g -Wall -Werror -Wfatal-errors $(INCLUDE) -DHPY_ABI_UNIVERSAL + +test: test_debug_handles test_stacktrace + ./test_debug_handles + ./test_stacktrace + +test_debug_handles: test_debug_handles.o ../hpy/debug/src/dhqueue.o + $(CC) -o $@ $^ + +test_stacktrace: test_stacktrace.o ../hpy/debug/src/stacktrace.o + $(CC) -o $@ $^ + +%.o : %.c + $(CC) -c $(CFLAGS) $< -o $@ diff --git a/graalpython/hpy/c_test/acutest.h b/graalpython/hpy/c_test/acutest.h new file mode 100644 index 0000000000..a05eb41cd0 --- /dev/null +++ b/graalpython/hpy/c_test/acutest.h @@ -0,0 +1,1794 @@ +/* + * Acutest -- Another C/C++ Unit Test facility + * + * + * Copyright 2013-2020 Martin Mitas + * Copyright 2019 Garrett D'Amore + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef ACUTEST_H +#define ACUTEST_H + + +/************************ + *** Public interface *** + ************************/ + +/* By default, "acutest.h" provides the main program entry point (function + * main()). However, if the test suite is composed of multiple source files + * which include "acutest.h", then this causes a problem of multiple main() + * definitions. To avoid this problem, #define macro TEST_NO_MAIN in all + * compilation units but one. + */ + +/* Macro to specify list of unit tests in the suite. + * The unit test implementation MUST provide list of unit tests it implements + * with this macro: + * + * TEST_LIST = { + * { "test1_name", test1_func_ptr }, + * { "test2_name", test2_func_ptr }, + * ... + * { NULL, NULL } // zeroed record marking the end of the list + * }; + * + * The list specifies names of each test (must be unique) and pointer to + * a function implementing it. The function does not take any arguments + * and has no return values, i.e. every test function has to be compatible + * with this prototype: + * + * void test_func(void); + * + * Note the list has to be ended with a zeroed record. + */ +#define TEST_LIST const struct acutest_test_ acutest_list_[] + + +/* Macros for testing whether an unit test succeeds or fails. These macros + * can be used arbitrarily in functions implementing the unit tests. + * + * If any condition fails throughout execution of a test, the test fails. + * + * TEST_CHECK takes only one argument (the condition), TEST_CHECK_ allows + * also to specify an error message to print out if the condition fails. + * (It expects printf-like format string and its parameters). The macros + * return non-zero (condition passes) or 0 (condition fails). + * + * That can be useful when more conditions should be checked only if some + * preceding condition passes, as illustrated in this code snippet: + * + * SomeStruct* ptr = allocate_some_struct(); + * if(TEST_CHECK(ptr != NULL)) { + * TEST_CHECK(ptr->member1 < 100); + * TEST_CHECK(ptr->member2 > 200); + * } + */ +#define TEST_CHECK_(cond,...) acutest_check_((cond), __FILE__, __LINE__, __VA_ARGS__) +#define TEST_CHECK(cond) acutest_check_((cond), __FILE__, __LINE__, "%s", #cond) + + +/* These macros are the same as TEST_CHECK_ and TEST_CHECK except that if the + * condition fails, the currently executed unit test is immediately aborted. + * + * That is done either by calling abort() if the unit test is executed as a + * child process; or via longjmp() if the unit test is executed within the + * main Acutest process. + * + * As a side effect of such abortion, your unit tests may cause memory leaks, + * unflushed file descriptors, and other phenomena caused by the abortion. + * + * Therefore you should not use these as a general replacement for TEST_CHECK. + * Use it with some caution, especially if your test causes some other side + * effects to the outside world (e.g. communicating with some server, inserting + * into a database etc.). + */ +#define TEST_ASSERT_(cond,...) \ + do { \ + if(!acutest_check_((cond), __FILE__, __LINE__, __VA_ARGS__)) \ + acutest_abort_(); \ + } while(0) +#define TEST_ASSERT(cond) \ + do { \ + if(!acutest_check_((cond), __FILE__, __LINE__, "%s", #cond)) \ + acutest_abort_(); \ + } while(0) + + +#ifdef __cplusplus +/* Macros to verify that the code (the 1st argument) throws exception of given + * type (the 2nd argument). (Note these macros are only available in C++.) + * + * TEST_EXCEPTION_ is like TEST_EXCEPTION but accepts custom printf-like + * message. + * + * For example: + * + * TEST_EXCEPTION(function_that_throw(), ExpectedExceptionType); + * + * If the function_that_throw() throws ExpectedExceptionType, the check passes. + * If the function throws anything incompatible with ExpectedExceptionType + * (or if it does not thrown an exception at all), the check fails. + */ +#define TEST_EXCEPTION(code, exctype) \ + do { \ + bool exc_ok_ = false; \ + const char *msg_ = NULL; \ + try { \ + code; \ + msg_ = "No exception thrown."; \ + } catch(exctype const&) { \ + exc_ok_= true; \ + } catch(...) { \ + msg_ = "Unexpected exception thrown."; \ + } \ + acutest_check_(exc_ok_, __FILE__, __LINE__, #code " throws " #exctype);\ + if(msg_ != NULL) \ + acutest_message_("%s", msg_); \ + } while(0) +#define TEST_EXCEPTION_(code, exctype, ...) \ + do { \ + bool exc_ok_ = false; \ + const char *msg_ = NULL; \ + try { \ + code; \ + msg_ = "No exception thrown."; \ + } catch(exctype const&) { \ + exc_ok_= true; \ + } catch(...) { \ + msg_ = "Unexpected exception thrown."; \ + } \ + acutest_check_(exc_ok_, __FILE__, __LINE__, __VA_ARGS__); \ + if(msg_ != NULL) \ + acutest_message_("%s", msg_); \ + } while(0) +#endif /* #ifdef __cplusplus */ + + +/* Sometimes it is useful to split execution of more complex unit tests to some + * smaller parts and associate those parts with some names. + * + * This is especially handy if the given unit test is implemented as a loop + * over some vector of multiple testing inputs. Using these macros allow to use + * sort of subtitle for each iteration of the loop (e.g. outputting the input + * itself or a name associated to it), so that if any TEST_CHECK condition + * fails in the loop, it can be easily seen which iteration triggers the + * failure, without the need to manually output the iteration-specific data in + * every single TEST_CHECK inside the loop body. + * + * TEST_CASE allows to specify only single string as the name of the case, + * TEST_CASE_ provides all the power of printf-like string formatting. + * + * Note that the test cases cannot be nested. Starting a new test case ends + * implicitly the previous one. To end the test case explicitly (e.g. to end + * the last test case after exiting the loop), you may use TEST_CASE(NULL). + */ +#define TEST_CASE_(...) acutest_case_(__VA_ARGS__) +#define TEST_CASE(name) acutest_case_("%s", name) + + +/* Maximal output per TEST_CASE call. Longer messages are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_CASE_MAXSIZE + #define TEST_CASE_MAXSIZE 64 +#endif + + +/* printf-like macro for outputting an extra information about a failure. + * + * Intended use is to output some computed output versus the expected value, + * e.g. like this: + * + * if(!TEST_CHECK(produced == expected)) { + * TEST_MSG("Expected: %d", expected); + * TEST_MSG("Produced: %d", produced); + * } + * + * Note the message is only written down if the most recent use of any checking + * macro (like e.g. TEST_CHECK or TEST_EXCEPTION) in the current test failed. + * This means the above is equivalent to just this: + * + * TEST_CHECK(produced == expected); + * TEST_MSG("Expected: %d", expected); + * TEST_MSG("Produced: %d", produced); + * + * The macro can deal with multi-line output fairly well. It also automatically + * adds a final new-line if there is none present. + */ +#define TEST_MSG(...) acutest_message_(__VA_ARGS__) + + +/* Maximal output per TEST_MSG call. Longer messages are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_MSG_MAXSIZE + #define TEST_MSG_MAXSIZE 1024 +#endif + + +/* Macro for dumping a block of memory. + * + * Its intended use is very similar to what TEST_MSG is for, but instead of + * generating any printf-like message, this is for dumping raw block of a + * memory in a hexadecimal form: + * + * TEST_CHECK(size_produced == size_expected && + * memcmp(addr_produced, addr_expected, size_produced) == 0); + * TEST_DUMP("Expected:", addr_expected, size_expected); + * TEST_DUMP("Produced:", addr_produced, size_produced); + */ +#define TEST_DUMP(title, addr, size) acutest_dump_(title, addr, size) + +/* Maximal output per TEST_DUMP call (in bytes to dump). Longer blocks are cut. + * You may define another limit prior including "acutest.h" + */ +#ifndef TEST_DUMP_MAXSIZE + #define TEST_DUMP_MAXSIZE 1024 +#endif + + +/* Common test initialization/clean-up + * + * In some test suites, it may be needed to perform some sort of the same + * initialization and/or clean-up in all the tests. + * + * Such test suites may use macros TEST_INIT and/or TEST_FINI prior including + * this header. The expansion of the macro is then used as a body of helper + * function called just before executing every single (TEST_INIT) or just after + * it ends (TEST_FINI). + * + * Examples of various ways how to use the macro TEST_INIT: + * + * #define TEST_INIT my_init_func(); + * #define TEST_INIT my_init_func() // Works even without the semicolon + * #define TEST_INIT setlocale(LC_ALL, NULL); + * #define TEST_INIT { setlocale(LC_ALL, NULL); my_init_func(); } + * + * TEST_FINI is to be used in the same way. + */ + + +/********************** + *** Implementation *** + **********************/ + +/* The unit test files should not rely on anything below. */ + +#include +#include +#include +#include +#include +#include + +#if defined(unix) || defined(__unix__) || defined(__unix) || defined(__APPLE__) + #define ACUTEST_UNIX_ 1 + #include + #include + #include + #include + #include + #include + #include + + #if defined CLOCK_PROCESS_CPUTIME_ID && defined CLOCK_MONOTONIC + #define ACUTEST_HAS_POSIX_TIMER_ 1 + #endif +#endif + +#if defined(_gnu_linux_) || defined(__linux__) + #define ACUTEST_LINUX_ 1 + #include + #include +#endif + +#if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) + #define ACUTEST_WIN_ 1 + #include + #include +#endif + +#ifdef __cplusplus + #include +#endif + +#ifdef __has_include + #if __has_include() + #include + #endif +#endif + +/* Enable the use of the non-standard keyword __attribute__ to silence warnings under some compilers */ +#if defined(__GNUC__) || defined(__clang__) + #define ACUTEST_ATTRIBUTE_(attr) __attribute__((attr)) +#else + #define ACUTEST_ATTRIBUTE_(attr) +#endif + +/* Note our global private identifiers end with '_' to mitigate risk of clash + * with the unit tests implementation. */ + +#ifdef __cplusplus + extern "C" { +#endif + +#ifdef _MSC_VER + /* In the multi-platform code like ours, we cannot use the non-standard + * "safe" functions from Microsoft C lib like e.g. sprintf_s() instead of + * standard sprintf(). Hence, lets disable the warning C4996. */ + #pragma warning(push) + #pragma warning(disable: 4996) +#endif + + +struct acutest_test_ { + const char* name; + void (*func)(void); +}; + +struct acutest_test_data_ { + unsigned char flags; + double duration; +}; + +enum { + ACUTEST_FLAG_RUN_ = 1 << 0, + ACUTEST_FLAG_SUCCESS_ = 1 << 1, + ACUTEST_FLAG_FAILURE_ = 1 << 2, +}; + +extern const struct acutest_test_ acutest_list_[]; + +int acutest_check_(int cond, const char* file, int line, const char* fmt, ...); +void acutest_case_(const char* fmt, ...); +void acutest_message_(const char* fmt, ...); +void acutest_dump_(const char* title, const void* addr, size_t size); +void acutest_abort_(void) ACUTEST_ATTRIBUTE_(noreturn); + + +#ifndef TEST_NO_MAIN + +static char* acutest_argv0_ = NULL; +static size_t acutest_list_size_ = 0; +static struct acutest_test_data_* acutest_test_data_ = NULL; +static size_t acutest_count_ = 0; +static int acutest_no_exec_ = -1; +static int acutest_no_summary_ = 0; +static int acutest_tap_ = 0; +static int acutest_skip_mode_ = 0; +static int acutest_worker_ = 0; +static int acutest_worker_index_ = 0; +static int acutest_cond_failed_ = 0; +static int acutest_was_aborted_ = 0; +static FILE *acutest_xml_output_ = NULL; + +static int acutest_stat_failed_units_ = 0; +static int acutest_stat_run_units_ = 0; + +static const struct acutest_test_* acutest_current_test_ = NULL; +static int acutest_current_index_ = 0; +static char acutest_case_name_[TEST_CASE_MAXSIZE] = ""; +static int acutest_test_already_logged_ = 0; +static int acutest_case_already_logged_ = 0; +static int acutest_verbose_level_ = 2; +static int acutest_test_failures_ = 0; +static int acutest_colorize_ = 0; +static int acutest_timer_ = 0; + +static int acutest_abort_has_jmp_buf_ = 0; +static jmp_buf acutest_abort_jmp_buf_; + + +static void +acutest_cleanup_(void) +{ + free((void*) acutest_test_data_); +} + +static void ACUTEST_ATTRIBUTE_(noreturn) +acutest_exit_(int exit_code) +{ + acutest_cleanup_(); + exit(exit_code); +} + +#if defined ACUTEST_WIN_ + typedef LARGE_INTEGER acutest_timer_type_; + static LARGE_INTEGER acutest_timer_freq_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + static void + acutest_timer_init_(void) + { + QueryPerformanceFrequency(´st_timer_freq_); + } + + static void + acutest_timer_get_time_(LARGE_INTEGER* ts) + { + QueryPerformanceCounter(ts); + } + + static double + acutest_timer_diff_(LARGE_INTEGER start, LARGE_INTEGER end) + { + double duration = (double)(end.QuadPart - start.QuadPart); + duration /= (double)acutest_timer_freq_.QuadPart; + return duration; + } + + static void + acutest_timer_print_diff_(void) + { + printf("%.6lf secs", acutest_timer_diff_(acutest_timer_start_, acutest_timer_end_)); + } +#elif defined ACUTEST_HAS_POSIX_TIMER_ + static clockid_t acutest_timer_id_; + typedef struct timespec acutest_timer_type_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + static void + acutest_timer_init_(void) + { + if(acutest_timer_ == 1) + acutest_timer_id_ = CLOCK_MONOTONIC; + else if(acutest_timer_ == 2) + acutest_timer_id_ = CLOCK_PROCESS_CPUTIME_ID; + } + + static void + acutest_timer_get_time_(struct timespec* ts) + { + clock_gettime(acutest_timer_id_, ts); + } + + static double + acutest_timer_diff_(struct timespec start, struct timespec end) + { + double endns; + double startns; + + endns = end.tv_sec; + endns *= 1e9; + endns += end.tv_nsec; + + startns = start.tv_sec; + startns *= 1e9; + startns += start.tv_nsec; + + return ((endns - startns)/ 1e9); + } + + static void + acutest_timer_print_diff_(void) + { + printf("%.6lf secs", + acutest_timer_diff_(acutest_timer_start_, acutest_timer_end_)); + } +#else + typedef int acutest_timer_type_; + static acutest_timer_type_ acutest_timer_start_; + static acutest_timer_type_ acutest_timer_end_; + + void + acutest_timer_init_(void) + {} + + static void + acutest_timer_get_time_(int* ts) + { + (void) ts; + } + + static double + acutest_timer_diff_(int start, int end) + { + (void) start; + (void) end; + return 0.0; + } + + static void + acutest_timer_print_diff_(void) + {} +#endif + +#define ACUTEST_COLOR_DEFAULT_ 0 +#define ACUTEST_COLOR_GREEN_ 1 +#define ACUTEST_COLOR_RED_ 2 +#define ACUTEST_COLOR_DEFAULT_INTENSIVE_ 3 +#define ACUTEST_COLOR_GREEN_INTENSIVE_ 4 +#define ACUTEST_COLOR_RED_INTENSIVE_ 5 + +static int ACUTEST_ATTRIBUTE_(format (printf, 2, 3)) +acutest_colored_printf_(int color, const char* fmt, ...) +{ + va_list args; + char buffer[256]; + int n; + + va_start(args, fmt); + vsnprintf(buffer, sizeof(buffer), fmt, args); + va_end(args); + buffer[sizeof(buffer)-1] = '\0'; + + if(!acutest_colorize_) { + return printf("%s", buffer); + } + +#if defined ACUTEST_UNIX_ + { + const char* col_str; + switch(color) { + case ACUTEST_COLOR_GREEN_: col_str = "\033[0;32m"; break; + case ACUTEST_COLOR_RED_: col_str = "\033[0;31m"; break; + case ACUTEST_COLOR_GREEN_INTENSIVE_: col_str = "\033[1;32m"; break; + case ACUTEST_COLOR_RED_INTENSIVE_: col_str = "\033[1;31m"; break; + case ACUTEST_COLOR_DEFAULT_INTENSIVE_: col_str = "\033[1m"; break; + default: col_str = "\033[0m"; break; + } + printf("%s", col_str); + n = printf("%s", buffer); + printf("\033[0m"); + return n; + } +#elif defined ACUTEST_WIN_ + { + HANDLE h; + CONSOLE_SCREEN_BUFFER_INFO info; + WORD attr; + + h = GetStdHandle(STD_OUTPUT_HANDLE); + GetConsoleScreenBufferInfo(h, &info); + + switch(color) { + case ACUTEST_COLOR_GREEN_: attr = FOREGROUND_GREEN; break; + case ACUTEST_COLOR_RED_: attr = FOREGROUND_RED; break; + case ACUTEST_COLOR_GREEN_INTENSIVE_: attr = FOREGROUND_GREEN | FOREGROUND_INTENSITY; break; + case ACUTEST_COLOR_RED_INTENSIVE_: attr = FOREGROUND_RED | FOREGROUND_INTENSITY; break; + case ACUTEST_COLOR_DEFAULT_INTENSIVE_: attr = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_INTENSITY; break; + default: attr = 0; break; + } + if(attr != 0) + SetConsoleTextAttribute(h, attr); + n = printf("%s", buffer); + SetConsoleTextAttribute(h, info.wAttributes); + return n; + } +#else + n = printf("%s", buffer); + return n; +#endif +} + +static void +acutest_begin_test_line_(const struct acutest_test_* test) +{ + if(!acutest_tap_) { + if(acutest_verbose_level_ >= 3) { + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Test %s:\n", test->name); + acutest_test_already_logged_++; + } else if(acutest_verbose_level_ >= 1) { + int n; + char spaces[48]; + + n = acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Test %s... ", test->name); + memset(spaces, ' ', sizeof(spaces)); + if(n < (int) sizeof(spaces)) + printf("%.*s", (int) sizeof(spaces) - n, spaces); + } else { + acutest_test_already_logged_ = 1; + } + } +} + +static void +acutest_finish_test_line_(int result) +{ + if(acutest_tap_) { + const char* str = (result == 0) ? "ok" : "not ok"; + + printf("%s %d - %s\n", str, acutest_current_index_ + 1, acutest_current_test_->name); + + if(result == 0 && acutest_timer_) { + printf("# Duration: "); + acutest_timer_print_diff_(); + printf("\n"); + } + } else { + int color = (result == 0) ? ACUTEST_COLOR_GREEN_INTENSIVE_ : ACUTEST_COLOR_RED_INTENSIVE_; + const char* str = (result == 0) ? "OK" : "FAILED"; + printf("[ "); + acutest_colored_printf_(color, "%s", str); + printf(" ]"); + + if(result == 0 && acutest_timer_) { + printf(" "); + acutest_timer_print_diff_(); + } + + printf("\n"); + } +} + +static void +acutest_line_indent_(int level) +{ + static const char spaces[] = " "; + int n = level * 2; + + if(acutest_tap_ && n > 0) { + n--; + printf("#"); + } + + while(n > 16) { + printf("%s", spaces); + n -= 16; + } + printf("%.*s", n, spaces); +} + +int ACUTEST_ATTRIBUTE_(format (printf, 4, 5)) +acutest_check_(int cond, const char* file, int line, const char* fmt, ...) +{ + const char *result_str; + int result_color; + int verbose_level; + + if(cond) { + result_str = "ok"; + result_color = ACUTEST_COLOR_GREEN_; + verbose_level = 3; + } else { + if(!acutest_test_already_logged_ && acutest_current_test_ != NULL) + acutest_finish_test_line_(-1); + + result_str = "failed"; + result_color = ACUTEST_COLOR_RED_; + verbose_level = 2; + acutest_test_failures_++; + acutest_test_already_logged_++; + } + + if(acutest_verbose_level_ >= verbose_level) { + va_list args; + + if(!acutest_case_already_logged_ && acutest_case_name_[0]) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Case %s:\n", acutest_case_name_); + acutest_test_already_logged_++; + acutest_case_already_logged_++; + } + + acutest_line_indent_(acutest_case_name_[0] ? 2 : 1); + if(file != NULL) { +#ifdef ACUTEST_WIN_ + const char* lastsep1 = strrchr(file, '\\'); + const char* lastsep2 = strrchr(file, '/'); + if(lastsep1 == NULL) + lastsep1 = file-1; + if(lastsep2 == NULL) + lastsep2 = file-1; + file = (lastsep1 > lastsep2 ? lastsep1 : lastsep2) + 1; +#else + const char* lastsep = strrchr(file, '/'); + if(lastsep != NULL) + file = lastsep+1; +#endif + printf("%s:%d: Check ", file, line); + } + + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + + printf("... "); + acutest_colored_printf_(result_color, "%s", result_str); + printf("\n"); + acutest_test_already_logged_++; + } + + acutest_cond_failed_ = (cond == 0); + return !acutest_cond_failed_; +} + +void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_case_(const char* fmt, ...) +{ + va_list args; + + if(acutest_verbose_level_ < 2) + return; + + if(acutest_case_name_[0]) { + acutest_case_already_logged_ = 0; + acutest_case_name_[0] = '\0'; + } + + if(fmt == NULL) + return; + + va_start(args, fmt); + vsnprintf(acutest_case_name_, sizeof(acutest_case_name_) - 1, fmt, args); + va_end(args); + acutest_case_name_[sizeof(acutest_case_name_) - 1] = '\0'; + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Case %s:\n", acutest_case_name_); + acutest_test_already_logged_++; + acutest_case_already_logged_++; + } +} + +void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_message_(const char* fmt, ...) +{ + char buffer[TEST_MSG_MAXSIZE]; + char* line_beg; + char* line_end; + va_list args; + + if(acutest_verbose_level_ < 2) + return; + + /* We allow extra message only when something is already wrong in the + * current test. */ + if(acutest_current_test_ == NULL || !acutest_cond_failed_) + return; + + va_start(args, fmt); + vsnprintf(buffer, TEST_MSG_MAXSIZE, fmt, args); + va_end(args); + buffer[TEST_MSG_MAXSIZE-1] = '\0'; + + line_beg = buffer; + while(1) { + line_end = strchr(line_beg, '\n'); + if(line_end == NULL) + break; + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf("%.*s\n", (int)(line_end - line_beg), line_beg); + line_beg = line_end + 1; + } + if(line_beg[0] != '\0') { + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf("%s\n", line_beg); + } +} + +void +acutest_dump_(const char* title, const void* addr, size_t size) +{ + static const size_t BYTES_PER_LINE = 16; + size_t line_beg; + size_t truncate = 0; + + if(acutest_verbose_level_ < 2) + return; + + /* We allow extra message only when something is already wrong in the + * current test. */ + if(acutest_current_test_ == NULL || !acutest_cond_failed_) + return; + + if(size > TEST_DUMP_MAXSIZE) { + truncate = size - TEST_DUMP_MAXSIZE; + size = TEST_DUMP_MAXSIZE; + } + + acutest_line_indent_(acutest_case_name_[0] ? 3 : 2); + printf((title[strlen(title)-1] == ':') ? "%s\n" : "%s:\n", title); + + for(line_beg = 0; line_beg < size; line_beg += BYTES_PER_LINE) { + size_t line_end = line_beg + BYTES_PER_LINE; + size_t off; + + acutest_line_indent_(acutest_case_name_[0] ? 4 : 3); + printf("%08lx: ", (unsigned long)line_beg); + for(off = line_beg; off < line_end; off++) { + if(off < size) + printf(" %02x", ((const unsigned char*)addr)[off]); + else + printf(" "); + } + + printf(" "); + for(off = line_beg; off < line_end; off++) { + unsigned char byte = ((const unsigned char*)addr)[off]; + if(off < size) + printf("%c", (iscntrl(byte) ? '.' : byte)); + else + break; + } + + printf("\n"); + } + + if(truncate > 0) { + acutest_line_indent_(acutest_case_name_[0] ? 4 : 3); + printf(" ... (and more %u bytes)\n", (unsigned) truncate); + } +} + +/* This is called just before each test */ +static void +acutest_init_(const char *test_name) +{ +#ifdef TEST_INIT + TEST_INIT + ; /* Allow for a single unterminated function call */ +#endif + + /* Suppress any warnings about unused variable. */ + (void) test_name; +} + +/* This is called after each test */ +static void +acutest_fini_(const char *test_name) +{ +#ifdef TEST_FINI + TEST_FINI + ; /* Allow for a single unterminated function call */ +#endif + + /* Suppress any warnings about unused variable. */ + (void) test_name; +} + +void +acutest_abort_(void) +{ + if(acutest_abort_has_jmp_buf_) { + longjmp(acutest_abort_jmp_buf_, 1); + } else { + if(acutest_current_test_ != NULL) + acutest_fini_(acutest_current_test_->name); + abort(); + } +} + +static void +acutest_list_names_(void) +{ + const struct acutest_test_* test; + + printf("Unit tests:\n"); + for(test = ´st_list_[0]; test->func != NULL; test++) + printf(" %s\n", test->name); +} + +static void +acutest_remember_(int i) +{ + if(acutest_test_data_[i].flags & ACUTEST_FLAG_RUN_) + return; + + acutest_test_data_[i].flags |= ACUTEST_FLAG_RUN_; + acutest_count_++; +} + +static void +acutest_set_success_(int i, int success) +{ + acutest_test_data_[i].flags |= success ? ACUTEST_FLAG_SUCCESS_ : ACUTEST_FLAG_FAILURE_; +} + +static void +acutest_set_duration_(int i, double duration) +{ + acutest_test_data_[i].duration = duration; +} + +static int +acutest_name_contains_word_(const char* name, const char* pattern) +{ + static const char word_delim[] = " \t-_/.,:;"; + const char* substr; + size_t pattern_len; + + pattern_len = strlen(pattern); + + substr = strstr(name, pattern); + while(substr != NULL) { + int starts_on_word_boundary = (substr == name || strchr(word_delim, substr[-1]) != NULL); + int ends_on_word_boundary = (substr[pattern_len] == '\0' || strchr(word_delim, substr[pattern_len]) != NULL); + + if(starts_on_word_boundary && ends_on_word_boundary) + return 1; + + substr = strstr(substr+1, pattern); + } + + return 0; +} + +static int +acutest_lookup_(const char* pattern) +{ + int i; + int n = 0; + + /* Try exact match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(strcmp(acutest_list_[i].name, pattern) == 0) { + acutest_remember_(i); + n++; + break; + } + } + if(n > 0) + return n; + + /* Try word match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(acutest_name_contains_word_(acutest_list_[i].name, pattern)) { + acutest_remember_(i); + n++; + } + } + if(n > 0) + return n; + + /* Try relaxed match. */ + for(i = 0; i < (int) acutest_list_size_; i++) { + if(strstr(acutest_list_[i].name, pattern) != NULL) { + acutest_remember_(i); + n++; + } + } + + return n; +} + + +/* Called if anything goes bad in Acutest, or if the unit test ends in other + * way then by normal returning from its function (e.g. exception or some + * abnormal child process termination). */ +static void ACUTEST_ATTRIBUTE_(format (printf, 1, 2)) +acutest_error_(const char* fmt, ...) +{ + if(acutest_verbose_level_ == 0) + return; + + if(acutest_verbose_level_ >= 2) { + va_list args; + + acutest_line_indent_(1); + if(acutest_verbose_level_ >= 3) + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "ERROR: "); + va_start(args, fmt); + vprintf(fmt, args); + va_end(args); + printf("\n"); + } + + if(acutest_verbose_level_ >= 3) { + printf("\n"); + } +} + +/* Call directly the given test unit function. */ +static int +acutest_do_run_(const struct acutest_test_* test, int index) +{ + int status = -1; + + acutest_was_aborted_ = 0; + acutest_current_test_ = test; + acutest_current_index_ = index; + acutest_test_failures_ = 0; + acutest_test_already_logged_ = 0; + acutest_cond_failed_ = 0; + +#ifdef __cplusplus + try { +#endif + acutest_init_(test->name); + acutest_begin_test_line_(test); + + /* This is good to do in case the test unit crashes. */ + fflush(stdout); + fflush(stderr); + + if(!acutest_worker_) { + acutest_abort_has_jmp_buf_ = 1; + if(setjmp(acutest_abort_jmp_buf_) != 0) { + acutest_was_aborted_ = 1; + goto aborted; + } + } + + acutest_timer_get_time_(´st_timer_start_); + test->func(); +aborted: + acutest_abort_has_jmp_buf_ = 0; + acutest_timer_get_time_(´st_timer_end_); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + if(acutest_test_failures_ == 0) { + acutest_colored_printf_(ACUTEST_COLOR_GREEN_INTENSIVE_, "SUCCESS: "); + printf("All conditions have passed.\n"); + + if(acutest_timer_) { + acutest_line_indent_(1); + printf("Duration: "); + acutest_timer_print_diff_(); + printf("\n"); + } + } else { + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + if(!acutest_was_aborted_) { + printf("%d condition%s %s failed.\n", + acutest_test_failures_, + (acutest_test_failures_ == 1) ? "" : "s", + (acutest_test_failures_ == 1) ? "has" : "have"); + } else { + printf("Aborted.\n"); + } + } + printf("\n"); + } else if(acutest_verbose_level_ >= 1 && acutest_test_failures_ == 0) { + acutest_finish_test_line_(0); + } + + status = (acutest_test_failures_ == 0) ? 0 : -1; + +#ifdef __cplusplus + } catch(std::exception& e) { + const char* what = e.what(); + acutest_check_(0, NULL, 0, "Threw std::exception"); + if(what != NULL) + acutest_message_("std::exception::what(): %s", what); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + printf("C++ exception.\n\n"); + } + } catch(...) { + acutest_check_(0, NULL, 0, "Threw an exception"); + + if(acutest_verbose_level_ >= 3) { + acutest_line_indent_(1); + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED: "); + printf("C++ exception.\n\n"); + } + } +#endif + + acutest_fini_(test->name); + acutest_case_(NULL); + acutest_current_test_ = NULL; + + return status; +} + +/* Trigger the unit test. If possible (and not suppressed) it starts a child + * process who calls acutest_do_run_(), otherwise it calls acutest_do_run_() + * directly. */ +static void +acutest_run_(const struct acutest_test_* test, int index, int master_index) +{ + int failed = 1; + acutest_timer_type_ start, end; + + acutest_current_test_ = test; + acutest_test_already_logged_ = 0; + acutest_timer_get_time_(&start); + + if(!acutest_no_exec_) { + +#if defined(ACUTEST_UNIX_) + + pid_t pid; + int exit_code; + + /* Make sure the child starts with empty I/O buffers. */ + fflush(stdout); + fflush(stderr); + + pid = fork(); + if(pid == (pid_t)-1) { + acutest_error_("Cannot fork. %s [%d]", strerror(errno), errno); + failed = 1; + } else if(pid == 0) { + /* Child: Do the test. */ + acutest_worker_ = 1; + failed = (acutest_do_run_(test, index) != 0); + acutest_exit_(failed ? 1 : 0); + } else { + /* Parent: Wait until child terminates and analyze its exit code. */ + waitpid(pid, &exit_code, 0); + if(WIFEXITED(exit_code)) { + switch(WEXITSTATUS(exit_code)) { + case 0: failed = 0; break; /* test has passed. */ + case 1: /* noop */ break; /* "normal" failure. */ + default: acutest_error_("Unexpected exit code [%d]", WEXITSTATUS(exit_code)); + } + } else if(WIFSIGNALED(exit_code)) { + char tmp[32]; + const char* signame; + switch(WTERMSIG(exit_code)) { + case SIGINT: signame = "SIGINT"; break; + case SIGHUP: signame = "SIGHUP"; break; + case SIGQUIT: signame = "SIGQUIT"; break; + case SIGABRT: signame = "SIGABRT"; break; + case SIGKILL: signame = "SIGKILL"; break; + case SIGSEGV: signame = "SIGSEGV"; break; + case SIGILL: signame = "SIGILL"; break; + case SIGTERM: signame = "SIGTERM"; break; + default: sprintf(tmp, "signal %d", WTERMSIG(exit_code)); signame = tmp; break; + } + acutest_error_("Test interrupted by %s.", signame); + } else { + acutest_error_("Test ended in an unexpected way [%d].", exit_code); + } + } + +#elif defined(ACUTEST_WIN_) + + char buffer[512] = {0}; + STARTUPINFOA startupInfo; + PROCESS_INFORMATION processInfo; + DWORD exitCode; + + /* Windows has no fork(). So we propagate all info into the child + * through a command line arguments. */ + _snprintf(buffer, sizeof(buffer)-1, + "%s --worker=%d %s --no-exec --no-summary %s --verbose=%d --color=%s -- \"%s\"", + acutest_argv0_, index, acutest_timer_ ? "--time" : "", + acutest_tap_ ? "--tap" : "", acutest_verbose_level_, + acutest_colorize_ ? "always" : "never", + test->name); + memset(&startupInfo, 0, sizeof(startupInfo)); + startupInfo.cb = sizeof(STARTUPINFO); + if(CreateProcessA(NULL, buffer, NULL, NULL, FALSE, 0, NULL, NULL, &startupInfo, &processInfo)) { + WaitForSingleObject(processInfo.hProcess, INFINITE); + GetExitCodeProcess(processInfo.hProcess, &exitCode); + CloseHandle(processInfo.hThread); + CloseHandle(processInfo.hProcess); + failed = (exitCode != 0); + if(exitCode > 1) { + switch(exitCode) { + case 3: acutest_error_("Aborted."); break; + case 0xC0000005: acutest_error_("Access violation."); break; + default: acutest_error_("Test ended in an unexpected way [%lu].", exitCode); break; + } + } + } else { + acutest_error_("Cannot create unit test subprocess [%ld].", GetLastError()); + failed = 1; + } + +#else + + /* A platform where we don't know how to run child process. */ + failed = (acutest_do_run_(test, index) != 0); + +#endif + + } else { + /* Child processes suppressed through --no-exec. */ + failed = (acutest_do_run_(test, index) != 0); + } + acutest_timer_get_time_(&end); + + acutest_current_test_ = NULL; + + acutest_stat_run_units_++; + if(failed) + acutest_stat_failed_units_++; + + acutest_set_success_(master_index, !failed); + acutest_set_duration_(master_index, acutest_timer_diff_(start, end)); +} + +#if defined(ACUTEST_WIN_) +/* Callback for SEH events. */ +static LONG CALLBACK +acutest_seh_exception_filter_(EXCEPTION_POINTERS *ptrs) +{ + acutest_check_(0, NULL, 0, "Unhandled SEH exception"); + acutest_message_("Exception code: 0x%08lx", ptrs->ExceptionRecord->ExceptionCode); + acutest_message_("Exception address: 0x%p", ptrs->ExceptionRecord->ExceptionAddress); + + fflush(stdout); + fflush(stderr); + + return EXCEPTION_EXECUTE_HANDLER; +} +#endif + + +#define ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ 0x0001 +#define ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ 0x0002 + +#define ACUTEST_CMDLINE_OPTID_NONE_ 0 +#define ACUTEST_CMDLINE_OPTID_UNKNOWN_ (-0x7fffffff + 0) +#define ACUTEST_CMDLINE_OPTID_MISSINGARG_ (-0x7fffffff + 1) +#define ACUTEST_CMDLINE_OPTID_BOGUSARG_ (-0x7fffffff + 2) + +typedef struct acutest_test_CMDLINE_OPTION_ { + char shortname; + const char* longname; + int id; + unsigned flags; +} ACUTEST_CMDLINE_OPTION_; + +static int +acutest_cmdline_handle_short_opt_group_(const ACUTEST_CMDLINE_OPTION_* options, + const char* arggroup, + int (*callback)(int /*optval*/, const char* /*arg*/)) +{ + const ACUTEST_CMDLINE_OPTION_* opt; + int i; + int ret = 0; + + for(i = 0; arggroup[i] != '\0'; i++) { + for(opt = options; opt->id != 0; opt++) { + if(arggroup[i] == opt->shortname) + break; + } + + if(opt->id != 0 && !(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) { + ret = callback(opt->id, NULL); + } else { + /* Unknown option. */ + char badoptname[3]; + badoptname[0] = '-'; + badoptname[1] = arggroup[i]; + badoptname[2] = '\0'; + ret = callback((opt->id != 0 ? ACUTEST_CMDLINE_OPTID_MISSINGARG_ : ACUTEST_CMDLINE_OPTID_UNKNOWN_), + badoptname); + } + + if(ret != 0) + break; + } + + return ret; +} + +#define ACUTEST_CMDLINE_AUXBUF_SIZE_ 32 + +static int +acutest_cmdline_read_(const ACUTEST_CMDLINE_OPTION_* options, int argc, char** argv, + int (*callback)(int /*optval*/, const char* /*arg*/)) +{ + + const ACUTEST_CMDLINE_OPTION_* opt; + char auxbuf[ACUTEST_CMDLINE_AUXBUF_SIZE_+1]; + int after_doubledash = 0; + int i = 1; + int ret = 0; + + auxbuf[ACUTEST_CMDLINE_AUXBUF_SIZE_] = '\0'; + + while(i < argc) { + if(after_doubledash || strcmp(argv[i], "-") == 0) { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else if(strcmp(argv[i], "--") == 0) { + /* End of options. All the remaining members are non-option arguments. */ + after_doubledash = 1; + } else if(argv[i][0] != '-') { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else { + for(opt = options; opt->id != 0; opt++) { + if(opt->longname != NULL && strncmp(argv[i], "--", 2) == 0) { + size_t len = strlen(opt->longname); + if(strncmp(argv[i]+2, opt->longname, len) == 0) { + /* Regular long option. */ + if(argv[i][2+len] == '\0') { + /* with no argument provided. */ + if(!(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) + ret = callback(opt->id, NULL); + else + ret = callback(ACUTEST_CMDLINE_OPTID_MISSINGARG_, argv[i]); + break; + } else if(argv[i][2+len] == '=') { + /* with an argument provided. */ + if(opt->flags & (ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ | ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_)) { + ret = callback(opt->id, argv[i]+2+len+1); + } else { + sprintf(auxbuf, "--%s", opt->longname); + ret = callback(ACUTEST_CMDLINE_OPTID_BOGUSARG_, auxbuf); + } + break; + } else { + continue; + } + } + } else if(opt->shortname != '\0' && argv[i][0] == '-') { + if(argv[i][1] == opt->shortname) { + /* Regular short option. */ + if(opt->flags & ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_) { + if(argv[i][2] != '\0') + ret = callback(opt->id, argv[i]+2); + else if(i+1 < argc) + ret = callback(opt->id, argv[++i]); + else + ret = callback(ACUTEST_CMDLINE_OPTID_MISSINGARG_, argv[i]); + break; + } else { + ret = callback(opt->id, NULL); + + /* There might be more (argument-less) short options + * grouped together. */ + if(ret == 0 && argv[i][2] != '\0') + ret = acutest_cmdline_handle_short_opt_group_(options, argv[i]+2, callback); + break; + } + } + } + } + + if(opt->id == 0) { /* still not handled? */ + if(argv[i][0] != '-') { + /* Non-option argument. */ + ret = callback(ACUTEST_CMDLINE_OPTID_NONE_, argv[i]); + } else { + /* Unknown option. */ + char* badoptname = argv[i]; + + if(strncmp(badoptname, "--", 2) == 0) { + /* Strip any argument from the long option. */ + char* assignment = strchr(badoptname, '='); + if(assignment != NULL) { + size_t len = assignment - badoptname; + if(len > ACUTEST_CMDLINE_AUXBUF_SIZE_) + len = ACUTEST_CMDLINE_AUXBUF_SIZE_; + strncpy(auxbuf, badoptname, len); + auxbuf[len] = '\0'; + badoptname = auxbuf; + } + } + + ret = callback(ACUTEST_CMDLINE_OPTID_UNKNOWN_, badoptname); + } + } + } + + if(ret != 0) + return ret; + i++; + } + + return ret; +} + +static void +acutest_help_(void) +{ + printf("Usage: %s [options] [test...]\n", acutest_argv0_); + printf("\n"); + printf("Run the specified unit tests; or if the option '--skip' is used, run all\n"); + printf("tests in the suite but those listed. By default, if no tests are specified\n"); + printf("on the command line, all unit tests in the suite are run.\n"); + printf("\n"); + printf("Options:\n"); + printf(" -s, --skip Execute all unit tests but the listed ones\n"); + printf(" --exec[=WHEN] If supported, execute unit tests as child processes\n"); + printf(" (WHEN is one of 'auto', 'always', 'never')\n"); + printf(" -E, --no-exec Same as --exec=never\n"); +#if defined ACUTEST_WIN_ + printf(" -t, --time Measure test duration\n"); +#elif defined ACUTEST_HAS_POSIX_TIMER_ + printf(" -t, --time Measure test duration (real time)\n"); + printf(" --time=TIMER Measure test duration, using given timer\n"); + printf(" (TIMER is one of 'real', 'cpu')\n"); +#endif + printf(" --no-summary Suppress printing of test results summary\n"); + printf(" --tap Produce TAP-compliant output\n"); + printf(" (See https://testanything.org/)\n"); + printf(" -x, --xml-output=FILE Enable XUnit output to the given file\n"); + printf(" -l, --list List unit tests in the suite and exit\n"); + printf(" -v, --verbose Make output more verbose\n"); + printf(" --verbose=LEVEL Set verbose level to LEVEL:\n"); + printf(" 0 ... Be silent\n"); + printf(" 1 ... Output one line per test (and summary)\n"); + printf(" 2 ... As 1 and failed conditions (this is default)\n"); + printf(" 3 ... As 1 and all conditions (and extended summary)\n"); + printf(" -q, --quiet Same as --verbose=0\n"); + printf(" --color[=WHEN] Enable colorized output\n"); + printf(" (WHEN is one of 'auto', 'always', 'never')\n"); + printf(" --no-color Same as --color=never\n"); + printf(" -h, --help Display this help and exit\n"); + + if(acutest_list_size_ < 16) { + printf("\n"); + acutest_list_names_(); + } +} + +static const ACUTEST_CMDLINE_OPTION_ acutest_cmdline_options_[] = { + { 's', "skip", 's', 0 }, + { 0, "exec", 'e', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 'E', "no-exec", 'E', 0 }, +#if defined ACUTEST_WIN_ + { 't', "time", 't', 0 }, + { 0, "timer", 't', 0 }, /* kept for compatibility */ +#elif defined ACUTEST_HAS_POSIX_TIMER_ + { 't', "time", 't', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 0, "timer", 't', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, /* kept for compatibility */ +#endif + { 0, "no-summary", 'S', 0 }, + { 0, "tap", 'T', 0 }, + { 'l', "list", 'l', 0 }, + { 'v', "verbose", 'v', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 'q', "quiet", 'q', 0 }, + { 0, "color", 'c', ACUTEST_CMDLINE_OPTFLAG_OPTIONALARG_ }, + { 0, "no-color", 'C', 0 }, + { 'h', "help", 'h', 0 }, + { 0, "worker", 'w', ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ }, /* internal */ + { 'x', "xml-output", 'x', ACUTEST_CMDLINE_OPTFLAG_REQUIREDARG_ }, + { 0, NULL, 0, 0 } +}; + +static int +acutest_cmdline_callback_(int id, const char* arg) +{ + switch(id) { + case 's': + acutest_skip_mode_ = 1; + break; + + case 'e': + if(arg == NULL || strcmp(arg, "always") == 0) { + acutest_no_exec_ = 0; + } else if(strcmp(arg, "never") == 0) { + acutest_no_exec_ = 1; + } else if(strcmp(arg, "auto") == 0) { + /*noop*/ + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --exec.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case 'E': + acutest_no_exec_ = 1; + break; + + case 't': +#if defined ACUTEST_WIN_ || defined ACUTEST_HAS_POSIX_TIMER_ + if(arg == NULL || strcmp(arg, "real") == 0) { + acutest_timer_ = 1; + #ifndef ACUTEST_WIN_ + } else if(strcmp(arg, "cpu") == 0) { + acutest_timer_ = 2; + #endif + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --time.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } +#endif + break; + + case 'S': + acutest_no_summary_ = 1; + break; + + case 'T': + acutest_tap_ = 1; + break; + + case 'l': + acutest_list_names_(); + acutest_exit_(0); + break; + + case 'v': + acutest_verbose_level_ = (arg != NULL ? atoi(arg) : acutest_verbose_level_+1); + break; + + case 'q': + acutest_verbose_level_ = 0; + break; + + case 'c': + if(arg == NULL || strcmp(arg, "always") == 0) { + acutest_colorize_ = 1; + } else if(strcmp(arg, "never") == 0) { + acutest_colorize_ = 0; + } else if(strcmp(arg, "auto") == 0) { + /*noop*/ + } else { + fprintf(stderr, "%s: Unrecognized argument '%s' for option --color.\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case 'C': + acutest_colorize_ = 0; + break; + + case 'h': + acutest_help_(); + acutest_exit_(0); + break; + + case 'w': + acutest_worker_ = 1; + acutest_worker_index_ = atoi(arg); + break; + case 'x': + acutest_xml_output_ = fopen(arg, "w"); + if (!acutest_xml_output_) { + fprintf(stderr, "Unable to open '%s': %s\n", arg, strerror(errno)); + acutest_exit_(2); + } + break; + + case 0: + if(acutest_lookup_(arg) == 0) { + fprintf(stderr, "%s: Unrecognized unit test '%s'\n", acutest_argv0_, arg); + fprintf(stderr, "Try '%s --list' for list of unit tests.\n", acutest_argv0_); + acutest_exit_(2); + } + break; + + case ACUTEST_CMDLINE_OPTID_UNKNOWN_: + fprintf(stderr, "Unrecognized command line option '%s'.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + + case ACUTEST_CMDLINE_OPTID_MISSINGARG_: + fprintf(stderr, "The command line option '%s' requires an argument.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + + case ACUTEST_CMDLINE_OPTID_BOGUSARG_: + fprintf(stderr, "The command line option '%s' does not expect an argument.\n", arg); + fprintf(stderr, "Try '%s --help' for more information.\n", acutest_argv0_); + acutest_exit_(2); + break; + } + + return 0; +} + + +#ifdef ACUTEST_LINUX_ +static int +acutest_is_tracer_present_(void) +{ + /* Must be large enough so the line 'TracerPid: ${PID}' can fit in. */ + static const int OVERLAP = 32; + + char buf[256+OVERLAP+1]; + int tracer_present = 0; + int fd; + size_t n_read = 0; + + fd = open("/proc/self/status", O_RDONLY); + if(fd == -1) + return 0; + + while(1) { + static const char pattern[] = "TracerPid:"; + const char* field; + + while(n_read < sizeof(buf) - 1) { + ssize_t n; + + n = read(fd, buf + n_read, sizeof(buf) - 1 - n_read); + if(n <= 0) + break; + n_read += n; + } + buf[n_read] = '\0'; + + field = strstr(buf, pattern); + if(field != NULL && field < buf + sizeof(buf) - OVERLAP) { + pid_t tracer_pid = (pid_t) atoi(field + sizeof(pattern) - 1); + tracer_present = (tracer_pid != 0); + break; + } + + if(n_read == sizeof(buf)-1) { + memmove(buf, buf + sizeof(buf)-1 - OVERLAP, OVERLAP); + n_read = OVERLAP; + } else { + break; + } + } + + close(fd); + return tracer_present; +} +#endif + +int +main(int argc, char** argv) +{ + int i; + + acutest_argv0_ = argv[0]; + +#if defined ACUTEST_UNIX_ + acutest_colorize_ = isatty(STDOUT_FILENO); +#elif defined ACUTEST_WIN_ + #if defined _BORLANDC_ + acutest_colorize_ = isatty(_fileno(stdout)); + #else + acutest_colorize_ = _isatty(_fileno(stdout)); + #endif +#else + acutest_colorize_ = 0; +#endif + + /* Count all test units */ + acutest_list_size_ = 0; + for(i = 0; acutest_list_[i].func != NULL; i++) + acutest_list_size_++; + + acutest_test_data_ = (struct acutest_test_data_*)calloc(acutest_list_size_, sizeof(struct acutest_test_data_)); + if(acutest_test_data_ == NULL) { + fprintf(stderr, "Out of memory.\n"); + acutest_exit_(2); + } + + /* Parse options */ + acutest_cmdline_read_(acutest_cmdline_options_, argc, argv, acutest_cmdline_callback_); + + /* Initialize the proper timer. */ + acutest_timer_init_(); + +#if defined(ACUTEST_WIN_) + SetUnhandledExceptionFilter(acutest_seh_exception_filter_); +#ifdef _MSC_VER + _set_abort_behavior(0, _WRITE_ABORT_MSG); +#endif +#endif + + /* By default, we want to run all tests. */ + if(acutest_count_ == 0) { + for(i = 0; acutest_list_[i].func != NULL; i++) + acutest_remember_(i); + } + + /* Guess whether we want to run unit tests as child processes. */ + if(acutest_no_exec_ < 0) { + acutest_no_exec_ = 0; + + if(acutest_count_ <= 1) { + acutest_no_exec_ = 1; + } else { +#ifdef ACUTEST_WIN_ + if(IsDebuggerPresent()) + acutest_no_exec_ = 1; +#endif +#ifdef ACUTEST_LINUX_ + if(acutest_is_tracer_present_()) + acutest_no_exec_ = 1; +#endif +#ifdef RUNNING_ON_VALGRIND + /* RUNNING_ON_VALGRIND is provided by optionally included */ + if(RUNNING_ON_VALGRIND) + acutest_no_exec_ = 1; +#endif + } + } + + if(acutest_tap_) { + /* TAP requires we know test result ("ok", "not ok") before we output + * anything about the test, and this gets problematic for larger verbose + * levels. */ + if(acutest_verbose_level_ > 2) + acutest_verbose_level_ = 2; + + /* TAP harness should provide some summary. */ + acutest_no_summary_ = 1; + + if(!acutest_worker_) + printf("1..%d\n", (int) acutest_count_); + } + + int index = acutest_worker_index_; + for(i = 0; acutest_list_[i].func != NULL; i++) { + int run = (acutest_test_data_[i].flags & ACUTEST_FLAG_RUN_); + if (acutest_skip_mode_) /* Run all tests except those listed. */ + run = !run; + if(run) + acutest_run_(´st_list_[i], index++, i); + } + + /* Write a summary */ + if(!acutest_no_summary_ && acutest_verbose_level_ >= 1) { + if(acutest_verbose_level_ >= 3) { + acutest_colored_printf_(ACUTEST_COLOR_DEFAULT_INTENSIVE_, "Summary:\n"); + + printf(" Count of all unit tests: %4d\n", (int) acutest_list_size_); + printf(" Count of run unit tests: %4d\n", acutest_stat_run_units_); + printf(" Count of failed unit tests: %4d\n", acutest_stat_failed_units_); + printf(" Count of skipped unit tests: %4d\n", (int) acutest_list_size_ - acutest_stat_run_units_); + } + + if(acutest_stat_failed_units_ == 0) { + acutest_colored_printf_(ACUTEST_COLOR_GREEN_INTENSIVE_, "SUCCESS:"); + printf(" All unit tests have passed.\n"); + } else { + acutest_colored_printf_(ACUTEST_COLOR_RED_INTENSIVE_, "FAILED:"); + printf(" %d of %d unit tests %s failed.\n", + acutest_stat_failed_units_, acutest_stat_run_units_, + (acutest_stat_failed_units_ == 1) ? "has" : "have"); + } + + if(acutest_verbose_level_ >= 3) + printf("\n"); + } + + if (acutest_xml_output_) { +#if defined ACUTEST_UNIX_ + char *suite_name = basename(argv[0]); +#elif defined ACUTEST_WIN_ + char suite_name[_MAX_FNAME]; + _splitpath(argv[0], NULL, NULL, suite_name, NULL); +#else + const char *suite_name = argv[0]; +#endif + fprintf(acutest_xml_output_, "\n"); + fprintf(acutest_xml_output_, "\n", + suite_name, (int)acutest_list_size_, acutest_stat_failed_units_, acutest_stat_failed_units_, + (int)acutest_list_size_ - acutest_stat_run_units_); + for(i = 0; acutest_list_[i].func != NULL; i++) { + struct acutest_test_data_ *details = ´st_test_data_[i]; + fprintf(acutest_xml_output_, " \n", acutest_list_[i].name, details->duration); + if (details->flags & ACUTEST_FLAG_FAILURE_) + fprintf(acutest_xml_output_, " \n"); + if (!(details->flags & ACUTEST_FLAG_FAILURE_) && !(details->flags & ACUTEST_FLAG_SUCCESS_)) + fprintf(acutest_xml_output_, " \n"); + fprintf(acutest_xml_output_, " \n"); + } + fprintf(acutest_xml_output_, "\n"); + fclose(acutest_xml_output_); + } + + acutest_cleanup_(); + + return (acutest_stat_failed_units_ == 0) ? 0 : 1; +} + + +#endif /* #ifndef TEST_NO_MAIN */ + +#ifdef _MSC_VER + #pragma warning(pop) +#endif + +#ifdef __cplusplus + } /* extern "C" */ +#endif + +#endif /* #ifndef ACUTEST_H */ diff --git a/graalpython/hpy/c_test/test_debug_handles.c b/graalpython/hpy/c_test/test_debug_handles.c new file mode 100644 index 0000000000..a5584fdb39 --- /dev/null +++ b/graalpython/hpy/c_test/test_debug_handles.c @@ -0,0 +1,99 @@ +#include +#include "acutest.h" // https://github.com/mity/acutest +#include "hpy/debug/src/debug_internal.h" + +static void check_DHQueue(DHQueue *q, HPy_ssize_t size, ...) +{ + va_list argp; + va_start(argp, size); + DHQueue_sanity_check(q); + TEST_CHECK(q->size == size); + DHQueueNode *h = q->head; + while(h != NULL) { + DHQueueNode *expected = va_arg(argp, DHQueueNode*); + TEST_CHECK(h == expected); + h = h->next; + } + va_end(argp); +} + +void test_DHQueue_init(void) +{ + DHQueue q; + DHQueue_init(&q); + TEST_CHECK(q.head == NULL); + TEST_CHECK(q.tail == NULL); + TEST_CHECK(q.size == 0); + DHQueue_sanity_check(&q); +} + +void test_DHQueue_append(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + check_DHQueue(&q, 3, &h1, &h2, &h3); +} + +void test_DHQueue_popfront(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h1); + check_DHQueue(&q, 2, &h2, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h2); + check_DHQueue(&q, 1, &h3); + + TEST_CHECK(DHQueue_popfront(&q) == &h3); + check_DHQueue(&q, 0); +} + + +void test_DHQueue_remove(void) +{ + DHQueue q; + DHQueueNode h1; + DHQueueNode h2; + DHQueueNode h3; + DHQueueNode h4; + DHQueue_init(&q); + DHQueue_append(&q, &h1); + DHQueue_append(&q, &h2); + DHQueue_append(&q, &h3); + DHQueue_append(&q, &h4); + + DHQueue_remove(&q, &h3); // try to remove something in the middle + check_DHQueue(&q, 3, &h1, &h2, &h4); + + DHQueue_remove(&q, &h1); // try to remove the head + check_DHQueue(&q, 2, &h2, &h4); + + DHQueue_remove(&q, &h4); // try to remove the tail + check_DHQueue(&q, 1, &h2); + + DHQueue_remove(&q, &h2); // try to remove the only element + check_DHQueue(&q, 0); +} + +#define MYTEST(X) { #X, X } + +TEST_LIST = { + MYTEST(test_DHQueue_init), + MYTEST(test_DHQueue_append), + MYTEST(test_DHQueue_popfront), + MYTEST(test_DHQueue_remove), + { NULL, NULL } +}; diff --git a/graalpython/hpy/c_test/test_stacktrace.c b/graalpython/hpy/c_test/test_stacktrace.c new file mode 100644 index 0000000000..5d02eced84 --- /dev/null +++ b/graalpython/hpy/c_test/test_stacktrace.c @@ -0,0 +1,22 @@ +// Smoke test for the create_stacktrace function + +#include +#include "acutest.h" // https://github.com/mity/acutest +#include "hpy/debug/src/debug_internal.h" + +void test_create_stacktrace(void) +{ + char *trace; + create_stacktrace(&trace, 16); + if (trace != NULL) { + printf("\n\nTRACE:\n%s\n\n", trace); + free(trace); + } +} + +#define MYTEST(X) { #X, X } + +TEST_LIST = { + MYTEST(test_create_stacktrace), + { NULL, NULL } +}; \ No newline at end of file diff --git a/graalpython/hpy/docs/Makefile b/graalpython/hpy/docs/Makefile new file mode 100644 index 0000000000..7164c8d0e5 --- /dev/null +++ b/graalpython/hpy/docs/Makefile @@ -0,0 +1,28 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +livehtml: + @sphinx-autobuild \ + "$(SOURCEDIR)" \ + -b html \ + --ignore "*~" \ + --ignore ".#*" \ + $(SPHINXOPTS) $(BUILDDIR)/html diff --git a/graalpython/hpy/docs/_static/README.txt b/graalpython/hpy/docs/_static/README.txt new file mode 100644 index 0000000000..4d871f5a5a --- /dev/null +++ b/graalpython/hpy/docs/_static/README.txt @@ -0,0 +1 @@ +Static files for the Sphinx documentation. diff --git a/graalpython/hpy/docs/_templates/README.txt b/graalpython/hpy/docs/_templates/README.txt new file mode 100644 index 0000000000..2dc51448da --- /dev/null +++ b/graalpython/hpy/docs/_templates/README.txt @@ -0,0 +1 @@ +Template files for the Sphinx documentation. diff --git a/graalpython/hpy/docs/api-reference/argument-parsing.rst b/graalpython/hpy/docs/api-reference/argument-parsing.rst new file mode 100644 index 0000000000..ff22665254 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/argument-parsing.rst @@ -0,0 +1,5 @@ +Argument Parsing +================ + +.. autocmodule:: runtime/argparse.c + :members: diff --git a/graalpython/hpy/docs/api-reference/build-value.rst b/graalpython/hpy/docs/api-reference/build-value.rst new file mode 100644 index 0000000000..a8cafb6bfe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/build-value.rst @@ -0,0 +1,5 @@ +Building Complex Python Objects +=============================== + +.. autocmodule:: runtime/buildvalue.c + :members: diff --git a/graalpython/hpy/docs/api-reference/formatting.rst b/graalpython/hpy/docs/api-reference/formatting.rst new file mode 100644 index 0000000000..f4ea6f26f4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/formatting.rst @@ -0,0 +1,5 @@ +String Formatting Helpers +========================= + +.. autocmodule:: runtime/format.c + :no-members: diff --git a/graalpython/hpy/docs/api-reference/function-index.rst b/graalpython/hpy/docs/api-reference/function-index.rst new file mode 100644 index 0000000000..5e2cbcb59b --- /dev/null +++ b/graalpython/hpy/docs/api-reference/function-index.rst @@ -0,0 +1,187 @@ + +.. note: DO NOT EDIT THIS FILE! + This file is automatically generated by hpy.tools.autogen.doc.autogen_function_index + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +HPy Core API Function Index +########################### + +* :c:func:`HPyBool_FromBool` +* :c:func:`HPyBytes_AS_STRING` +* :c:func:`HPyBytes_AsString` +* :c:func:`HPyBytes_Check` +* :c:func:`HPyBytes_FromString` +* :c:func:`HPyBytes_FromStringAndSize` +* :c:func:`HPyBytes_GET_SIZE` +* :c:func:`HPyBytes_Size` +* :c:func:`HPyCallable_Check` +* :c:func:`HPyCapsule_Get` +* :c:func:`HPyCapsule_IsValid` +* :c:func:`HPyCapsule_New` +* :c:func:`HPyCapsule_Set` +* :c:func:`HPyContextVar_Get` +* :c:func:`HPyContextVar_New` +* :c:func:`HPyContextVar_Set` +* :c:func:`HPyDict_Check` +* :c:func:`HPyDict_Copy` +* :c:func:`HPyDict_Keys` +* :c:func:`HPyDict_New` +* :c:func:`HPyErr_Clear` +* :c:func:`HPyErr_ExceptionMatches` +* :c:func:`HPyErr_NewException` +* :c:func:`HPyErr_NewExceptionWithDoc` +* :c:func:`HPyErr_NoMemory` +* :c:func:`HPyErr_Occurred` +* :c:func:`HPyErr_SetFromErrnoWithFilename` +* :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` +* :c:func:`HPyErr_SetObject` +* :c:func:`HPyErr_SetString` +* :c:func:`HPyErr_WarnEx` +* :c:func:`HPyErr_WriteUnraisable` +* :c:func:`HPyField_Load` +* :c:func:`HPyField_Store` +* :c:func:`HPyFloat_AsDouble` +* :c:func:`HPyFloat_FromDouble` +* :c:func:`HPyGlobal_Load` +* :c:func:`HPyGlobal_Store` +* :c:func:`HPyImport_ImportModule` +* :c:func:`HPyIter_Check` +* :c:func:`HPyIter_Next` +* :c:func:`HPyListBuilder_Build` +* :c:func:`HPyListBuilder_Cancel` +* :c:func:`HPyListBuilder_New` +* :c:func:`HPyListBuilder_Set` +* :c:func:`HPyList_Append` +* :c:func:`HPyList_Check` +* :c:func:`HPyList_Insert` +* :c:func:`HPyList_New` +* :c:func:`HPyLong_AsDouble` +* :c:func:`HPyLong_AsInt32_t` +* :c:func:`HPyLong_AsInt64_t` +* :c:func:`HPyLong_AsSize_t` +* :c:func:`HPyLong_AsSsize_t` +* :c:func:`HPyLong_AsUInt32_t` +* :c:func:`HPyLong_AsUInt32_tMask` +* :c:func:`HPyLong_AsUInt64_t` +* :c:func:`HPyLong_AsUInt64_tMask` +* :c:func:`HPyLong_AsVoidPtr` +* :c:func:`HPyLong_FromInt32_t` +* :c:func:`HPyLong_FromInt64_t` +* :c:func:`HPyLong_FromSize_t` +* :c:func:`HPyLong_FromSsize_t` +* :c:func:`HPyLong_FromUInt32_t` +* :c:func:`HPyLong_FromUInt64_t` +* :c:func:`HPyNumber_Check` +* :c:func:`HPySlice_New` +* :c:func:`HPySlice_Unpack` +* :c:func:`HPyTracker_Add` +* :c:func:`HPyTracker_Close` +* :c:func:`HPyTracker_ForgetAll` +* :c:func:`HPyTracker_New` +* :c:func:`HPyTupleBuilder_Build` +* :c:func:`HPyTupleBuilder_Cancel` +* :c:func:`HPyTupleBuilder_New` +* :c:func:`HPyTupleBuilder_Set` +* :c:func:`HPyTuple_Check` +* :c:func:`HPyTuple_FromArray` +* :c:func:`HPyType_FromSpec` +* :c:func:`HPyType_GenericNew` +* :c:func:`HPyType_GetName` +* :c:func:`HPyType_IsSubtype` +* :c:func:`HPyUnicode_AsASCIIString` +* :c:func:`HPyUnicode_AsLatin1String` +* :c:func:`HPyUnicode_AsUTF8AndSize` +* :c:func:`HPyUnicode_AsUTF8String` +* :c:func:`HPyUnicode_Check` +* :c:func:`HPyUnicode_DecodeASCII` +* :c:func:`HPyUnicode_DecodeFSDefault` +* :c:func:`HPyUnicode_DecodeFSDefaultAndSize` +* :c:func:`HPyUnicode_DecodeLatin1` +* :c:func:`HPyUnicode_EncodeFSDefault` +* :c:func:`HPyUnicode_FromEncodedObject` +* :c:func:`HPyUnicode_FromString` +* :c:func:`HPyUnicode_FromWideChar` +* :c:func:`HPyUnicode_ReadChar` +* :c:func:`HPyUnicode_Substring` +* :c:func:`HPy_ASCII` +* :c:func:`HPy_Absolute` +* :c:func:`HPy_Add` +* :c:func:`HPy_And` +* :c:func:`HPy_AsPyObject` +* :c:func:`HPy_Bytes` +* :c:func:`HPy_Call` +* :c:func:`HPy_CallMethod` +* :c:func:`HPy_CallTupleDict` +* :c:func:`HPy_Close` +* :c:func:`HPy_Compile_s` +* :c:func:`HPy_Contains` +* :c:func:`HPy_DelItem` +* :c:func:`HPy_DelItem_i` +* :c:func:`HPy_DelItem_s` +* :c:func:`HPy_DelSlice` +* :c:func:`HPy_Divmod` +* :c:func:`HPy_Dup` +* :c:func:`HPy_EvalCode` +* :c:func:`HPy_FatalError` +* :c:func:`HPy_Float` +* :c:func:`HPy_FloorDivide` +* :c:func:`HPy_FromPyObject` +* :c:func:`HPy_GetAttr` +* :c:func:`HPy_GetAttr_s` +* :c:func:`HPy_GetItem` +* :c:func:`HPy_GetItem_i` +* :c:func:`HPy_GetItem_s` +* :c:func:`HPy_GetIter` +* :c:func:`HPy_GetSlice` +* :c:func:`HPy_HasAttr` +* :c:func:`HPy_HasAttr_s` +* :c:func:`HPy_Hash` +* :c:func:`HPy_InPlaceAdd` +* :c:func:`HPy_InPlaceAnd` +* :c:func:`HPy_InPlaceFloorDivide` +* :c:func:`HPy_InPlaceLshift` +* :c:func:`HPy_InPlaceMatrixMultiply` +* :c:func:`HPy_InPlaceMultiply` +* :c:func:`HPy_InPlaceOr` +* :c:func:`HPy_InPlacePower` +* :c:func:`HPy_InPlaceRemainder` +* :c:func:`HPy_InPlaceRshift` +* :c:func:`HPy_InPlaceSubtract` +* :c:func:`HPy_InPlaceTrueDivide` +* :c:func:`HPy_InPlaceXor` +* :c:func:`HPy_Index` +* :c:func:`HPy_Invert` +* :c:func:`HPy_Is` +* :c:func:`HPy_IsTrue` +* :c:func:`HPy_LeavePythonExecution` +* :c:func:`HPy_Length` +* :c:func:`HPy_Long` +* :c:func:`HPy_Lshift` +* :c:func:`HPy_MatrixMultiply` +* :c:func:`HPy_Multiply` +* :c:func:`HPy_Negative` +* :c:func:`HPy_Or` +* :c:func:`HPy_Positive` +* :c:func:`HPy_Power` +* :c:func:`HPy_ReenterPythonExecution` +* :c:func:`HPy_Remainder` +* :c:func:`HPy_Repr` +* :c:func:`HPy_RichCompare` +* :c:func:`HPy_RichCompareBool` +* :c:func:`HPy_Rshift` +* :c:func:`HPy_SetAttr` +* :c:func:`HPy_SetAttr_s` +* :c:func:`HPy_SetCallFunction` +* :c:func:`HPy_SetItem` +* :c:func:`HPy_SetItem_i` +* :c:func:`HPy_SetItem_s` +* :c:func:`HPy_SetSlice` +* :c:func:`HPy_Str` +* :c:func:`HPy_Subtract` +* :c:func:`HPy_TrueDivide` +* :c:func:`HPy_Type` +* :c:func:`HPy_TypeCheck` +* :c:func:`HPy_Xor` diff --git a/graalpython/hpy/docs/api-reference/helpers.rst b/graalpython/hpy/docs/api-reference/helpers.rst new file mode 100644 index 0000000000..df7725f8d4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/helpers.rst @@ -0,0 +1,5 @@ +Misc Helpers +============ + +.. autocmodule:: runtime/helpers.c + :members: diff --git a/graalpython/hpy/docs/api-reference/hpy-call.rst b/graalpython/hpy/docs/api-reference/hpy-call.rst new file mode 100644 index 0000000000..d685898f1e --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-call.rst @@ -0,0 +1,5 @@ +HPy Call API +============ + +.. autocmodule:: autogen/public_api.h + :members: HPy_Call,HPy_CallMethod,HPy_CallTupleDict diff --git a/graalpython/hpy/docs/api-reference/hpy-ctx.rst b/graalpython/hpy/docs/api-reference/hpy-ctx.rst new file mode 100644 index 0000000000..040903f005 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-ctx.rst @@ -0,0 +1,9 @@ +HPy Context +=========== + +The ``HPyContext`` structure is also part of the API since it provides handles +for built-in objects. For a high-level description of the context, please also +read :ref:`api:hpycontext`. + +.. autocstruct:: hpy/cpython/autogen_ctx.h::_HPyContext_s + :members: diff --git a/graalpython/hpy/docs/api-reference/hpy-dict.rst b/graalpython/hpy/docs/api-reference/hpy-dict.rst new file mode 100644 index 0000000000..fb9400f4c1 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-dict.rst @@ -0,0 +1,5 @@ +HPy Dict +======== + +.. autocmodule:: autogen/public_api.h + :members: HPyDict_Check, HPyDict_New, HPyDict_Keys, HPyDict_Copy diff --git a/graalpython/hpy/docs/api-reference/hpy-err.rst b/graalpython/hpy/docs/api-reference/hpy-err.rst new file mode 100644 index 0000000000..aeba618295 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-err.rst @@ -0,0 +1,5 @@ +Exception Handling +================== + +.. autocmodule:: autogen/public_api.h + :members: HPyErr_SetFromErrnoWithFilename, HPyErr_SetFromErrnoWithFilenameObjects, HPy_FatalError, HPyErr_SetString, HPyErr_SetObject, HPyErr_Occurred, HPyErr_ExceptionMatches, HPyErr_NoMemory, HPyErr_Clear, HPyErr_NewException, HPyErr_NewExceptionWithDoc, HPyErr_WarnEx, HPyErr_WriteUnraisable diff --git a/graalpython/hpy/docs/api-reference/hpy-eval.rst b/graalpython/hpy/docs/api-reference/hpy-eval.rst new file mode 100644 index 0000000000..4e73e6d6fe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-eval.rst @@ -0,0 +1,8 @@ +HPy Eval +======== + +.. autocmodule:: hpy.h + :members: HPy_SourceKind + +.. autocmodule:: autogen/public_api.h + :members: HPy_Compile_s,HPy_EvalCode diff --git a/graalpython/hpy/docs/api-reference/hpy-field.rst b/graalpython/hpy/docs/api-reference/hpy-field.rst new file mode 100644 index 0000000000..2973af4c5e --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-field.rst @@ -0,0 +1,5 @@ +HPyField +======== + +.. autocmodule:: autogen/public_api.h + :members: HPyField_Load,HPyField_Store diff --git a/graalpython/hpy/docs/api-reference/hpy-gil.rst b/graalpython/hpy/docs/api-reference/hpy-gil.rst new file mode 100644 index 0000000000..b64f96042d --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-gil.rst @@ -0,0 +1,5 @@ +Leave/enter Python execution (GIL) +================================== + +.. autocmodule:: autogen/public_api.h + :members: HPy_LeavePythonExecution,HPy_ReenterPythonExecution diff --git a/graalpython/hpy/docs/api-reference/hpy-global.rst b/graalpython/hpy/docs/api-reference/hpy-global.rst new file mode 100644 index 0000000000..9cbaa539e4 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-global.rst @@ -0,0 +1,5 @@ +HPyGlobal +========= + +.. autocmodule:: autogen/public_api.h + :members: HPyGlobal_Store,HPyGlobal_Load diff --git a/graalpython/hpy/docs/api-reference/hpy-object.rst b/graalpython/hpy/docs/api-reference/hpy-object.rst new file mode 100644 index 0000000000..eeb5a6544b --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-object.rst @@ -0,0 +1,5 @@ +HPy Object +========== + +.. autocmodule:: autogen/public_api.h + :members: HPy_IsTrue,HPy_GetAttr,HPy_GetAttr_s,HPy_HasAttr,HPy_HasAttr_s,HPy_SetAttr,HPy_SetAttr_s,HPy_GetItem,HPy_GetItem_s,HPy_GetItem_i,HPy_GetSlice,HPy_SetItem,HPy_SetItem_s,HPy_SetItem_i,HPy_SetSlice,HPy_DelItem,HPy_DelItem_s,HPy_DelItem_i,HPy_DelSlice,HPy_Type,HPy_TypeCheck,HPy_Is,HPy_Repr,HPy_Str,HPy_ASCII,HPy_Bytes,HPy_RichCompare,HPy_RichCompareBool,HPy_Hash,HPy_SetCallFunction diff --git a/graalpython/hpy/docs/api-reference/hpy-sequence.rst b/graalpython/hpy/docs/api-reference/hpy-sequence.rst new file mode 100644 index 0000000000..21b7756e03 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-sequence.rst @@ -0,0 +1,23 @@ +HPy Lists and Tuples +==================== + +Building Tuples and Lists +------------------------- + +.. autocmodule:: autogen/public_api.h + :members: HPyTupleBuilder_New,HPyTupleBuilder_Set,HPyTupleBuilder_Build,HPyTupleBuilder_Cancel + +.. autocmodule:: autogen/public_api.h + :members: HPyListBuilder_New,HPyListBuilder_Set,HPyListBuilder_Build,HPyListBuilder_Cancel + +Tuples +------ + +.. autocmodule:: autogen/public_api.h + :members: HPyTuple_Check,HPyTuple_FromArray + +Lists +----- + +.. autocmodule:: autogen/public_api.h + :members: HPyList_Check,HPyList_New,HPyList_Append,HPyList_Insert diff --git a/graalpython/hpy/docs/api-reference/hpy-type.rst b/graalpython/hpy/docs/api-reference/hpy-type.rst new file mode 100644 index 0000000000..9fccdfa37a --- /dev/null +++ b/graalpython/hpy/docs/api-reference/hpy-type.rst @@ -0,0 +1,50 @@ +HPy Types and Modules +===================== + +Types, modules and their attributes (i.e. methods, members, slots, get-set +descriptors) are defined in a similar way. Section `HPy Type`_ documents the +type-specific and `HPy Module`_ documents the module-specific part. Section +`HPy Definition`_ documents how to define attributes for both, types and +modules. + + +HPy Type +-------- + +Definition +~~~~~~~~~~ + +.. autocmodule:: hpy/hpytype.h + :members: HPyType_Spec,HPyType_BuiltinShape,HPyType_SpecParam,HPyType_SpecParam_Kind,HPyType_HELPERS,HPyType_LEGACY_HELPERS,HPy_TPFLAGS_DEFAULT,HPy_TPFLAGS_BASETYPE,HPy_TPFLAGS_HAVE_GC + +Construction and More +~~~~~~~~~~~~~~~~~~~~~ + +.. autocmodule:: autogen/public_api.h + :members: HPyType_FromSpec, HPyType_GetName, HPyType_IsSubtype + +HPy Module +---------- + +.. c:macro:: HPY_EMBEDDED_MODULES + + If ``HPY_EMBEDDED_MODULES`` is defined, this means that there will be + several embedded HPy modules (and so, several ``HPy_MODINIT`` usages) in the + same binary. In this case, some restrictions apply: + + 1. all of the module's methods/member/slots/... must be defined in the same + file + 2. the embedder **MUST** declare the module to be *embeddable* by using macro + :c:macro:`HPY_MOD_EMBEDDABLE`. + +.. autocmodule:: hpy/hpymodule.h + :members: HPY_MOD_EMBEDDABLE,HPyModuleDef,HPy_MODINIT + +HPy Definition +-------------- + +Defining slots, methods, members, and get-set descriptors for types and modules +is done with HPy definition (represented by C struct :c:struct:`HPyDef`). + +.. autocmodule:: hpy/hpydef.h + :members: HPyDef,HPyDef_Kind,HPySlot,HPyMeth,HPyMember_FieldType,HPyMember,HPyGetSet,HPyDef_SLOT,HPyDef_METH,HPyDef_MEMBER,HPyDef_GET,HPyDef_SET,HPyDef_GETSET,HPyDef_CALL_FUNCTION diff --git a/graalpython/hpy/docs/api-reference/index.rst b/graalpython/hpy/docs/api-reference/index.rst new file mode 100644 index 0000000000..89380d6377 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/index.rst @@ -0,0 +1,69 @@ +API Reference +============= + +HPy's public API consists of three parts: + +1. The **Core API** as defined in the :doc:`public-api` +2. **HPy Helper** functions +3. **Inline Helper** functions + +Core API +-------- + +The **Core API** consists of inline functions that call into the Python +interpreter. Those functions will be implemented by each Python interpreter. In +:term:`CPython ABI` mode, many of these inline functions will just delegate to +a C API functions. In :term:`HPy Universal ABI` mode, they will call a function +pointer from the HPy context. This is the source of the performance change +between the modes. + +.. toctree:: + :maxdepth: 2 + + function-index + hpy-ctx + hpy-object + hpy-type + hpy-call + hpy-field + hpy-global + hpy-dict + hpy-sequence + hpy-gil + hpy-err + hpy-eval + public-api + + +HPy Helper Functions +-------------------- + +**HPy Helper** functions are functions (written in C) that will be compiled +together with the HPy extension's sources. The appropriate source files are +automatically added to the extension sources. The helper functions will, of +course, use the core API to interact with the interpreter. The main reason for +having the helper functions in the HPy extension is to avoid compatibility +problems due to different compilers. + +.. toctree:: + :maxdepth: 2 + + argument-parsing + build-value + formatting + structseq + helpers + + +Inline Helper Functions +----------------------- + +**Inline Helper** functions are ``static inline`` functions (written in C). +Those functions are usually small convenience functions that everyone could +write but in order to avoid duplicated effort, they are defined by HPy. + +.. toctree:: + :maxdepth: 2 + + inline-helpers + diff --git a/graalpython/hpy/docs/api-reference/inline-helpers.rst b/graalpython/hpy/docs/api-reference/inline-helpers.rst new file mode 100644 index 0000000000..2e719caf77 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/inline-helpers.rst @@ -0,0 +1,14 @@ +Inline Helpers +============== + +**Inline Helper** functions are ``static inline`` functions (written in C). +Those functions are usually small convenience functions that everyone could +write but in order to avoid duplicated effort, they are defined by HPy. + +One category of inline helpers are functions that convert the commonly used +but not fixed width C types, such as ``int``, or ``long long``, to HPy API. +The HPy API always uses well-defined fixed width types like ``int32`` or +``unsigned int8``. + +.. autocmodule:: hpy/inline_helpers.h + :members: diff --git a/graalpython/hpy/docs/api-reference/public-api.rst b/graalpython/hpy/docs/api-reference/public-api.rst new file mode 100644 index 0000000000..992df114f8 --- /dev/null +++ b/graalpython/hpy/docs/api-reference/public-api.rst @@ -0,0 +1,9 @@ +Public API Header +================= + +The core API is defined in `public_api.h +`_: + +.. literalinclude:: ../../hpy/tools/autogen/public_api.h + :language: c + :linenos: diff --git a/graalpython/hpy/docs/api-reference/structseq.rst b/graalpython/hpy/docs/api-reference/structseq.rst new file mode 100644 index 0000000000..2772b3edfe --- /dev/null +++ b/graalpython/hpy/docs/api-reference/structseq.rst @@ -0,0 +1,15 @@ +Struct Sequences +================ + +Struct sequences are subclasses of tuple. Similar to the API for creating +tuples, HPy provides an API to create struct sequences. This is a builder API +such that the struct sequence is guaranteed not to be written after it is +created. + +.. note:: + + There is no specific getter function for struct sequences. Just use one of + :c:func:`HPy_GetItem`, :c:func:`HPy_GetItem_i`, or :c:func:`HPy_GetItem_s`. + +.. autocmodule:: hpy/runtime/structseq.h + :members: HPyStructSequence_Field,HPyStructSequence_Desc,HPyStructSequence_UnnamedField,HPyStructSequence_NewType,HPyStructSequence_New diff --git a/graalpython/hpy/docs/api.rst b/graalpython/hpy/docs/api.rst new file mode 100644 index 0000000000..7146e024e2 --- /dev/null +++ b/graalpython/hpy/docs/api.rst @@ -0,0 +1,581 @@ +HPy API Introduction +==================== + +Handles +------- + +The "H" in HPy stands for **handle**, which is a central concept: handles are +used to hold a C reference to Python objects, and they are represented by the +C ``HPy`` type. They play the same role as ``PyObject *`` in the ``Python.h`` +API, albeit with some important differences which are detailed below. + +When they are no longer needed, handles must be closed by calling +``HPy_Close``, which plays more or less the same role as ``Py_DECREF``. +Similarly, if you need a new handle for an existing object, you can duplicate +it by calling ``HPy_Dup``, which plays more or less the same role as +``Py_INCREF``. + +The HPy API strictly follows these rules: + +- ``HPy`` handles returned by a function are **never borrowed**, i.e., + the caller must either close or return it. +- ``HPy`` handles passed as function arguments are **never stolen**; + if you receive a ``HPy`` handle argument from your caller, you should never close it. + +These rules makes the code simpler to reason about. Moreover, no reference +borrowing enables the Python implementations to use whatever internal +representation they wish. For example, the object returned by ``HPy_GetItem_i`` +may be created on demand from some compact internal representation, which does +not need to convert itself to full blown representation in order to hold onto +the borrowed object. + +We strongly encourage the users of HPy to also internally follow these rules +for their own internal APIs and helper functions. For the sake of simplicity +and easier local reasoning and also because in the future, code adhering +to those rules may be suitable target for some scalable and precise static +analysis tool. + +The concept of handles is certainly not unique to HPy. Other examples include +Unix file descriptors, where you have ``dup()`` and ``close()``, and Windows' +``HANDLE``, where you have ``DuplicateHandle()`` and ``CloseHandle()``. + + +Handles vs ``PyObject *`` +~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to fully understand the way HPy handles work, it is useful to discuss +the ``Pyobject *`` pointer in ``Python.h``. These pointers always +point to the same object, and a python object's identity is completely given +by its address in memory, and two pointers with the same address can +be passed to ``Python.h`` API functions interchangeably. As a result, ``Py_INCREF`` +and ``Py_DECREF`` can be called with any reference to an object as long as the +total number of calls of ``incref`` is equal to the number of calls of ``decref`` +at the end of the object lifetime. + +Whereas using HPy API, each handle must be closed independently. + +Thus, the following perfectly valid piece of code using ``Python.h``:: + + void foo(void) + { + PyObject *x = PyLong_FromLong(42); // implicit INCREF on x + PyObject *y = x; + Py_INCREF(y); // INCREF on y + /* ... */ + Py_DECREF(x); + Py_DECREF(x); // two DECREF on x + } + +Becomes using HPy API: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: foo + :end-before: // END: foo + +Calling any HPy function on a closed handle is an error. Calling +``HPy_Close()`` on the same handle twice is an error. Forgetting to call +``HPy_Close()`` on a handle results in a memory leak. When running in +:ref:`debug-mode:debug mode`, HPy actively checks that you don't +close a handle twice and that you don't forget to close any. + + +.. note:: + Debug mode is a good example of how powerful it is to decouple the + identity and therefore the lifetime of handles and those of objects. + If you find a memory leak on CPython, you know that you are missing a + ``Py_DECREF`` somewhere but the only way to find the corresponding + ``Py_INCREF`` is to manually and carefully study the source code. + On the other hand, if you forget to call ``HPy_Close()``, debug mode + is able to identify the precise code location which created the unclosed + handle. Similarly, if you try to operate on a closed handle, it will + identify the precise code locations which created and closed it. This is + possible because handles are associated with a single call to a C/API + function. As a result, given a handle that is leaked or used after freeing, + it is possible to identify exactly the C/API function that produced it. + + +Remember that ``Python.h`` guarantees that multiple references to the same +object results in the very same ``PyObject *`` pointer. Thus, it is +possible to compare the pointer addresses to check whether they refer +to the same object:: + + int is_same_object(PyObject *x, PyObject *y) + { + return x == y; + } + +On the other hand, in HPy, each handle is independent and it is common to have +two different handles which point to the same underlying object, so comparing +two handles directly is ill-defined. To prevent this kind of common error +(especially when porting existing code to HPy), the ``HPy`` C type is opaque +and the C compiler actively forbids comparisons between them. To check for +identity, you can use ``HPy_Is()``: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: is_same_object + :end-before: // END: is_same_object + +.. note:: + The main benefit of opaque handle semantics is that implementations are + allowed to use very different models of memory management. On CPython, + implementing handles is trivial because ``HPy`` is basically ``PyObject *`` + in disguise, and ``HPy_Dup()`` and ``HPy_Close()`` are just aliases for + ``Py_INCREF`` and ``Py_DECREF``. + + Unlike CPython, PyPy does not use reference counting to manage memory: + instead, it uses a *moving GC*, which means that the address of an object + might change during its lifetime, and this makes it hard to implement + semantics like ``PyObject *``'s where the address *identifies* the object, + and this is directly exposed to the user. HPy solves this problem: on + PyPy, handles are integers which represent indices into a list, which + is itself managed by the GC. When an address changes, the GC edits the + list, without having to touch all the handles which have been passed to C. + + +HPyContext +----------- + +All HPy function calls take an ``HPyContext`` as a first argument, which +represents the Python interpreter all the handles belong to. Strictly +speaking, it would be possible to design the HPy API without using +``HPyContext``: after all, all HPy function calls are ultimately mapped to +``Python.h`` function call, where there is no notion of context. + +One of the reasons to include ``HPyContext`` from the day one is to be +future-proof: it is conceivable to use it to hold the interpreter or the +thread state in the future, in particular when there will be support for +sub-interpreters. Another possible usage could be to embed different versions +or implementations of Python inside the same process. In addition, the +``HPyContext`` may also be extended by adding new functions to the end without +breaking any extensions built against the current ``HPyContext``. + +Moreover, ``HPyContext`` is used by the :term:`HPy Universal ABI` to contain a +sort of virtual function table which is used by the C extensions to call back +into the Python interpreter. + +.. _simple example: + +A simple example +----------------- + +In this section, we will see how to write a simple C extension using HPy. It +is assumed that you are already familiar with the existing ``Python.h`` API, so we +will underline the similarities and the differences with it. + +We want to create a function named ``myabs`` and ``double`` which takes a +single argument and computes its absolute value: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: myabs + :end-before: // END: myabs + +There are a couple of points which are worth noting: + + * We use the macro ``HPyDef_METH`` to declare we are going to define a HPy + function called ``myabs``. + + * The function will be available under the name ``"myabs"`` in our Python + module. + + * The actual C function which implements ``myabs`` is called ``myabs_impl`` + and is inferred by the macro. The macro takes the name and adds ``_impl`` + to the end of it. + + * It uses the ``HPyFunc_O`` calling convention. Like ``METH_O`` in ``Python.h``, + ``HPyFunc_O`` means that the function receives a single argument on top of + ``self``. + + * ``myabs_impl`` takes two arguments of type ``HPy``: handles for ``self`` + and the argument, which are guaranteed to be valid. They are automatically + closed by the caller, so there is no need to call ``HPy_Close`` on them. + + * ``myabs_impl`` returns a handle, which has to be closed by the caller. + + * ``HPy_Absolute`` is the equivalent of ``PyNumber_Absolute`` and + computes the absolute value of the given argument. + + * We also do not call ``HPy_Close`` on the result returned to the caller. + We must return a valid handle. + +.. note:: + Among other things, + the ``HPyDef_METH`` macro is needed to maintain compatibility with CPython. + In CPython, C functions and methods have a C signature that is different to + the one used by HPy: they don't receive an ``HPyContext`` and their arguments + have the type ``PyObject *`` instead of ``HPy``. The macro automatically + generates a trampoline function whose signature is appropriate for CPython and + which calls the ``myabs_impl``. This trampoline is then used from both the + CPython ABI and the CPython implementation of the universal ABI, but other + implementations of the universal ABI will usually call directly the HPy + function itself. + +The second function definition is a bit different: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: double + :end-before: // END: double + +This shows off the other way of creating functions. + + * This example is much the same but the difference is that we use + ``HPyDef_METH_IMPL`` to define a function named ``double``. + + * The difference between ``HPyDef_METH_IMPL`` and ``HPyDef_METH`` is that + the former needs to be given a name for a the functions as the third + argument. + +Now, we can define our module: + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: methodsdef + :end-before: // END: methodsdef + +This part is very similar to the one you would write with ``Python.h``. Note that +we specify ``myabs`` (and **not** ``myabs_impl``) in the method table. There +is also the ``.legacy_methods`` field, which allows to add methods that use the +``Python.h`` API, i.e., the value should be an array of ``PyMethodDef``. This +feature enables support for hybrid extensions in which some of the methods +are still written using the ``Python.h`` API. + +Note that the HPy module does not specify its name. HPy does not support the legacy +single phase module initialization and the only module initialization approach is +the multi-phase initialization (`PEP 489 `_). +With multi-phase module initialization, +the name of the module is always taken from the ``ModuleSpec`` (`PEP 451 `_) +, i.e., most likely from the name used in the ``import {{name}}`` statement that +imported your module. + +This is the only difference stemming from multi-phase module initialization in this +simple example. +As long as there is no need for any further initialization, we can just "register" +our module using the ``HPy_MODINIT`` convenience macro. The first argument is the +name of the extension file and is needed for HPy, among other things, to be able +to generate the entry point for CPython called ``PyInit_{{name}}``. The second argument +is the ``HPyModuleDef`` we just defined. + +.. literalinclude:: examples/simple-example/simple.c + :start-after: // BEGIN: moduledef + :end-before: // END: moduledef + +Building the module +~~~~~~~~~~~~~~~~~~~~ + +Let's write a ``setup.py`` to build our extension: + +.. literalinclude:: examples/simple-example/setup.py + :language: python + +We can now build the extension by running ``python setup.py build_ext -i``. On +CPython, it will target the :term:`CPython ABI` by default, so you will end up with +a file named e.g. ``simple.cpython-37m-x86_64-linux-gnu.so`` which can be +imported directly on CPython with no dependency on HPy. + +To target the :term:`HPy Universal ABI` instead, it is possible to pass the +option ``--hpy-abi=universal`` to ``setup.py``. The following command will +produce a file called ``simple.hpy.so`` (note that you need to specify +``--hpy-abi`` **before** ``build_ext``, since it is a global option):: + + python setup.py --hpy-abi=universal build_ext -i + +.. note:: + This command will also produce a Python file named ``simple.py``, which + loads the HPy module using the ``universal.load`` function from + the ``hpy`` Python package. + +VARARGS calling convention +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If we want to receive more than a single arguments, we need the +``HPy_METH_VARARGS`` calling convention. Let's add a function ``add_ints`` +which adds two integers: + +.. literalinclude:: examples/snippets/hpyvarargs.c + :start-after: // BEGIN: add_ints + :end-before: // END: add_ints + +There are a few things to note: + + * The C signature is different than the corresponding ``Python.h`` + ``METH_VARARGS``: in particular, instead of taking a tuple ``PyObject *args``, + we take an array of ``HPy`` and its size. This allows the call to happen + more efficiently, because you don't need to create a tuple just to pass the + arguments. + + * We call ``HPyArg_Parse`` to parse the arguments. Contrarily to almost all + the other HPy functions, this is **not** a thin wrapper around + ``PyArg_ParseTuple`` because as stated above we don't have a tuple to pass + to it, although the idea is to mimic its behavior as closely as + possible. The parsing logic is implemented from scratch inside HPy, and as + such there might be missing functionality during the early stages of HPy + development. + + * If an error occurs, we return ``HPy_NULL``: we cannot simply ``return NULL`` + because ``HPy`` is not a pointer type. + +Once we have written our function, we can add it to the ``SimpleMethods[]`` +table, which now becomes: + +.. literalinclude:: examples/snippets/hpyvarargs.c + :start-after: // BEGIN: methodsdef + :end-before: // END: methodsdef + +Creating types in HPy +--------------------- + +Creating Python types in an HPy extension is again very similar to the C API +with the difference that HPy only supports creating types from a specification. +This is necessary because there is no such C-level type as ``PyTypeObject`` +since that would expose the internal implementation. + + +Creating a simple type in HPy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +This section assumes that the user wants to define a type that stores some data +in a C-level structure. As an example, we will create a simple C structure +``PointObject`` that represents a two-dimensional point. + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: PointObject + :end-before: // END: PointObject + +The macro call ``HPyType_HELPERS(PointObject)`` generates useful helper +facilities for working with the type. It generates a C enum +``PointObject_SHAPE`` and a helper function ``PointObject_AsStruct``. The enum +is used in the type specification. The helper function is used to efficiently +retrieving the pointer ``PointObject *`` from an HPy handle to be able to access +the C structure. We will use this helper function to implement the methods, +get-set descriptors, and slots. + +It makes sense to expose fields ``PointObject.x`` and ``PointObject.y`` as +Python-level members. To do so, we need to define members by specifying their +name, type, and location using HPy's convenience macro ``HPyDef_MEMBER``: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: members + :end-before: // END: members + +The first argument of the macro is the name for the C glabal variable that will +store the necessary information. We will need that later for registration of +the type. The second, third, and fourth arguments are the Python-level name, the +C type of the member, and the offset in the C structure, respectively. + +Similarly, methods and get-set descriptors can be defined. For example, method +``foo`` is an instance method that takes no arguments (the self argument is, of +course, implicit), does some computation with fields ``x`` and ``y`` and +returns a Python ``int``: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: methods + :end-before: // END: methods + +Get-set descriptors are also defined in a very similar way as methods. The +following example defines a get-set descriptor for attribute ``z`` which is +calculated from the ``x`` and ``y`` fields of the struct. + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: getset + :end-before: // END: getset + +It is also possible to define a get-descriptor or a set-descriptor by using +HPy's macros ``HPyDef_GET`` and ``HPyDef_SET`` in the same way. + +HPy also supports type slots. In this example, we will define slot +``HPy_tp_new`` (which corresponds to magic method ``__new__``) to initialize +fields ``x`` and ``y`` when constructing the object: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: slots + :end-before: // END: slots + +After everything was defined, we need to create a list of all defines such that +we are able to eventually register them to the type: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: defines + :end-before: // END: defines + +Please note that it is required to terminate the list with ``NULL``. +We can now create the actual type specification by appropriately filling an +``HPyType_Spec`` structure: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: spec + :end-before: // END: spec + +First, we need to define the name of the type by setting a C string to member +``name``. Since this type has a C structure, we need to define the ``basicsize`` +and best practice is to set it to ``sizeof(PointObject)``. Also best practice is +to set ``builtin_shape`` to ``PointObject_SHAPE`` where ``PointObject_SHAPE`` is +generated by the previous usage of macro ``HPyType_HELPERS(PointObject)``. Last +but not least, we need to register the defines by setting field ``defines`` to +the previously defined array ``Point_defines``. + +The type specification for the simple type ``simple_type.Point`` represented in +C by structure ``PointObject`` is now complete. All that remains is to create +the type object and add it to the module. + +We will define a module execute slot, which is executed by the runtime right +after the module is created. The purpose of the execute slot is to initialize +the newly created module object. We can then add the type by using +:c:func:`HPyHelpers_AddType`: + +.. literalinclude:: examples/hpytype-example/simple_type.c + :start-after: // BEGIN: add_type + :end-before: // END: add_type + +Also look at the full example at: :doc:`examples/hpytype-example/simple_type`. + + +Legacy types +~~~~~~~~~~~~ + +A type whose struct starts with ``PyObject_HEAD`` (either directly by +embedding it in the type struct or indirectly by embedding another struct like +``PyLongObject``) is a *legacy type*. A legacy type must set +``.builtin_shape = HPyType_BuiltinShape_Legacy`` +in its ``HPyType_Spec``. The counterpart (i.e. a non-legacy type) is called HPy +pure type. + +Legacy types are available to allow gradual porting of existing CPython +extensions. It is possible to reuse existing ``PyType_Slot`` entities (i.e. +slots, methods, members, and get/set descriptors). The idea is that you can then +migrate one after each other while still running the tests. + +The major restriction when using legacy types is that you cannot build a +universal binary of your HPy extension (i.e. you cannot use :term:`HPy Universal +ABI`). The resulting binary will be specific to the Python interpreter used for +building. Therefore, the goal should always be to fully migrate to HPy pure +types. + +A type with ``.legacy_slots != NULL`` is required to have +``HPyType_BuiltinShape_Legacy`` and to include ``PyObject_HEAD`` at the start of +its struct. It would be easy to relax this requirement on CPython (where the +``PyObject_HEAD`` fields are always present) but a large burden on other +implementations (e.g. PyPy, GraalPy) where a struct starting with +``PyObject_HEAD`` might not exist. + +Types created via the old Python C API are automatically legacy types. + +This section does not provide a dedicated example for how to create and use +legacy types because the :doc:`porting-example/index` already shows how that +is useful during incremental migration to HPy. + +Inherit from a built-in type +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +HPy also supports inheriting from following built-in types: + + * ``type`` + + * ``int`` + + * ``float`` + + * ``unicode`` + + * ``tuple`` + + * ``list`` + +Inheriting from built-in types is straight forward if you don't have a C +structure that represents your type. In other words, you can simply inherit +from, e.g., ``str`` if the ``basicsize`` in your type specification is ``0``. +For example: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: spec_Dummy + :end-before: // END: spec_Dummy + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: add_Dummy + :end-before: // END: add_Dummy + +This case is simple because there is no ``Dummy_AsStruct`` since there is no +associated C-level structure. + +It is, however, more involved if your type also defines its own C structure +(i.e. ``basicsize > 0`` in the type specification). In this case, it is strictly +necessary to use the right *built-in shape*. + +**What is the right built-in shape?** + +This question is easy to answer: Each built-in shape (except of +:c:enumerator:`HPyType_BuiltinShape_Legacy`) represents a built-in type. You +need to use the built-in shape that fits to the specified base class. The +mapping is described in :c:enum:`HPyType_BuiltinShape`. + +Let's do an example. Assume we want to define a type that stores the natural +language of a unicode string to the unicode object but the object should still +just behave like a Python unicode object. So, we define struct +``LanguageObject``: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: LanguageObject + :end-before: // END: LanguageObject + +As you can see, we already specify the built-in shape here using +``HPyType_HELPERS(LanguageObject, HPyType_BuiltinShape_Unicode)``. Then, in the +type specification, we do: + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: spec_Language + :end-before: // END: spec_Language + +In the last step, when actually creating the type from the specification, we +need to define that its base class is ``str`` (aka. ``UnicodeType``): + +.. literalinclude:: examples/hpytype-example/builtin_type.c + :start-after: // BEGIN: add_Language + :end-before: // END: add_Language + +Function ``LanguageObject_AsStruct`` (which is generated by ``HPyType_HELPERS``) +will then return a pointer to ``LanguageObject``. + +To summarize this: Specifying a type that inherits from a built-in type needs to +be considered in three places: + +1. Pass the appropriate built-in shape to :c:macro:`HPyType_HELPERS`. +2. Assign ``SHAPE(TYPE)`` to :c:member:`HPyType_Spec.builtin_shape`. +3. Specify the desired base class in the type specification parameters. + +For more information about the built-in shape and for a technical explanation +for why it is required, see :c:member:`HPyType_Spec.builtin_shape` and +:c:enum:`HPyType_BuiltinShape`. + +More Examples +------------- + +The :doc:`porting-example/index` shows another complete example +of HPy extension ported from Python/C API. + +The `HPy project space `_ on GitHub +contains forks of some popular Python extensions ported to HPy as +a proof of concept/feasibility studies, such as the +`Kiwi solver `_. +Note that those forks may not be up to date with their upstream projects +or with the upstream HPy changes. + +HPy unit tests +~~~~~~~~~~~~~~ + +HPy usually has tests for each API function. This means that there is lots of +examples available by looking at the tests. However, the test source uses +many macros and is hard to read. To overcome this we supply a utility to +export clean C sources for the tests. Since the HPy tests are not shipped by +default, you need to clone the HPy repository from GitHub: + +.. code-block:: console + + > git clone https://github.com/hpyproject/hpy.git + +After that, install all test requirements and dump the sources: + +.. code-block:: console + + > cd hpy + > python3 -m pip install pytest filelock + > python3 -m pytest --dump-dir=test_sources test/ + +This will dump the generated test sources into folder ``test_sources``. Note, +that the tests won't be executed but skipped with an appropriate message. diff --git a/graalpython/hpy/docs/changelog.rst b/graalpython/hpy/docs/changelog.rst new file mode 100644 index 0000000000..e2915b98d3 --- /dev/null +++ b/graalpython/hpy/docs/changelog.rst @@ -0,0 +1,235 @@ +Changelog +========= + +Version 0.9 (April 25th, 2023) +------------------------------ + +This release adds numerous major features and indicates the end of HPy's *alhpa* +phase. We've migrated several key packages to HPy (for a list, see our website +https://hpyproject.org) and we are now confident that HPy is mature enough for +being used as serious extension API. We also plan that the next major release +will be ``1.0``. + +Major new features +~~~~~~~~~~~~~~~~~~ + +Support subclasses of built-in types + It is now possible to create pure HPy types that inherit from built-in types + like ``type`` or ``float``. This was already possible before but in a very + limited way, i.e., by setting :c:member:`HPyType_Spec.basicsize` to ``0``. In + this case, the type implicitly inherited the basic size of the supertype but + that also means that you cannot have a custom C struct. It is now possible + inherit from a built-in type **AND** have a custom C struct. For further + reference, see :c:member:`HPyType_Spec.builtin_shape` and + :c:enum:`HPyType_BuiltinShape`. + +Support for metaclasses + HPy now supports creating types with metaclasses. This can be done by passing + type specification parameter with kind + :c:enumerator:`HPyType_SpecParam_Metaclass` when calling + :c:func:`HPyType_FromSpec`. + +:term:`HPy Hybrid ABI` + In addition to :term:`CPython ABI` and :term:`HPy Universal ABI`, we now + introduced the Hybrid ABI. The major difference is that whenever you use a + legacy API like :c:func:`HPy_AsPyObject` or :c:func:`HPy_FromPyObject`, the + prdouced binary will then be specific to one interpreter. This was necessary + to ensure that universal binaries are really portable and can be used on any + HPy-capable interpreter. + +:doc:`trace-mode` + Similar to the :doc:`debug-mode`, HPy now provides the Trace Mode that can be + enabled at runtime and helps analyzing API usage and identifying performance + issues. + +:ref:`porting-guide:multi-phase module initialization` + HPy now support multi-phase module initialization which is an important + feature in particular needed for two important use cases: (1) module state + support (which is planned to be introduced in the next major release), and (2) + subinterpreters. We decided to drop support for single-phase module + initialization since this makes the API cleaner and easier to use. + +HPy :ref:`porting-guide:calling protocol` + This was a big missing piece and is now eventually available. It enables slot + ``HPy_tp_call``, which can now be used in the HPy type specification. We + decided to use a calling convention similar to CPython's vectorcall calling + convention. This is: the arguments are passed in a C array and the keyword + argument names are provided as a Python tuple. Before this release, the only + way to create a callable type was to set the special method ``__call__``. + However, this has several disadvantages. In particlar, poor performance on + CPython (and maybe other implementations) and it was not possible to have + specialized call function implementations per object (see + :c:func:`HPy_SetCallFunction`) + +Added APIs +~~~~~~~~~~ + +Deleting attributes and items + :c:func:`HPy_DelAttr`, :c:func:`HPy_DelAttr_s`, :c:func:`HPy_DelItem`, :c:func:`HPy_DelItem_i`, :c:func:`HPy_DelItem_s` + +Capsule API + :c:func:`HPyCapsule_New`, :c:func:`HPyCapsule_IsValid`, :c:func:`HPyCapsule_Get`, :c:func:`HPyCapsule_Set` + +Eval API + :c:func:`HPy_Compile_s` and :c:func:`HPy_EvalCode` + +Formatting helpers + :c:func:`HPyUnicode_FromFormat` and :c:func:`HPyErr_Format` + +Contextvar API + :c:func:`HPyContextVar_New`, :c:func:`HPyContextVar_Get`, :c:func:`HPyContextVar_Set` + +Unicode API + :c:func:`HPyUnicode_FromEncodedObject` and :c:func:`HPyUnicode_Substring` + +Dict API + :c:func:`HPyDict_Keys` and :c:func:`HPyDict_Copy` + +Type API + :c:func:`HPyType_GetName` and :c:func:`HPyType_IsSubtype` + +Slice API + :c:func:`HPySlice_Unpack` and :c:func:`HPySlice_AdjustIndices` + +Structseq API + :c:func:`HPyStructSequence_NewType`, :c:func:`HPyStructSequence_New` + +Call API + :c:func:`HPy_Call`, :c:func:`HPy_CallMethod`, :c:func:`HPy_CallMethodTupleDict`, :c:func:`HPy_CallMethodTupleDict_s` + +HPy call protocol + :c:func:`HPy_SetCallFunction` + +Debug mode +~~~~~~~~~~ + +* Detect closing and returning (without dup) of context handles +* Detect invalid usage of stored ``HPyContext *`` pointer +* Detect invalid usage of tuple and list builders +* Added Windows support for checking invalid use of raw data pointers (e.g + ``HPyUnicode_AsUTF8AndSize``) after handle was closed. +* Added support for backtrace on MacOS + +Documentation +~~~~~~~~~~~~~ + +* Added incremental :doc:`porting-example/index` +* Added :doc:`quickstart` guide +* Extended :doc:`api-reference/index` +* Added :doc:`api-reference/function-index` +* Added possiblity to generate examples from tests with argument ``--dump-dir`` + (see :ref:`api:hpy unit tests`) +* Added initial :doc:`contributing/index` docs + +Incompatible changes to version 0.0.4 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Simplified ``HPyDef_*`` macros +* Changed macro :c:macro:`HPy_MODINIT` because of multi-phase module init + support. +* Replace environment variable ``HPY_DEBUG`` by ``HPY`` (see :doc:`debug-mode` + or :doc:`trace-mode`). +* Changed signature of ``HPyFunc_VARARGS`` and ``HPyFunc_ KEYWORDS`` to align + with HPy's call protocol calling convention. + +Supported Python versions +~~~~~~~~~~~~~~~~~~~~~~~~~ + +* Added Python 3.11 support +* Preliminary Python 3.12 support +* Dropped Python 3.6 support (since EOL) +* Dropped Python 3.7 support (since EOL by June 2023) + +Misc +~~~~ + +* Ensure deterministic auto-generation +* Ensure ABI backwards compatibility + + * Explicitly define slot within HPyContext of function pointers and handles + * Compile HPy ABI version into binary and verify at load time +* Added proper support for object members ``HPyMember_OBJECT`` +* Changed :c:func:`HPyBytes_AsString` and :c:func:`HPyBytes_AS_STRING` to return ``const char *`` +* Use fixed-width integers in context functions + + + +Version 0.0.4 (May 25th, 2022) +------------------------------ + +New Features/API: + + - HPy headers are C++ compliant + - Python 3.10 support + - `HPyField `_: + References to Python objects that can be stored in raw native memory owned by Python objects. + + - New API functions: ``HPyField_Load``, ``HPyField_Store`` + - `HPyGlobal `_: + References to Python objects that can be stored into a C global variable. + + - New API functions: ``HPyGlobal_Load``, ``HPyGlobal_Store`` + - Note: ``HPyGlobal`` does not allow to share Python objects between (sub)interpreters + + - `GIL support `_ + - New API functions: ``HPy_ReenterPythonExecution``, ``HPy_LeavePythonExecution`` + + - `Value building support `_ (``HPy_BuildValue``) + + - New type slots + + - ``HPy_mp_ass_subscript``, ``HPy_mp_length``, ``HPy_mp_subscript`` + - ``HPy_tp_finalize`` + + - Other new API functions + + - ``HPyErr_SetFromErrnoWithFilename``, ``HPyErr_SetFromErrnoWithFilenameObjects`` + - ``HPyErr_ExceptionMatches`` + - ``HPyErr_WarnEx`` + - ``HPyErr_WriteUnraisable`` + - ``HPy_Contains`` + - ``HPyLong_AsVoidPtr`` + - ``HPyLong_AsDouble`` + - ``HPyUnicode_AsASCIIString``, ``HPyUnicode_DecodeASCII`` + - ``HPyUnicode_AsLatin1String``, ``HPyUnicode_DecodeLatin1`` + - ``HPyUnicode_DecodeFSDefault``, ``HPyUnicode_DecodeFSDefaultAndSize`` + - ``HPyUnicode_ReadChar`` + +Debug mode: + + - Support activation of debug mode via environment variable ``HPY_DEBUG`` + - Support capturing stack traces of handle allocations + - Check for invalid use of raw data pointers (e.g ``HPyUnicode_AsUTF8AndSize``) after handle was closed. + - Detect invalid handles returned from extension functions + - Detect incorrect closing of handles passed as arguments + +Misc Changes: + + - Removed unnecessary prefix ``"m_"`` from fields of ``HPyModuleDef`` (incompatible change) + - For HPy implementors: new pytest mark for HPy tests assuming synchronous GC + +Version 0.0.3 (September 22nd, 2021) +------------------------------------ + +This release adds various new API functions (see below) and extends the debug +mode with the ability to track closed handles. +The default ABI mode now is 'universal' for non-CPython implementations. +Also, the type definition of ``HPyContext`` was changed and it's no longer a +pointer type. +The name of the HPy dev package was changed to 'hpy' (formerly: 'hpy.devel'). +Macro HPy_CAST was replaced by HPy_AsStruct. + +New features: + + - Added helper HPyHelpers_AddType for creating new types + - Support format specifier 's' in HPyArg_Parse + - Added API functions: HPy_Is, HPy_AsStructLegacy (for legacy types), + HPyBytes_FromStringAndSize, HPyErr_NewException, HPyErr_NewExceptionWithDoc, + HPyUnicode_AsUTF8AndSize, HPyUnicode_DecodeFSDefault, HPyImport_ImportModule + - Debug mode: Implemented tracking of closed handles + - Debug mode: Add hook for invalid handle access + +Bug fixes: + + - Distinguish between pure and legacy types + - Fix Sphinx doc errors diff --git a/graalpython/hpy/docs/conf.py b/graalpython/hpy/docs/conf.py new file mode 100644 index 0000000000..45a9077616 --- /dev/null +++ b/graalpython/hpy/docs/conf.py @@ -0,0 +1,134 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import sys +import os +import re +import datetime + +# -- Project information ----------------------------------------------------- + +project = "HPy" +copyright = "2019-{}, HPy Collective".format(datetime.date.today().year) +author = "HPy Collective" + +# The full version, including alpha/beta/rc tags +release = "0.9" + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autosectionlabel", + "sphinx_c_autodoc", + "sphinx_c_autodoc.viewcode", + "sphinx_rtd_theme", +] + +autosectionlabel_prefix_document = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# -- autodoc ----------------------------------------------------------------- + +autodoc_member_order = "bysource" + +# -- sphinx_c_autodoc -------------------------------------------------------- + +c_autodoc_roots = [ + "../hpy/devel/include", + "../hpy/devel/src", + "../hpy/tools", +] + + +def pre_process(app, filename, contents, *args): + # FIXME: the missing typedef for 'HPy' causes the file formatting to fail + if filename.endswith('public_api.h'): + contents[0] = '#include "../../devel/include/hpy.h"\n' + contents[0] + if filename.endswith('autogen_ctx.h'): + contents[0] = 'typedef int HPy;' + contents[0] + + # remove HPyAPI_HELPER so that the sphinx-c-autodoc and clang + # find and render the API functions + contents[:] = [ + re.sub(r"^(HPyAPI_HELPER|HPyAPI_INLINE_HELPER|HPy_ID\(\d+\))", r"", part, flags=re.MULTILINE) + for part in contents + ] + + +def setup(app): + app.connect("c-autodoc-pre-process", pre_process) + + +def setup_clang(): + """ + Make sure clang is set up correctly for the sphinx_c_autodoc extension. + + The Python clang package requires a matching libclang*.so. Our + ``doc/requirements.txt`` file specifies ``clang==10.0.1``, so we need + ``libclang-10``. + + On Ubuntu 20.04 (and possibly later), this can be installed with + ``apt install libclang-10-dev`` and the Python clang package finds the + appropriate .so automatically. + + However, ReadTheDocs has an older Ubuntu that only packages libclang-6.0. + The Python ``clang==10.0.1`` packages supports this older .so, but + needs to be explicitly told where to find it. + + If you encounter issues with a local build, please start by checking that + the ``libclang-10-dev`` system package or equivalent is installed. + """ + from clang import cindex + if 'READTHEDOCS' in os.environ: + # TODO: Hopefully we can remove this setting of the libclang path once + # readthedocs updates its docker image to Ubuntu 20.04 which + # supports clang-10 and clang-11. + cindex.Config.set_library_file( + "/usr/lib/x86_64-linux-gnu/libclang-6.0.so.1" + ) + elif sys.platform == "darwin": + cindex.Config.set_library_file( + "/Library/Developer/CommandLineTools/usr/lib/libclang.dylib" + ) + + +setup_clang() + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# most the the code examples will be in C, let's make it the default +highlight_language = 'C' diff --git a/graalpython/hpy/docs/contributing/index.rst b/graalpython/hpy/docs/contributing/index.rst new file mode 100644 index 0000000000..1dba877a90 --- /dev/null +++ b/graalpython/hpy/docs/contributing/index.rst @@ -0,0 +1,33 @@ +Contributing +============ + +Getting Started +--------------- + +TBD + + +Adding New API +-------------- + +1. Add the function to ``hpy/tools/autogen/public_api.h``. If the CPython + equivalent function name is not the same (after removing the leading ``H``, + add an appropriate CPython function mapping in ``hpy/tools/autogen/conf.py``. + If the signature is complicated or there is no clear equivalent function, + the mapping should be ``None``, and follow the directions in the next step. + Otherwise all the needed functions will be autogenerated. + +2. If the function cannot be autogenerated (i.e. the mapping does not exist), + you must write the wrapper by hand. Add the function to ``NO_WRAPPER`` in + ``hpy/tools/autogen/debug.py``, and add a ``ctx_fname`` function to + ``hyp/devel/src/runtime/*.c`` (possibly adding the new file to ``setup.py``), + add a debug wrapper to ``hpy/debug/src/debug_ctx.c``, add a implementation + that uses the ``ctx`` variant to ``hpy/devel/include/hpy/cpython/misc.h`` and + add the declaration to ``hpy/devel/include/hpy/runtime/ctx_funcs.h``. + +3. Run ``make autogen`` which will turn the mapping into autogenerated functions + +4. Add a test for the functionality + +5. Build with ``python setup.py build_ext``. After that works, build with + ``python -m pip install -e .``, then run the test with ``python -m pytest ...``. diff --git a/graalpython/hpy/docs/debug-mode.rst b/graalpython/hpy/docs/debug-mode.rst new file mode 100644 index 0000000000..0f59bc9486 --- /dev/null +++ b/graalpython/hpy/docs/debug-mode.rst @@ -0,0 +1,169 @@ +Debug Mode +========== + +HPy includes a debug mode which does useful run-time checks to ensure that C +extensions use the API correctly. Its features include: + + 1. No special compilation flags are required: it is enough to compile the + extension with the Universal ABI. + + 2. Debug mode can be activated at *import time*, and it can be activated + per-extension. + + 3. You pay the overhead of debug mode only if you use it. Extensions loaded + without the debug mode run at full speed. + +This is possible because the whole of the HPy API is provided as part of the HPy +context, so debug mode can pass in a special debugging context without affecting +the performance of the regular context at all. + +.. note:: The debug mode is only available if the module (you want to use it + for) was built for :term:`HPy Universal ABI`. + +The debugging context can already check for: + +* Leaked handles. +* Handles used after they are closed. +* Tuple and list builder used after they were *closed* (i.e. cancelled or the + tuple/list was built). +* Reading from a memory which is no longer guaranteed to be still valid, + for example, the buffer returned by :c:func:`HPyUnicode_AsUTF8AndSize`, + :c:func:`HPyBytes_AsString`, and :c:func:`HPyBytes_AS_STRING`, after the + corresponding ``HPy`` handle was closed. +* Writing to memory which should be read-only, for example the buffer + returned by :c:func:`HPyUnicode_AsUTF8AndSize`, :c:func:`HPyBytes_AsString`, + and :c:func:`HPyBytes_AS_STRING` + + +Activating Debug Mode +--------------------- + +Debug mode works *only* for extensions built with HPy universal ABI. + +To enable debug mode, use environment variable ``HPY``. If ``HPY=debug``, then +all HPy modules are loaded with the trace context. Alternatively, it is also +possible to specify the mode per module like this: +``HPY=modA:debug,modB:debug``. + +In order to verify that your extension is being loaded in debug mode, use +environment variable ``HPY_LOG``. If this variable is set, then all HPy +extensions built in universal ABI mode print a message when loaded, such as: + +.. code-block:: console + + > import snippets + Loading 'snippets' in HPy universal mode with a debug context + +.. Note: the output above is tested in test_leak_detector_with_traces_output + +If the extension is built in CPython ABI mode, then the ``HPY_LOG`` environment +variable has no effect. + +An HPy extension module may be also explicitly loaded in debug mode using: + +.. code-block:: python + + from hpy.universal import load, MODE_DEBUG + mod = load(module_name, so_filename, mode=MODE_DEBUG) + +When loading HPy extensions explicitly, environment variables ``HPY_LOG`` +and ``HPY`` have no effect for that extension. + + +Using Debug Mode +---------------- + +By default, when debug mode detects an error it will either abort the process +(using :c:func:`HPy_FatalError`) or raise a fatal exception. This may sound very +strict but in general, it is not safe to continue the execution. + +When testing, aborting the process is unwanted. Module ``hpy.debug`` exposes the +``LeakDetector`` class to detect leaked ``HPy`` handles. For example: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: def test_leak_detector + :end-before: # END: test_leak_detector + +Additionally, the debug module also provides a pytest fixture, ``hpy_debug``, +that for the time being, enables the ``LeakDetector``. In the future, it +may also enable other useful debugging facilities. + +.. literalinclude:: examples/tests.py + :language: python + :start-at: from hpy.debug.pytest import hpy_debug + :end-at: # Run some HPy extension code + +.. warning:: The usage of ``LeakDetector`` or ``hpy_debug`` by itself does not + enable HPy debug mode! If debug mode is not enabled for any extension, then + those features have no effect. + +When dealing with handle leaks, it is useful to get a stack trace of the +allocation of the leaked handle. This feature has large memory requirements +and is therefore opt-in. It can be activated by: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: hpy.debug.set_handle_stack_trace_limit + :end-at: hpy.debug.set_handle_stack_trace_limit + +and disabled by: + +.. literalinclude:: examples/tests.py + :language: python + :start-at: hpy.debug.disable_handle_stack_traces + :end-at: hpy.debug.disable_handle_stack_traces + + +Example +------- + +.. note: The following output is tested in test_leak_detector_with_traces_output + +Following HPy function leaks a handle: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: test_leak_stacktrace + :end-before: // END: test_leak_stacktrace + +When this script is executed in debug mode: + +.. literalinclude:: examples/debug-example.py + :language: python + :end-before: print("SUCCESS") + +The output is:: + + Traceback (most recent call last): + File "/path/to/hpy/docs/examples/debug-example.py", line 7, in + snippets.test_leak_stacktrace() + File "/path/to/hpy/debug/leakdetector.py", line 43, in __exit__ + self.stop() + File "/path/to/hpy/debug/leakdetector.py", line 36, in stop + raise HPyLeakError(leaks) + hpy.debug.leakdetector.HPyLeakError: 1 unclosed handle: + + Allocation stacktrace: + /path/to/site-packages/hpy-0.0.4.dev227+gd7eeec6.d20220510-py3.8-linux-x86_64.egg/hpy/universal.cpython-38d-x86_64-linux-gnu.so(debug_ctx_Long_FromLong+0x45) [0x7f1d928c48c4] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x122c) [0x7f1d921a622c] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x14b1) [0x7f1d921a64b1] + /path/to/site-packages/hpy-0.0.4.dev227+gd7eeec6.d20220510-py3.8-linux-x86_64.egg/hpy/universal.cpython-38d-x86_64-linux-gnu.so(debug_ctx_CallRealFunctionFromTrampoline+0xca) [0x7f1d928bde1e] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x129b) [0x7f1d921a629b] + /path/to/site-packages/hpy_snippets-0.0.0-py3.8-linux-x86_64.egg/snippets.hpy.so(+0x1472) [0x7f1d921a6472] + /path/to/libpython3.8d.so.1.0(+0x10a022) [0x7f1d93807022] + /path/to/libpython3.8d.so.1.0(+0x1e986b) [0x7f1d938e686b] + /path/to/libpython3.8d.so.1.0(+0x2015e9) [0x7f1d938fe5e9] + /path/to/libpython3.8d.so.1.0(_PyEval_EvalFrameDefault+0x1008c) [0x7f1d938f875a] + /path/to/libpython3.8d.so.1.0(PyEval_EvalFrameEx+0x64) [0x7f1d938e86b8] + /path/to/libpython3.8d.so.1.0(_PyEval_EvalCodeWithName+0xfaa) [0x7f1d938fc8af] + /path/to/libpython3.8d.so.1.0(PyEval_EvalCodeEx+0x86) [0x7f1d938fca25] + /path/to/libpython3.8d.so.1.0(PyEval_EvalCode+0x4b) [0x7f1d938e862b] + +For the time being, HPy uses the glibc ``backtrace`` and ``backtrace_symbols`` +`functions `_. +Therefore all their caveats and limitations apply. Usual recommendations to get +more symbols in the traces and not only addresses, such as ``snippets.hpy.so(+0x122c)``, are: + +* link your native code with ``-rdynamic`` flag (``LDFLAGS="-rdynamic"``) +* build your code without optimizations and with debug symbols (``CFLAGS="-O0 -g"``) +* use ``addr2line`` command line utility, e.g.: ``addr2line -e /path/to/snippets.hpy.so -C -f +0x122c`` diff --git a/graalpython/hpy/docs/examples/debug-example.py b/graalpython/hpy/docs/examples/debug-example.py new file mode 100644 index 0000000000..0cd16cae36 --- /dev/null +++ b/graalpython/hpy/docs/examples/debug-example.py @@ -0,0 +1,10 @@ +# Run with HPY=debug +import hpy.debug +import snippets + +hpy.debug.set_handle_stack_trace_limit(16) +from hpy.debug.pytest import LeakDetector +with LeakDetector() as ld: + snippets.test_leak_stacktrace() + +print("SUCCESS") # Should not be actually printed diff --git a/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c b/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c new file mode 100644 index 0000000000..5ec749b690 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/builtin_type.c @@ -0,0 +1,106 @@ +#include +#include + +// BEGIN: spec_Dummy +static HPyType_Spec Dummy_spec = { + .name = "builtin_type.Dummy", + .basicsize = 0 +}; + +// END: spec_Dummy +static void make_Dummy(HPyContext *ctx, HPy module) +{ +// BEGIN: add_Dummy + HPyType_SpecParam param[] = { + { HPyType_SpecParam_Base, ctx->h_UnicodeType }, + { (HPyType_SpecParam_Kind)0 } + }; + if (!HPyHelpers_AddType(ctx, module, "Dummy", &Dummy_spec, param)) + return; +// END: add_Dummy +} + +// BEGIN: LanguageObject +typedef struct { + char *language; +} LanguageObject; +HPyType_HELPERS(LanguageObject, HPyType_BuiltinShape_Unicode) +// END: LanguageObject + +HPyDef_GETSET(Language_lang, "lang") +static HPy Language_lang_get(HPyContext *ctx, HPy self, void *closure) +{ + LanguageObject *data = LanguageObject_AsStruct(ctx, self); + return HPyUnicode_FromString(ctx, data->language); +} +static int Language_lang_set(HPyContext *ctx, HPy self, HPy value, void *closure) +{ + LanguageObject *data = LanguageObject_AsStruct(ctx, self); + HPy_ssize_t size; + const char *s = HPyUnicode_AsUTF8AndSize(ctx, value, &size); + if (s == NULL) + return -1; + data->language = (char *)calloc(size+1, sizeof(char)); + strncpy(data->language, s, size); + return 0; +} + +HPyDef_SLOT(Language_destroy, HPy_tp_destroy) +static void Language_destroy_impl(void *data) +{ + LanguageObject *ldata = (LanguageObject *)data; + if (ldata->language) + free(ldata->language); +} + +HPyDef *Language_defines[] = { + &Language_lang, + &Language_destroy, + NULL +}; + + +// BEGIN: spec_Language +static HPyType_Spec Language_spec = { + .name = "builtin_type.Language", + .basicsize = sizeof(LanguageObject), + .builtin_shape = SHAPE(LanguageObject), + .defines = Language_defines +}; +// END: spec_Language + +static void make_Language(HPyContext *ctx, HPy module) +{ +// BEGIN: add_Language + HPyType_SpecParam param[] = { + { HPyType_SpecParam_Base, ctx->h_UnicodeType }, + { (HPyType_SpecParam_Kind)0 } + }; + if (!HPyHelpers_AddType(ctx, module, "Language", &Language_spec, param)) + return; +// END: add_Language +} + +HPyDef_SLOT(simple_exec, HPy_mod_exec) +static int simple_exec_impl(HPyContext *ctx, HPy m) { + make_Dummy(ctx, m); + if (HPyErr_Occurred(ctx)) + return -1; + + make_Language(ctx, m); + if (HPyErr_Occurred(ctx)) + return -1; + + return 0; // success +} + +static HPyDef *mod_defines[] = { + &simple_exec, // 'simple_exec' is generated by the HPyDef_SLOT macro + NULL, +}; + +static HPyModuleDef moduledef = { + .defines = mod_defines +}; + +HPy_MODINIT(builtin_type, moduledef) diff --git a/graalpython/hpy/docs/examples/hpytype-example/setup.py b/graalpython/hpy/docs/examples/hpytype-example/setup.py new file mode 100644 index 0000000000..f740a424e1 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/setup.py @@ -0,0 +1,11 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-type-example", + hpy_ext_modules=[ + Extension('simple_type', sources=['simple_type.c']), + Extension('builtin_type', sources=['builtin_type.c']), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/hpytype-example/simple_type.c b/graalpython/hpy/docs/examples/hpytype-example/simple_type.c new file mode 100644 index 0000000000..40b1fba715 --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/simple_type.c @@ -0,0 +1,102 @@ +#include + +// BEGIN: PointObject +typedef struct { + long x; + long y; +} PointObject; +HPyType_HELPERS(PointObject) +// END: PointObject + +// BEGIN: members +HPyDef_MEMBER(Point_x, "x", HPyMember_LONG, offsetof(PointObject, x)) +HPyDef_MEMBER(Point_y, "y", HPyMember_LONG, offsetof(PointObject, y)) +// END: members + +// BEGIN: methods +HPyDef_METH(Point_foo, "foo", HPyFunc_NOARGS) +static HPy Point_foo_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, point->x * 10 + point->y); +} +// END: methods + +// BEGIN: getset +HPyDef_GETSET(Point_z, "z", .closure=(void *)1000) +static HPy Point_z_get(HPyContext *ctx, HPy self, void *closure) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, point->x*10 + point->y + (long)(HPy_ssize_t)closure); +} + +static int Point_z_set(HPyContext *ctx, HPy self, HPy value, void *closure) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + long current = point->x*10 + point->y + (long)(HPy_ssize_t)closure; + long target = HPyLong_AsLong(ctx, value); // assume no exception + point->y += target - current; + return 0; +} +// END: getset + +// BEGIN: slots +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl(HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + long x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} +// END: slots + +// BEGIN: defines +static HPyDef *Point_defines[] = { + &Point_x, + &Point_y, + &Point_z, + &Point_new, + &Point_foo, + NULL +}; +// END: defines + +// BEGIN: spec +static HPyType_Spec Point_spec = { + .name = "simple_type.Point", + .basicsize = sizeof(PointObject), + .builtin_shape = PointObject_SHAPE, + .defines = Point_defines +}; +// END: spec + +// BEGIN: add_type +HPyDef_SLOT(simple_exec, HPy_mod_exec) +static int simple_exec_impl(HPyContext *ctx, HPy m) { + if (!HPyHelpers_AddType(ctx, m, "Point", &Point_spec, NULL)) { + return -1; + } + return 0; // success +} + +static HPyDef *mod_defines[] = { + &simple_exec, // 'simple_exec' is generated by the HPyDef_SLOT macro + NULL, +}; + +static HPyModuleDef moduledef = { + .defines = mod_defines, + // ... +// END: add_type + .doc = "A simple HPy type", +}; + +HPy_MODINIT(simple_type, moduledef) \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst b/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst new file mode 100644 index 0000000000..d611e5113c --- /dev/null +++ b/graalpython/hpy/docs/examples/hpytype-example/simple_type.rst @@ -0,0 +1,8 @@ +:orphan: + +simple_type.c +============= + +.. literalinclude:: ./simple_type.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/examples/mixed-example/mixed.c b/graalpython/hpy/docs/examples/mixed-example/mixed.c new file mode 100644 index 0000000000..d041f3f41c --- /dev/null +++ b/graalpython/hpy/docs/examples/mixed-example/mixed.c @@ -0,0 +1,49 @@ +/* Simple C module that shows how to mix CPython API and HPY. + * At the moment, this code is not referenced from the documentation, but it is + * tested nonetheless. + */ + +#include "hpy.h" + +/* a HPy style function */ +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + + +/* Add an old-style function */ +static PyObject * +add_ints2(PyObject *self, PyObject *args) +{ + + long a, b, ret; + if (!PyArg_ParseTuple(args, "ll", &a, &b)) + return NULL; + ret = a + b; + return PyLong_FromLong(ret); +} + +static HPyDef *hpy_defines[] = { + &add_ints, + NULL +}; + +static PyMethodDef py_defines[] = { + {"add_ints_legacy", add_ints2, METH_VARARGS, "add two ints"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static HPyModuleDef moduledef = { + .doc = "HPy Example of mixing CPython API and HPy API", + .size = 0, + .defines = hpy_defines, + .legacy_methods = py_defines +}; + + +HPy_MODINIT(mixed, moduledef) diff --git a/graalpython/hpy/docs/examples/mixed-example/setup.py b/graalpython/hpy/docs/examples/mixed-example/setup.py new file mode 100644 index 0000000000..75af734583 --- /dev/null +++ b/graalpython/hpy/docs/examples/mixed-example/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-mixed-example", + hpy_ext_modules=[ + Extension('mixed', sources=[path.join(path.dirname(__file__), 'mixed.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/quickstart/quickstart.c b/graalpython/hpy/docs/examples/quickstart/quickstart.c new file mode 100644 index 0000000000..ce9542046a --- /dev/null +++ b/graalpython/hpy/docs/examples/quickstart/quickstart.c @@ -0,0 +1,39 @@ +// quickstart.c + +// This header file is the entrypoint to the HPy API: +#include "hpy.h" + +// HPy method: the HPyDef_METH macro generates some boilerplate code, +// the same code can be also written manually if desired +HPyDef_METH(say_hello, "say_hello", HPyFunc_NOARGS) +static HPy say_hello_impl(HPyContext *ctx, HPy self) +{ + // Methods take HPyContext, which must be passed as the first argument to + // all HPy API functions. Other than that HPyUnicode_FromString does the + // same thing as PyUnicode_FromString. + // + // HPy type represents a "handle" to a Python object, but may not be + // a pointer to the object itself. It should be fully "opaque" to the + // users. Try uncommenting the following two lines to see the difference + // from PyObject*: + // + // if (self == self) + // HPyUnicode_FromString(ctx, "Surprise? Try HPy_Is(ctx, self, self)"); + + return HPyUnicode_FromString(ctx, "Hello world"); +} + +static HPyDef *QuickstartMethods[] = { + &say_hello, // 'say_hello' generated for us by the HPyDef_METH macro + NULL, +}; + +static HPyModuleDef quickstart_def = { + .doc = "HPy Quickstart Example", + .defines = QuickstartMethods, +}; + +// The Python interpreter will create the module for us from the +// HPyModuleDef specification. Additional initialization can be +// done in the HPy_mod_exec slot +HPy_MODINIT(quickstart, quickstart_def) diff --git a/graalpython/hpy/docs/examples/quickstart/setup.py b/graalpython/hpy/docs/examples/quickstart/setup.py new file mode 100644 index 0000000000..0d56aefec4 --- /dev/null +++ b/graalpython/hpy/docs/examples/quickstart/setup.py @@ -0,0 +1,13 @@ +# setup.py + +from setuptools import setup, Extension +from os import path + +DIR = path.dirname(__file__) +setup( + name="hpy-quickstart", + hpy_ext_modules=[ + Extension('quickstart', sources=[path.join(DIR, 'quickstart.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/simple-example/setup.py b/graalpython/hpy/docs/examples/simple-example/setup.py new file mode 100644 index 0000000000..af81861942 --- /dev/null +++ b/graalpython/hpy/docs/examples/simple-example/setup.py @@ -0,0 +1,10 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-simple-example", + hpy_ext_modules=[ + Extension('simple', sources=[path.join(path.dirname(__file__), 'simple.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/simple-example/simple.c b/graalpython/hpy/docs/examples/simple-example/simple.c new file mode 100644 index 0000000000..49883a950a --- /dev/null +++ b/graalpython/hpy/docs/examples/simple-example/simple.c @@ -0,0 +1,43 @@ +/* Simple C module that defines single simple function "myabs". + * We need to have a separate standalone package for those snippets, because we + * want to show the source code in its entirety, including the HPyDef array + * initialization, the module definition, and the setup.py script, so there is + * no room left for mixing these code snippets with other code snippets. + */ + +// BEGIN: myabs +#include "hpy.h" + +HPyDef_METH(myabs, "myabs", HPyFunc_O) +static HPy myabs_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Absolute(ctx, arg); +} +// END: myabs + +// BEGIN: double +HPyDef_METH_IMPL(double_num, "double", double_impl, HPyFunc_O) +static HPy double_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Add(ctx, arg, arg); +} +// END: double + +// BEGIN: methodsdef +static HPyDef *SimpleMethods[] = { + &myabs, + &double_num, + NULL, +}; + +static HPyModuleDef simple = { + .doc = "HPy Example", + .size = 0, + .defines = SimpleMethods, + .legacy_methods = NULL +}; +// END: methodsdef + +// BEGIN: moduledef +HPy_MODINIT(simple, simple) +// END: moduledef \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/hpycall.c b/graalpython/hpy/docs/examples/snippets/hpycall.c new file mode 100644 index 0000000000..9fc51e1f3e --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpycall.c @@ -0,0 +1,196 @@ +#include + +// BEGIN EuclideanVectorObject +typedef struct { + long x; + long y; +} EuclideanVectorObject; +HPyType_HELPERS(EuclideanVectorObject) +// END EuclideanVectorObject + +// BEGIN HPy_tp_call +HPyDef_SLOT(call, HPy_tp_call) +static HPy +call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + static const char *keywords[] = { "x1", "y1", NULL }; + long x1, y1; + HPyTracker ht; + if (!HPyArg_ParseKeywords(ctx, &ht, args, nargs, kwnames, "ll", keywords, + &x1, &y1)) { + return HPy_NULL; + } + EuclideanVectorObject *data = EuclideanVectorObject_AsStruct(ctx, self); + return HPyLong_FromLong(ctx, data->x * x1 + data->y * y1); +} +// END HPy_tp_call + +// BEGIN HPy_SetCallFunction +HPyDef_CALL_FUNCTION(special_call) +static HPy +special_call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + HPy tmp = call_impl(ctx, self, args, nargs, kwnames); + HPy res = HPy_Negative(ctx, tmp); + HPy_Close(ctx, tmp); + return res; +} + +HPyDef_SLOT(new, HPy_tp_new) +static HPy +new_impl(HPyContext *ctx, HPy cls, const HPy *args, HPy_ssize_t nargs, HPy kw) +{ + static const char *keywords[] = { "x", "y", "use_special_call", NULL }; + HPyTracker ht; + long x, y; + HPy use_special_call = ctx->h_False; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "ll|O", keywords, + &x, &y, &use_special_call)) { + return HPy_NULL; + } + EuclideanVectorObject *vector; + HPy h_point = HPy_New(ctx, cls, &vector); + if (HPy_IsNull(h_point)) { + HPyTracker_Close(ctx, ht); + return HPy_NULL; + } + if (HPy_IsTrue(ctx, use_special_call) && + HPy_SetCallFunction(ctx, h_point, &special_call) < 0) { + HPyTracker_Close(ctx, ht); + HPy_Close(ctx, h_point); + return HPy_NULL; + } + HPyTracker_Close(ctx, ht); + vector->x = x; + vector->y = y; + return h_point; +} +// END HPy_SetCallFunction + +// BEGIN FooObject +typedef struct { + void *a; + HPyCallFunction call_func; + void *b; +} FooObject; +HPyType_HELPERS(FooObject) +// END FooObject + +// BEGIN vectorcalloffset +HPyDef_MEMBER(Foo_call_func_offset, "__vectorcalloffset__", HPyMember_HPYSSIZET, + offsetof(FooObject, call_func), .readonly=1) + +HPyDef_CALL_FUNCTION(Foo_call_func) +static HPy +Foo_call_func_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + return HPyUnicode_FromString(ctx, + "hello manually initialized call function"); +} + +HPyDef_SLOT(Foo_new, HPy_tp_new) +static HPy Foo_new_impl(HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + FooObject *data; + HPy h_obj = HPy_New(ctx, cls, &data); + if (HPy_IsNull(h_obj)) + return HPy_NULL; + data->call_func = Foo_call_func; + return h_obj; +} +// END vectorcalloffset + +// BEGIN pack_args +// function using legacy 'tp_call' calling convention +static HPy +Pack_call_legacy(HPyContext *ctx, HPy self, HPy args, HPy kwd) +{ + // use 'args' and 'kwd' + return HPy_Dup(ctx, ctx->h_None); +} + +// function using HPy calling convention +HPyDef_SLOT(Pack_call, HPy_tp_call) +static HPy +Pack_call_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs, + HPy kwnames) +{ + HPy args_tuple, kwd; + HPy result; + if (!HPyHelpers_PackArgsAndKeywords(ctx, args, nargs, kwnames, + &args_tuple, &kwd)) { + return HPy_NULL; + } + result = Pack_call_legacy(ctx, self, args_tuple, kwd); + HPy_Close(ctx, args_tuple); + HPy_Close(ctx, kwd); + return result; +} +// END pack_args + +static HPyDef *Point_defines[] = { + &call, + &new, + NULL +}; +static HPyType_Spec EuclideanVector_spec = { + .name = "hpycall.EuclideanVector", + .basicsize = sizeof(EuclideanVectorObject), + .builtin_shape = SHAPE(EuclideanVectorObject), + .defines = Point_defines +}; + +static HPyDef *Foo_defines[] = { + &Foo_call_func_offset, + &Foo_new, + NULL +}; +static HPyType_Spec Foo_spec = { + .name = "hpycall.Foo", + .basicsize = sizeof(FooObject), + .builtin_shape = SHAPE(FooObject), + .defines = Foo_defines +}; + +static HPyDef *Pack_defines[] = { + &Pack_call, + NULL +}; +static HPyType_Spec Pack_spec = { + .name = "hpycall.Pack", + .defines = Pack_defines +}; + +HPyDef_SLOT(init, HPy_mod_exec) +static int init_impl(HPyContext *ctx, HPy m) +{ + if (!HPyHelpers_AddType(ctx, m, "EuclideanVector", &EuclideanVector_spec, NULL)) { + return -1; + } + if (!HPyHelpers_AddType(ctx, m, "Foo", &Foo_spec, NULL)) { + return -1; + } + if (!HPyHelpers_AddType(ctx, m, "Pack", &Pack_spec, NULL)) { + return -1; + } + return 0; +} + +static HPyDef *moduledefs[] = { + &init, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy call protocol usage example", + .size = 0, + .legacy_methods = NULL, + .defines = moduledefs, + +}; + +HPy_MODINIT(hpycall, moduledef) diff --git a/graalpython/hpy/docs/examples/snippets/hpyinit.c b/graalpython/hpy/docs/examples/snippets/hpyinit.c new file mode 100644 index 0000000000..f23bba29eb --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpyinit.c @@ -0,0 +1,22 @@ +#include "hpy.h" + +// BEGIN +HPyDef_SLOT(my_exec, HPy_mod_exec) +int my_exec_impl(HPyContext *ctx, HPy mod) { + + // Some initialization: add types, constants, ... + + return 0; // success +} + +static HPyDef *Methods[] = { + &my_exec, // HPyDef_SLOT macro generated `my_exec` for us + NULL, +}; + +static HPyModuleDef mod_def = { + .defines = Methods +}; + +HPy_MODINIT(hpyinit, mod_def) +// END \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/hpyvarargs.c b/graalpython/hpy/docs/examples/snippets/hpyvarargs.c new file mode 100644 index 0000000000..a95617d542 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/hpyvarargs.c @@ -0,0 +1,45 @@ +/* Simple C module that defines simple functions "myabs" and "add_ints". + * + * This module represents an incremental change over the "simple" package + * and shows how to add a method with VARARGS calling convention. + * + * We need to have a separate standalone C module for those snippets, because we + * want to show the source code including the HPyDef array initialization, so + * there is no room left for adding other entry points for other code snippets. + */ + +#include "hpy.h" + +// This is here to make the module look like an incremental change to simple-example +HPyDef_METH(myabs, "myabs", HPyFunc_O) +static HPy myabs_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Absolute(ctx, arg); +} + +// BEGIN: add_ints +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} +// END: add_ints + +// BEGIN: methodsdef +static HPyDef *SimpleMethods[] = { + &myabs, + &add_ints, + NULL, +}; +// END: methodsdef + +static HPyModuleDef def = { + .doc = "HPy Example of varargs calling convention", + .size = 0, + .defines = SimpleMethods +}; + +HPy_MODINIT(hpyvarargs, def) diff --git a/graalpython/hpy/docs/examples/snippets/legacyinit.c b/graalpython/hpy/docs/examples/snippets/legacyinit.c new file mode 100644 index 0000000000..f10381b617 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/legacyinit.c @@ -0,0 +1,20 @@ +#include + +// BEGIN +static struct PyModuleDef mod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "legacyinit", + .m_size = -1 +}; + +PyMODINIT_FUNC +PyInit_legacyinit(void) +{ + PyObject *mod = PyModule_Create(&mod_def); + if (mod == NULL) return NULL; + + // Some initialization: add types, constants, ... + + return mod; +} +// END \ No newline at end of file diff --git a/graalpython/hpy/docs/examples/snippets/setup.py b/graalpython/hpy/docs/examples/snippets/setup.py new file mode 100644 index 0000000000..6c097b0385 --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, Extension +from os import path + +setup( + name="hpy-snippets", + hpy_ext_modules=[ + Extension('hpyvarargs', sources=[path.join(path.dirname(__file__), 'hpyvarargs.c')]), + Extension('snippets', sources=[path.join(path.dirname(__file__), 'snippets.c')]), + Extension('hpyinit', sources=[path.join(path.dirname(__file__), 'hpyinit.c')]), + Extension('hpycall', sources=[path.join(path.dirname(__file__), 'hpycall.c')]), + ], + ext_modules=[ + Extension('legacyinit', sources=[path.join(path.dirname(__file__), 'legacyinit.c')]), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/examples/snippets/snippets.c b/graalpython/hpy/docs/examples/snippets/snippets.c new file mode 100644 index 0000000000..7e60a52cab --- /dev/null +++ b/graalpython/hpy/docs/examples/snippets/snippets.c @@ -0,0 +1,81 @@ +/* Module with various code snippets used in the docs. + * All code snippets should be put into this file if possible. Notable + * exception are code snippets showing definition of the module or the + * HPyDef array initialization. Remember to also add tests to ../tests.py + */ +#include "hpy.h" + +// ------------------------------------ +// Snippets used in api.rst + +// BEGIN: foo +void foo(HPyContext *ctx) +{ + HPy x = HPyLong_FromLong(ctx, 42); + HPy y = HPy_Dup(ctx, x); + /* ... */ + // we need to close x and y independently + HPy_Close(ctx, x); + HPy_Close(ctx, y); +} +// END: foo + +// BEGIN: is_same_object +int is_same_object(HPyContext *ctx, HPy x, HPy y) +{ + // return x == y; // compilation error! + return HPy_Is(ctx, x, y); +} +// END: is_same_object + +// dummy entry point so that we can test the snippets: +HPyDef_METH(test_foo_and_is_same_object, "test_foo_and_is_same_object", HPyFunc_VARARGS) +static HPy test_foo_and_is_same_object_impl(HPyContext *ctx, HPy self, + const HPy *args, size_t nargs) +{ + foo(ctx); // not much we can test here + return HPyLong_FromLong(ctx, is_same_object(ctx, args[0], args[1])); +} + +// BEGIN: test_leak_stacktrace +HPyDef_METH(test_leak_stacktrace, "test_leak_stacktrace", HPyFunc_NOARGS) +static HPy test_leak_stacktrace_impl(HPyContext *ctx, HPy self) +{ + HPy num = HPyLong_FromLong(ctx, 42); + if (HPy_IsNull(num)) { + return HPy_NULL; + } + // No HPy_Close(ctx, num); + return HPy_Dup(ctx, ctx->h_None); +} +// END: test_leak_stacktrace + +// BEGIN: add +HPyDef_METH(add, "add", HPyFunc_VARARGS) +static HPy add_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + if (nargs != 2) { + HPyErr_SetString(ctx, ctx->h_TypeError, "expected exactly two args"); + return HPy_NULL; + } + return HPy_Add(ctx, args[0], args[1]); +} +// END: add + +// ------------------------------------ +// Dummy module definition, so that we can test the snippets + +static HPyDef *Methods[] = { + &test_foo_and_is_same_object, + &test_leak_stacktrace, + &add, + NULL, +}; + +static HPyModuleDef snippets = { + .doc = "Various HPy code snippets for the docs", + .size = 0, + .defines = Methods +}; + +HPy_MODINIT(snippets, snippets) diff --git a/graalpython/hpy/docs/examples/tests.py b/graalpython/hpy/docs/examples/tests.py new file mode 100644 index 0000000000..8a8ee5f5df --- /dev/null +++ b/graalpython/hpy/docs/examples/tests.py @@ -0,0 +1,124 @@ +import os +import os.path +import re +import subprocess +import sys + +import simple +import mixed +import hpyvarargs +import snippets +import simple_type +import builtin_type +import hpycall + + +def test_simple_abs(): + assert simple.myabs(-42) == 42 + assert simple.myabs(42) == 42 + + +def test_hpyvarargs(): + assert hpyvarargs.add_ints(40, 2) == 42 + + +def test_mixed_add_ints(): + assert mixed.add_ints_legacy(40, 2) == 42 + assert mixed.add_ints(40, 2) == 42 + + +def test_snippets(): + x = 2 + assert snippets.test_foo_and_is_same_object(x, x) == 1 + assert snippets.test_foo_and_is_same_object(x, 42) == 0 + + +def test_simple_type(): + p = simple_type.Point(4, 5) + assert p.x == 4 + assert p.y == 5 + assert p.foo() == 45 + assert p.z == 1045 + p.z = 2000 + assert p.y == 960 + assert p.z == 2000 + + +def test_quickstart(): + import quickstart + assert quickstart.say_hello() == "Hello world" +# END: test_quickstart + + +def test_builtin_type(): + obj = builtin_type.Dummy("hello") + assert obj == "hello" + + obj = builtin_type.Language("hello") + obj.lang = "en" + assert obj == "hello" + assert obj.lang == "en" + + +def test_leak_detector(): + from hpy.debug.pytest import LeakDetector + with LeakDetector() as ld: + # add_ints is an HPy C function. If it forgets to close a handle, + # LeakDetector will complain + assert mixed.add_ints(40, 2) == 42 +# END: test_leak_detector + +from hpy.debug.pytest import hpy_debug +def test_that_uses_leak_detector_fixture(hpy_debug): + # Run some HPy extension code + assert mixed.add_ints(40, 2) == 42 + + +def test_leak_detector_with_traces(): + import hpy.debug + hpy.debug.set_handle_stack_trace_limit(16) + assert mixed.add_ints(40, 2) == 42 + hpy.debug.disable_handle_stack_traces() + + +def test_leak_detector_with_traces_output(): + # Update the debug documentation if anything here changes! + env = os.environ.copy() + env['HPY'] = 'debug' + env['HPY_LOG'] = '1' + script = os.path.join(os.path.dirname(__file__), 'debug-example.py') + result = subprocess.run([sys.executable, script], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Rudimentary check that the output contains what we have in the documentation + out = result.stdout.decode('latin-1') + assert out == "Loading 'snippets' in HPy universal mode with a debug context" + os.linesep + err = result.stderr.decode('latin-1') + assert 'hpy.debug.leakdetector.HPyLeakError: 1 unclosed handle:' in err + assert re.search('', err) + assert 'Allocation stacktrace:' in err + if sys.platform.startswith(("linux", "darwin")): + assert 'snippets.hpy0.so' in err # Should be somewhere in the stack trace + else: + assert 'At the moment this is only supported on Linux with glibc' in err + +def test_trace_mode_output(): + # Update the trace mode documentation if anything here changes! + env = os.environ.copy() + env['HPY'] = 'trace' + env['HPY_LOG'] = '1' + script = os.path.join(os.path.dirname(__file__), 'trace-example.py') + result = subprocess.run([sys.executable, script], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Rudimentary check that the output contains what we have in the documentation + out = result.stdout.decode('latin-1') + assert 'get_call_counts()["ctx_Add"] == 1' in out + +def test_call_dot_product(): + vec = hpycall.EuclideanVector(4, 5) + assert vec(6, 7) == 4 * 6 + 5 * 7 + vec = hpycall.EuclideanVector(4, 5, use_special_call=True) + assert vec(6, 7) == -(4 * 6 + 5 * 7) + foo = hpycall.Foo() + assert foo() == 'hello manually initialized call function' + pack = hpycall.Pack() + assert pack() is None diff --git a/graalpython/hpy/docs/examples/trace-example.py b/graalpython/hpy/docs/examples/trace-example.py new file mode 100644 index 0000000000..2a4d92838e --- /dev/null +++ b/graalpython/hpy/docs/examples/trace-example.py @@ -0,0 +1,9 @@ +# Run with HPY=trace +from hpy.trace import get_call_counts +import snippets + +add_count_0 = get_call_counts()["ctx_Add"] +snippets.add(1, 2) == 3 +add_count_1 = get_call_counts()["ctx_Add"] + +print('get_call_counts()["ctx_Add"] == %d' % (add_count_1 - add_count_0)) diff --git a/graalpython/hpy/docs/index.rst b/graalpython/hpy/docs/index.rst new file mode 100644 index 0000000000..9ca101b753 --- /dev/null +++ b/graalpython/hpy/docs/index.rst @@ -0,0 +1,80 @@ +.. HPy documentation master file, created by + sphinx-quickstart on Thu Apr 2 23:01:08 2020. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +HPy: a better API for Python +=============================== + +HPy provides a new API for extending Python in C. + +There are several advantages to writing C extensions in HPy: + + - **Speed**: it runs much faster on PyPy, GraalPy, and at native speed on CPython + + - **Deployment**: it is possible to compile a single binary which runs unmodified on all + supported Python implementations and versions -- think "stable ABI" on steroids + + - **Simplicity**: it is simpler and more manageable than the ``Python.h`` API, both for + the users and the Pythons implementing it + + - **Debugging**: it provides an improved debugging experience. Debug mode can be turned + on at runtime without the need to recompile the extension or the Python running it. + HPy design is more suitable for automated checks. + +The official `Python/C API `_, +also informally known as ``#include ``, is +specific to the current implementation of CPython: it exposes a lot of +internal details which makes it hard to: + + - implement it for other Python implementations (e.g. PyPy, GraalPy, + Jython, ...) + + - experiment with new approaches inside CPython itself, for example: + + - use a tracing garbage collection instead of reference counting + - remove the global interpreter lock (GIL) to take full advantage of multicore architectures + - use tagged pointers to reduce memory footprint + +Where to go next: +----------------- + + - Show me the code: + + - :doc:`Quickstart` + - :ref:`Simple documented HPy extension example` + - :doc:`Tutorial: porting Python/C API extension to HPy` + + - Details: + + - :doc:`HPy overview: motivation, goals, current status` + - :doc:`HPy API concepts introduction` + - :doc:`Python/C API to HPy Porting guide` + - :doc:`HPy API reference` + + +Full table of contents: +----------------------- + +.. toctree:: + :maxdepth: 2 + + quickstart + overview + api + porting-guide + porting-example/index + debug-mode + trace-mode + api-reference/index + contributing/index + misc/index + changelog + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/graalpython/hpy/docs/leysin-2020-design-decisions.md b/graalpython/hpy/docs/leysin-2020-design-decisions.md new file mode 100644 index 0000000000..b50bb081c5 --- /dev/null +++ b/graalpython/hpy/docs/leysin-2020-design-decisions.md @@ -0,0 +1,295 @@ +Leysin Sprint 2020 Design Decisions +=================================== + +Closing and duping HPy_NULL +--------------------------- + +Question: Should `HPy_Close` silently ignore attempts to close `HPy_NULL`? + +Decisions: + +* `HPy_Close` should function like `Py_XDECREF` and silently not close `HPy_NULL`. +* `HPy_Dup` should silently not dup `HPy_NULL` for symmetry with `HPy_Close`, + i.e. `HPy_Dup(ctx, HPy_NULL)` should return `HPy_NULL`. +* Both the API and ABI versions should have this behaviour. +* We should add tests for this behaviour. + + +Attribute and item access +------------------------- + +Question: Should we have separate HPySequence_GetItem and HPyMapping_GetItem +or just one HPy_GetItem? Should we explore the idea of protocols for such +access. + +Decisions: + +* We should start with `HPy_GetItem` and `HPy_SetItem` and `HPy_GetItem_i` and + `HPy_SetItem_i` for item access and `HPy_GetAttr`, `HPy_SetAttr`, + `HPy_GetAttr_s` and `HPy_SetAttr_s` for attribute access.. +* We could add an `hpy/compat.h` that supports accessing the old Python mapping + and sequence get item slots. For example, `HPyDict_GetItem_NotBorrowed`. + `PyDict_GetItem` returns a borrower reference. `HPyDict_GetItem_NotBorrowed` + should return a new handle that has to be closed. +* HPy API functions MUST NOT return a borrowed reference and we should add this + to our official documentation. +* We should add an `HPyBuffer` protocol later (with the design still to be + decided). + +Proposed function signatures: + +```C +HPy HPy_GetItem(HPyContext ctx, HPy obj, HPy idx); +HPy HPy_GetItem_i(HPyContext ctx, HPy obj, HPy_ssize_t idx); +HPy HPy_GetItem_s(HPyContext ctx, HPy obj, const char * idx); // UTF8 bytes + +int HPy_SetItem(HPyContext ctx, HPy obj, HPy idx, HPy value); +int HPy_SetItem_i(HPyContext ctx, HPy obj, HPy_ssize_t idx, HPy value); +int HPy_SetItem_s(HPyContext ctx, HPy obj, const char * idx, HPy value); // UTF8 bytes + +HPy HPy_GetAttr(HPyContext ctx, HPy obj, HPy idx); +HPy HPy_GetAttr_s(HPyContext ctx, HPy obj, const char * idx); // UTF8 bytes + +int HPy_SetAttr(HPyContext ctx, HPy obj, HPy idx, HPy value); +int HPy_SetAttr_s(HPyContext ctx, HPy obj, const char * idx, HPy value); // UTF8 bytes +``` + +Macro for returning None +------------------------ + +Question: Should we have an HPy_RETURN_NONE macro? + +Decisions: + +* Yes we should, but it should be `HPy_RETURN_NONE(ctx)`. +* We should also change `HPy_IsNull(x)` to `HPy_IsNull(ctx, x)`. + +Proposed macros: + +```C +#define HPy_RETURN_NONE(ctx) return HPy_Dup(ctx, ctx->h_None); +#define HPy_IsNull(ctx, x) ...; +``` + +Passing handles as void pointers +-------------------------------- + +Question: Should we add `HPy_AsVoidP` and `HPy_FromVoidP` to the API? Should they +be on the ctx or not? + +Decisions: + +* Yes, they should be part of the API. +* They should take the ctx as an argument in case a future implementation needs + it (much like `HPy_IsNull`). + +Proposed functions: + +```C +// universal +static inline HPy HPy_FromVoidP(HPyContext ctx, void *p) { return (HPy){(HPy_ssize_t)p}; } +static inline void* HPy_AsVoidP(HPyContext ctx, HPy h) { return (void*)h._i; } + +// cpython (the -/+4 is to avoid people casting it to PyObject) +static inline HPy HPy_FromVoidP(HPyContext ctx, void *p) { return (HPy){(HPy_ssize_t)p - 4}; } +static inline void* HPy_AsVoidP(HPyContext ctx, HPy h) { return (void*)h._o + 4; } +``` + +/* + * Should we implement HPy_Dump? + * + * Question: What should it print and where should it print it? + * + * Decision: It's useful for debugging if all macros are also available as functions + * definitions. + * + * Decision: It should dump to stderr just like PyObject_Dump. + */ + +/* + * How do we silence warnings from using HPy_METH_KEYWORDS? + * + * Decision: Write a cast to HPyMeth. + */ + +{"add_ints_kw", (HPyMeth) add_ints_kw, HPy_METH_KEYWORDS, ""} + +/* + * How should HPyErr_Format be implemented? Should we avoid va_args? + * + * Decision: + */ + +HPyErr_Format(ctx, const char *fmt, ...) { + const char *msg = HPyStr_Format(ctx, fmt, ...); + HPyErr_SetString(msg); +} + +ctx->ctx_HPyErr_Format(???) + +/* + * Should make specifying values for optional arguments & dup / closing them + * less messy? + * + * Decision: Right now, No. In the future, someone should invent + * ARGUMENT_CLINIC for HPy. + */ + +/* + * Should we rename "struct _object*" to "PyObject*" and "_HPy_PyCFunction" + * to "PyCFunction" in universal/hpy.h? + * + * Decision: No. This would generate a warning if one imports Python.h. + * + * Decision: Comment in the code that this _object* is PyObject* and why we + * cannot call it that. + * + * Decision: In user documentation, just call ing PyObject *. + */ + +/* + * How should the API pass around exceptions? + * + * Decision: Follow CPython for each API call. + * + */ + +// E.g. API call returns HPy: + +HPy h = HPyLong_FromLong(ctx, 5); +if (HPy_IsNull(ctx, h)) { + // handle error + // python error has been set by the API call +} + +// or + +HPy h = HPyLong_FromLong(ctx, 5); +if (HPyErr_Occurred(ctx)) { + // handle error + // python error has been set by the API call +} + +// E.g. API call returns a value that is not an HPy: + +long l = HPyLong_AsLong(ctx, h); +if (l == -1 && HPyErr_Occurred(ctx)) { + // handle error + // python error has been set by the API call +} + +// E.g. API call returns a success or error flag: + +// int error = HPyArg_Parse(...); +if (!HPyArg_Parse(...)) { + // handle error + // python error has been set by the API call +} + +/* + * How should support for creating custom Python types look? + * + * Decisions: + * + * + */ + +// When using C-API: + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ + PyObject *x; + PyObject *y; + double *data; + int size; +} PointObject; + +// When using HPy: + +typedef struct { + HPyObject_HEAD + double x; + double y; +} HPy_Point; + +typedef struct { + HPyObject_HEAD + HPyField a; + HPyField b; +} HPy_Rectangle; + +/* Possible HPy code */ + +typedef struct { + ??? ob_type; +} _HPy_ObjectHeader; + +#define HPyObject_HEAD _HPy_ObjectHeader head; + +#define HPy_STORE(ctx, obj, field, value) ((ctx)->ctx_HPy_StoreInto((_HPy_ObjectHeader *) obj, &((obj)->field), value)) +#define HPy_STORE_INTO(ctx, obj, pointer, value) ((ctx)->ctx_HPy_StoreInto((_HPy_ObjectHeader *) obj, pointer, value)) + +// Using the debug mode ctx, this should check that obj->ob_type->tp_traverse is not NULL. +void HPy_StoreInto(HPyContext ctx, _HPy_ObjectHeader *obj, HPyField *pointer, HPy value) + +// HPy_New: +// +// * Should return a handle. +// * It should be possible to go from the handle to struct, but not +// from struct to handle. E.g. HPy_CAST(ctx, HPyRectangle, h) -> HPyRectangle, +// but no inverse. +// * We should not have an HPy_Init at the moment -- HPy_New both allocates +// the object and initializes it. We will add separate allocation and init +// later if we encounter a need for it. +// * HPyTypeSpec should follow the CPython type spec. + +#define HPy_CAST(ctx, return_type, h) (return_type *) ctx->ctx_HPy_Cast(h) + +void* HPy_Cast(ctx, HPy h); + +HPy HPy_TypeFromSpec(ctx, HPyTypeSpec type_spec); +HPy HPy_New(ctx, HPy h_type); + +/* end of possible HPy code */ + +HPy new_rect(HPy p1, HPy p2) { + // HPy_New always initialize the whole object to 0. We can also have + // HPy_NewUninitialized if we don't want to pay the penalty + HPy_Rectangle *rect; + HPy rect_handle = HPy_New(ctx, HPy_Rectangle, h_rectangle_type, &rect); + + // HPy_Store does a write barrier on PyPy, and DECREF the old rect->a on + // CPython if needed + HPy_Store(ctx, rect, a, p1); + HPy_Store(ctx, rect, b, p2); + return rect_handle; +} + +double calc_diagonal(HPy rect_handle) { + // rect is valid until rect_handle is closed. on PyPy we pin the object, and + // we unpin it when we close the handle + HPy_Rectangle *rect = HPy_Cast(HPy_Rectangle, rect_handle); + + // HPy_Load reads a field and turn it into a handle + HPy p1_handle = HPy_Load(rect->a); // p1 is a handle which must be closed + HPy_Point *p1 = HPy_Cast(HPy_Point, p1_handle); + + // for C99 compilers, we can also provide a macro which declares p2 and + // p2_handle automatically and does the equivalent of the two lines above + HPY_LOAD(HPy_Point, p2, rect->b); + + double diag = sqrt(p1->x - p2->x /* etc. etc. */); + + // close all the handles + HPy_Close(p1_handle); + HPy_Close(p2_handle); + + return diag; +} + +/* + * Should HPy support Python without the GIL? + * + * Problem left as an exercise for the reader. + */ diff --git a/graalpython/hpy/docs/misc/embedding.rst b/graalpython/hpy/docs/misc/embedding.rst new file mode 100644 index 0000000000..cb85d2bc10 --- /dev/null +++ b/graalpython/hpy/docs/misc/embedding.rst @@ -0,0 +1,46 @@ +Embedding HPy modules +===================== + +There might be cases where it is beneficial or even necessary to embed multiple +HPy modules into one library. HPy itself already makes use of that. The debug +and the trace module do not have individual libraries but are embedded into the +universal module. + +To achieve that, the embedder will use the macro :c:macro:`HPy_MODINIT` several times. +Unfortunately, this macro defines global state and cannot repeatedly be used by +default. In order to correctly embed several HPy modules into one library, the +embedder needs to consider following: + +* The modules must be compiled with preprocessor macro + :c:macro:`HPY_EMBEDDED_MODULES` defined to enable this feature. + +* There is one major restriction: All HPy-specific module pieces must be + in the same compilation unit. *HPy-specific pieces* are things like the + module's init function (``HPy_MODINIT``) and all slots, members, methods of + the module or any type of it (``HPyDef_*``). The implementation functions + (usually the ``*_impl`` functions) of the slots, members, methods, etc. and + any helper functions may still be in different compilation units. The reason + for this is that the global state induced by ``HPy_MODINIT`` is, of course, + made local (e.g. using C modifier ``static``). + +* It is also necessary to use macro :c:macro:`HPY_MOD_EMBEDDABLE` before the + first usage of any ``HPyDef_*`` macro. + +Also refer to the API reference :ref:`api-reference/hpy-type:hpy module`. + + +**Example** + +.. code-block:: c + + // compile with -DHPY_EMBEDDED_MODULES + + HPY_MOD_EMBEDDABLE(hpymodA) + + HPyDef_METH(foo, /* ... */) + static HPy foo_impl(/* ... */) + { + // ... + } + + HPy_MODINIT(extension_name, hpymodA) diff --git a/graalpython/hpy/docs/misc/index.rst b/graalpython/hpy/docs/misc/index.rst new file mode 100644 index 0000000000..f305c8ed01 --- /dev/null +++ b/graalpython/hpy/docs/misc/index.rst @@ -0,0 +1,7 @@ +Misc Notes +========== + +.. toctree:: + :maxdepth: 1 + + embedding diff --git a/graalpython/hpy/docs/module-state.txt b/graalpython/hpy/docs/module-state.txt new file mode 100644 index 0000000000..3682a8820e --- /dev/null +++ b/graalpython/hpy/docs/module-state.txt @@ -0,0 +1,44 @@ +How to replace global variables +------------------------------- + +In a given .c source, write: + + +typedef struct { + long x; + HPy y; +} my_globals_t; + +static void my_globals_traverse(traversefunc traverse, my_globals_t *g) +{ + traverse(g->y); +} + +HPyGlobalSpec my_globals = { + .m_size = sizeof(my_globals_t), + .m_traverse = my_globals_traverse +}; + + +There can be several HPyGlobalSpec structures around; in CPython it's done as +part of the PyModuleDef type, but there is no real reason for why it should +be tightly tied to a module. + + +To use: + + my_globals_t *g = HPy_GetState(ctx, &my_globals); + g->x++; + HPy_DoRandomStuffWithHandle(ctx, g->y); + + +Implementation: the type HPyGlobalSpec contains extra internal fields +which should give us a very fast cache: _last_ctx and _last_result, +and HPy_GetState() can be: + + if (ctx == globspec->_last_ctx) + return globspec->_last_result; + else + look up globspec in a dictionary attached to ctx, or vice-versa, + or maybe initialize globspec->_index with a unique incrementing + index and use that to index an array attached to ctx diff --git a/graalpython/hpy/docs/overview.rst b/graalpython/hpy/docs/overview.rst new file mode 100644 index 0000000000..e4c9e30b86 --- /dev/null +++ b/graalpython/hpy/docs/overview.rst @@ -0,0 +1,444 @@ +HPy Overview +============ + +Motivation and goals +--------------------- + +The superpower of the Python ecosystem is its libraries, which are developed by +users. Over time, these libraries have grown in number, quality, and +applicability. While it is possible to write python libraries entirely in +python, many of them, especially in the scientific community, are written in C +and exposed to Python using the `Python.h API +`_. The existence of these C +extensions using the ``Python.h`` API leads to some issues: + + 1. Usually, alternative implementation of the Python programming language + want to support C extensions. To do so, they must implement the same + ``Python.h`` API or provide a compatibility layer. + + 2. CPython developers cannot experiment with new designs or refactoring + without breaking compatibility with existing extensions. + +Over the years, it has become evident that emulating ``Python.h`` in an +efficient way is `challenging, if not impossible +`_. +To summarize, it is mainly due to leaking of implementation details of CPython +into the C/API - which makes it difficult to make different design choices than +those made by CPython. As such - the main goal of HPy is to provide a **C API +which makes as few assumptions as possible about the design decisions of any +implementation of Python, allowing diverse implementations to support it +efficiently and without compromise**. In particular, **reference counting is not +part of the API**: we want a more generic way of managing resources that is +possible to implement with different strategies, including the existing +reference counting and/or with a moving *Garbage Collector* (like the ones used +by PyPy, GraalPy or Java, for example). Moreover, each implementation can +experiment with new memory layout of objects, add optimizations, etc. The +following is a list of sub-goals. + + +Performance on CPython + HPy is usable on CPython from day 1 with no performance impact compared to + the existing ``Python.h`` API. + + +Incremental adoption + It is possible to port existing C extensions piece by piece and to use + the old and the new API side-by-side during the transition. + + +Easy migration + It should be easy to migrate existing C extensions to HPy. Thanks to an + appropriate and regular naming convention it should be obvious what the + HPy equivalent of any existing ``Python.h`` API is. When a perfect replacement + does not exist, the documentation explains what the alternative options are. + + +Better debugging + In debug mode, you get early and precise errors and warnings when you make + some specific kind of mistakes and/or violate the API rules and + assumptions. For example, you get an error if you try to use a handle + (see :ref:`api:handles`) which has already been closed. It is possible to + turn on the debug mode at startup time, *without needing to recompile*. + +Simplicity + The HPy API aims to be smaller and easier to study/use/manage than the + existing ``Python.h`` API. Sometimes there is a trade-off between this goal and + the others above, in particular *Performance on CPython* and *Easy migration*. + The general approach is to have an API which is "as simple as possible" while + not violating the other goals. + + +Universal binaries + It is possible to compile extensions to a single binary which is + ABI-compatible across multiple Python versions and/or multiple + implementation. See :ref:`hpy-target-abis`. + + +Opt-in low level data structures + Internal details might still be available, but in a opt-in way: for example, + if Cython wants to iterate over a list of integers, it can ask if the + implementation provides a direct low-level access to the content (e.g. in + the form of a ``int64_t[]`` array) and use that. But at the same time, be + ready to handle the generic fallback case. + + +API vs ABI +----------- + +HPy defines *both* an API and an ABI. Before digging further into details, +let's distinguish them: + + - The **API** works at the level of source code: it is the set of functions, + macros, types and structs which developers can use to write their own + extension modules. For C programs, the API is generally made available + through one or more header files (``*.h``). + + - The **ABI** works at the level of compiled code: it is the interface between + the host interpreter and the compiled DLL. Given a target CPU and + operating system it defines things like the set of exported symbols, the + precise memory layout of objects, the size of types, etc. + +In general it is possible to compile the same source into multiple compiled +libraries, each one targeting a different ABI. :pep:`3149` states that the +filename of the compiled extension should contain the *ABI tag* to specify +what the target ABI is. For example, if you compile an extension called +``simple.c`` on CPython 3.8, you get a DLL called +``simple.cpython-38-x86_64-linux-gnu.so``: + + - ``cpython-38`` is the ABI tag, in this case CPython 3.8 + + - ``x86_64`` is the CPU architecture + + - ``linux-gnu`` is the operating system + +The same source code compiled on PyPy3.6 7.2.0 results in a file called +``simple.pypy38-pp73-x86_64-linux-gnu.so``: + + - ``pypy38-pp73`` is the ABI tag, in this case "PyPy3.8", version "7.3.x" + +The HPy C API is exposed to the user by including ``hpy.h`` and it is +explained in its own section of the documentation. + + +Legacy and compatibility features +--------------------------------- + +To allow an incremental transition to HPy, it is possible to use both +``hpy.h`` and ``Python.h`` API calls in the same extension. Using *HPy legacy +features* you can: + + - mix ``Python.h`` and HPy method defs in the same HPy module + + - mix ``Python.h`` and HPy method defs and slots in the same HPy type + + - convert ``HPy`` handles to and from ``PyObject *`` using + ``HPy_AsPyObject()`` and ``HPy_FromPyObject()`` + + +Thanks to this, you can port your code to HPy one method and one type at a +time, while keeping the extension fully functional during the transition +period. See the :ref:`porting-guide:Porting guide` for a concrete example. + +Legacy features are available only if you target the CPython or HPy Hybrid +ABIs, as explained in the next section. + + +.. _hpy-target-abis: + +Target ABIs +----------- + +Depending on the compilation options, an HPy extension can target three +different ABIs: + +.. glossary:: + + CPython ABI + In this mode, HPy is implemented as a set of C macros and ``static inline`` + functions which translate the HPy API into the CPython API at compile + time. The result is a compiled extension which is indistinguishable from a + "normal" one and can be distributed using all the standard tools and will + run at the very same speed. + + *Legacy features* are available. + + The output filename is e.g. ``simple.cpython-38-x86_64-linux-gnu.so``. + + + HPy Universal ABI + As the name suggests, the HPy Universal ABI is designed to be loaded and + executed by a variety of different Python implementations. Compiled + extensions can be loaded unmodified on all the interpreters which support + it. PyPy and GraalPy support it natively. CPython supports it by using the + ``hpy.universal`` package, and there is a small speed penalty [#f1]_ compared to + the CPython ABI. + + *Legacy features* are **not** available and it is forbidden to ``#include ``. + + The resulting filename is e.g. ``simple.hpy0.so``. + + HPy Hybrid ABI + + The HPy Hybrid ABI is essentially the same as the Universal ABI, with + the big difference that it allows to ``#include ``, to use the + legacy features and thus to allow incremental porting. + + At the ABI level the resulting binary depends on *both* HPy and the + specific Python implementation which was used to compile the extension. + As the name suggests, this means that the binary is not "universal", + thus negating some of the benefits of HPy. The main benefit of using + the HPy Hybrid ABI instead of the CPython ABI is being able to use the + :ref:`debug-mode:Debug mode` on the HPy parts, and faster speed on + alternative implementations. + + *Legacy features* are available. + + The resulting filename is e.g. ``simple.hpy0-cp38.so``. + + +Moreover, each alternative Python implementation could decide to implement its +own non-universal ABI if it makes sense for them. For example, a hypothetical +project *DummyPython* could decide to ship its own ``hpy.h`` which implements +the HPy API but generates a DLL which targets the DummyPython ABI. + +This means that to compile an extension for CPython, you can choose whether to +target the CPython ABI or the Universal ABI. The advantage of the former is +that it runs at native speed, while the advantage of the latter is that you +can distribute a single binary, although with a small speed penalty on +CPython. Obviously, nothing stops you compiling and distributing both +versions: this is very similar to what most projects are already doing, since +they automatically compile and distribute extensions for many different +CPython versions. + +From the user point of view, extensions compiled for the CPython ABI can be +distributed and installed as usual, while those compiled for the HPy Universal +or HPy Hybrid ABIs require installing the ``hpy.universal`` package on +CPython and have no further requirements on Pythons that support HPy natively. + + +Benefits for the Python ecosystem +--------------------------------- + +The HPy project offers some benefits to the python ecosystem, both to Python +users and to library developers. + + - C extensions can achieve much better speed on alternative implementions, + including PyPy and GraalPy: according to early :ref:`benchmarks`, an + extension written in HPy can be ~3x faster than the equivalent extension + written using ``Python.h``. + - Improved debugging: when you load extensions in :ref:`debug-mode:debug mode`, + many common mistakes are checked and reported automatically. + - Universal binaries: libraries can choose to distribute only Universal ABI + binaries. By doing so, they can support all Python implementations and + version of CPython (like PyPy, GraalPy, CPython 3.10, CPython 3.11, etc) + for which an HPy loader exists, including those that do not yet exist! This + currently comes with a small speed penalty on CPython, but for + non-performance critical libraries it might still be a good tradeoff. + - Python environments: With general availability of universal ABI binaries for + popular packages, users can create equivalent python environments that + target different Python implementations. Thus, Python users can try their + workload against different implementations and pick the one best suited for + their usage. + - In a situation where most or all popular Python extensions target the + universal ABI, it will be more feasible for CPython to make breaking changes + to its C/API for performance or maintainability reasons. + + +Cython extensions +----------------- + +If you use Cython, you can't use HPy directly. There is a +`work in progress `_ to +add Cython backend which emits HPy code instead of using ``Python.h`` code: once this is +done, you will get the benefits of HPy automatically. + + +Extensions in other languages +----------------------------- + +On the API side, HPy is designed with C in mind, so it is not directly useful +if you want to write an extension in a language other than C. + +However, Python bindings for other languages could decide to target the +:term:`HPy Universal ABI` instead of the :term:`CPython ABI`, and generate +extensions which can be loaded seamlessly on all Python implementations which +supports it. This is the route taken, for example, by `Rust +`_. + + +Benefits for alternative Python implementations +----------------------------------------------- + +If you are writing an alternative Python implementation, there is a good +chance that you already know how painful it is to support the ``Python.h`` API. +HPy is designed to be both faster and easier to implement! + +You have two choices: + + - support the Universal ABI: in this case, you just need to export the + needed functions and to add a hook to ``dlopen()`` the desired libraries + + - use a custom ABI: in this case, you have to write your own replacement for + ``hpy.h`` and recompile the C extensions with it. + + +Current status and roadmap +-------------------------- + +HPy left the early stages of development and already provides a noticeable set +of features. As on April 2023, the following milestones have been reached: + + - some prominent real-world Python packages have been ported to HPy API. There + is a list of HPy-compatible packages we know about on the HPy website + `hpyproject.org `_. + + - one can write extensions which expose module-level functions, with all + the various kinds of calling conventions. + + - there is support for argument parsing (i.e., the equivalents of + ``PyArg_ParseTuple`` and ``PyArg_ParseTupleAndKeywords``), and a + convenient complex value building (i.e., the equivalent ``Py_BuildValue``). + + - one can implement custom types, whose struct may contain references to other + Python objects using ``HPyField``. + + - there is a support for globally accessible Python object handles: ``HPyGlobal``, + which can still provide isolation for subinterpreters if needed. + + - there is support for raising and catching exceptions. + + - debug mode has been implemented and can be activated at run-time without + recompiling. It can detect leaked handles or handles used after + being closed. + + - trace mode has been implemented and can be activated just like the debug + mode. It helps analyzing the API usage (in particular wrt. performance). + + - wheels can be built for HPy extensions with ``python setup.py bdist_wheel`` + and can be installed with ``pip install``. + + - it is possible to choose between the :term:`CPython ABI` and the + :term:`HPy Universal ABI` when compiling an extension module. + + - extensions compiled with the CPython ABI work out of the box on + CPython. + + - it is possible to load HPy Universal extensions on CPython, thanks to the + ``hpy.universal`` package. + + - it is possible to load HPy Universal extensions on + PyPy (using the PyPy `hpy branch `_). + + - it is possible to load HPy Universal extensions on `GraalPy + `_. + + - there is support for multi-phase module initialization. + + - support for metaclasses has been added. + + +However, there is still a long road before HPy is usable for the general +public. In particular, the following features are on our roadmap but have not +been implemented yet: + + - many of the original ``Python.h`` functions have not been ported to + HPy yet. Porting most of them is straightforward, so for now the priority + is to test HPy with real-world Python packages and primarily resolve the + "hard" features to prove that the HPy approach works. + + - add C-level module state to complement the ``HPyGlobal`` approach. While ``HPyGlobal`` + is easier to use, it will make the migration simpler for existing extensions that + use CPython module state. + + - the integration with Cython is work in progress + + - it is not clear yet how to approach pybind11 and similar C++ bindings. They serve two use-cases: + + - As C++ wrappers for CPython API. HPy is fundamentally different in some ways, so fully compatible + pybind11 port of this API to HPy does not make sense. There can be a similar or even partially pybind11 + compatible C++ wrapper for HPy adhering to the HPy semantics and conventions (e.g., passing the + HPyContext pointer argument around, no reference stealing, etc.). + + - Way to expose (or "bind") mostly pure C++ functions as Python functions where the C++ templating + machinery takes care of the conversion between the Python world, i.e., ``PyObject*``, and the C++ + types. Porting this abstraction to HPy is possible and desired in the future. To determine the priority + or such effort, we need to get more knowledge about existing pybind11 use-cases. + + +.. _benchmarks: + +Early benchmarks +----------------- + +To validate our approach, we ported a simple yet performance critical module +to HPy. We chose `ultrajson `_ +because it is simple enough to require porting only a handful of API +functions, but at the same time it is performance critical and performs many +API calls during the parsing of a JSON file. + +This `blog post `_ +explains the results in more detail, but they can be summarized as follows: + + - ``ujson-hpy`` compiled with the CPython ABI is as fast as the original + ``ujson``. + + - A bit surprisingly, ``ujson-hpy`` compiled with the HPy Universal ABI is + only 10% slower on CPython. We need more evidence than a single benchmark + of course, but if the overhead of the HPy Universal ABI is only 10% on + CPython, many projects may find it small enough that the benefits + of distributing extensions using only the HPy Universal ABI out weight + the performance costs. + + - On PyPy, ``ujson-hpy`` runs 3x faster than the original ``ujson``. Note + the HPy implementation on PyPy is not fully optimized yet, so we expect + even bigger speedups eventually. + + +Projects involved +----------------- + +HPy was born during EuroPython 2019, were a small group of people started to +discuss the problems of the ``Python.h`` API and how it would be nice to +have a way to fix them. Since then, it has gathered the attention and interest +of people who are involved in many projects within the Python ecosystem. The +following is a (probably incomplete) list of projects whose core developers +are involved in HPy, in one way or the other. The mere presence in this list +does not mean that the project as a whole endorse or recognize HPy in any way, +just that some of the people involved contributed to the +code/design/discussions of HPy: + + - PyPy + + - CPython + + - Cython + + - GraalPy + + - RustPython + + - rust-hpy (fork of the `cpython crate `_) + + +Related work +------------- + +A partial list of alternative implementations which offer a ``Python.h`` +compatibility layer include: + + - `PyPy `_ + + - `Jython `_ + + - `IronPython `_ + + - `GraalPy `_ + +.. rubric:: Footnotes + +.. [#f1] The reason for this minor performance penalty is a layer of pointer + indirection. For instance, ``ctx->HPyLong_FromLong`` is called from the + CPython extension, which in universal mode simply forwards the call to + ``PyLong_FromLong``. It is technically possible to implement a CPython + universal module loader which edits the program's executable code at runtime + to replace that call. Note that this is not at all trivial. diff --git a/graalpython/hpy/docs/porting-example/index.rst b/graalpython/hpy/docs/porting-example/index.rst new file mode 100644 index 0000000000..e5a6dcd7b8 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/index.rst @@ -0,0 +1,356 @@ +Porting Example +=============== + +HPy supports *incrementally* porting an existing C extension from the +original Python C API to the HPy API and to have the extension compile and +run at each step along the way. + +Here we walk through porting a small C extension that implements a Point type +with some simple methods (a norm and a dot product). The Point type is minimal, +but does contain additional C attributes (the x and y values of the point) +and an attribute (obj) that contains a Python object (that we will need to +convert from a ``PyObject *`` to an ``HPyField``). + +There is a separate C file illustrating each step of the incremental port: + +* :doc:`steps/step_00_c_api`: The original C API version that we are going to + port. + +* :doc:`steps/step_01_hpy_legacy`: A possible first step where all methods still + receive ``PyObject *`` arguments and may still cast them to ``PyPointObject *`` + if they are instances of Point. + +* :doc:`steps/step_02_hpy_legacy`: Shows how to transition some methods to HPy + methods that receive ``HPy`` handles as arguments while still supporting legacy + methods that receive ``PyObject *`` arguments. + +* :doc:`steps/step_03_hpy_final`: The completed port to HPy where all methods + receive ``HPy`` handles and ``PyObject_HEAD`` has been removed. + +Take a moment to read through :doc:`steps/step_00_c_api`. Then, once you're +ready, keep reading. + +Each section below corresponds to one of the three porting steps above: + +.. contents:: + :local: + :depth: 2 + +.. note:: + The steps used here are one approach to porting a module. The specific + steps are not required. They're just an example approach. + + +Step 01: Converting the module to a (legacy) HPy module +------------------------------------------------------- + +First for the easy bit -- let's include ``hpy.h``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: #include + :end-at: #include + +We'd like to differentiate between references to ``PyPointObject`` that have +been ported to HPy and those that haven't, so let's rename it to ``PointObject`` +and alias ``PyPointObject`` to ``PointObject``. We'll keep ``PyPointObject`` for +the instances that haven't been ported yet (the legacy ones) and use +``PointObject`` where we have ported the references: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: typedef PointObject PyPointObject; + :end-at: typedef PointObject PyPointObject; + +For this step, all references will be to ``PyPointObject`` -- we'll only start +porting references in the next step. + +Let's also call ``HPyType_LEGACY_HELPERS`` to define some helper functions +for use with the ``PointObject`` struct: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPyType_LEGACY_HELPERS(PointObject) + :end-at: HPyType_LEGACY_HELPERS(PointObject) + +Again, we won't use these helpers in this step -- we're just setting things +up for later. + +Now for the big steps. + +We need to replace ``PyType_Spec`` for the ``Point`` type with the equivalent +``HPyType_Spec``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: // HPy type methods and slots (no methods or slots have been ported yet) + :end-before: // Legacy module methods (the "dot" method is still a PyCFunction) + +Initially the list of ported methods in ``point_defines`` is empty and all of +the methods are still in ``Point_slots`` which we have renamed to +``Point_legacy_slots`` for clarity. + +``SHAPE(PointObject)`` is a macro that retrieves the shape of ``PointObject`` as it +was defined by the ``HPyType_LEGACY_HELPERS`` macro and will be set to +``HPyType_BuiltinShape_Legacy`` until we replace the legacy macro with the +``HPyType_HELPERS`` one. Any type with ``legacy_slots`` or that still includes +``PyObject_HEAD`` in its struct should have ``.builtin_shape`` set to +``HPyType_BuiltinShape_Legacy``. + +Similarly we replace ``PyModuleDef`` with ``HPyModuleDef``: + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: // Legacy module methods (the "dot" method is still a PyCFunction) + :end-before: // END-OF: HPyModuleDef + +Like the type, the list of ported methods in ``module_defines`` is initially +almost empty: all the regular methods are still in ``PointModuleMethods`` which has +been renamed to ``PointModuleLegacyMethods``. However, because HPy supports only +multiphase module initialization, we must convert our module initialization code +to an "exec" slot on the module and add that slot to ``module_defines``. + +Now all that is left is to replace the module initialization function with +one that uses ``HPy_MODINIT``. The first argument is the name of the extension, +i.e., what was ``XXX`` in ``PyInit_XXX``, and the second argument +is the ``HPyModuleDef``. + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPy_MODINIT(step_01_hpy_legacy, moduledef) + +And we're done! + +Instead of the ``PyInit_XXX``, we now have an "exec" slot on the module. +We implement it with a C function that that takes an ``HPyContext *ctx`` and ``HPy mod`` +as arguments. The ``ctx`` must be forwarded as the first argument to calls to +HPy API methods. The ``mod`` argument is a handle for the module object. The runtime +creates the module for us from the provided ``HPyModuleDef``. There is no need to +call API like ``PyModule_Create`` explicitly. + +Next step is to replace ``PyType_FromSpec`` by ``HPyType_FromSpec``. + +``HPy_SetAttr_s`` is used to add the ``Point`` class to the module. HPy requires no +special ``PyModule_AddObject`` method. + +.. literalinclude:: steps/step_01_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(module_exec, HPy_mod_exec) + :end-at: } + + +Step 02: Transition some methods to HPy +--------------------------------------- + +In the previous step we put in place the type and module definitions required +to create an HPy extension module. In this step we will port some individual +methods. + +Let us start by migrating ``Point_traverse``. First we need to change +``PyObject *obj`` in the ``PointObject`` struct to ``HPyField obj``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +``HPy`` handles can only be short-lived -- i.e. local variables, arguments to +functions or return values. ``HPyField`` is the way to store long-lived +references to Python objects. For more information, please refer to the +documentation of :ref:`api-reference/hpy-field:HPyField`. + +Now we can update ``Point_traverse``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(Point_traverse, HPy_tp_traverse) + :end-before: // this is a method for creating a Point + +In the first line we used the ``HPyDef_SLOT`` macro to define a small structure +that describes the slot being implemented. The first argument, ``Point_traverse``, +is the name to assign the structure to. By convention, the ``HPyDef_SLOT`` macro +expects a function called ``Point_traverse_impl`` implementing the slot. The +second argument, ``HPy_tp_traverse``, specifies the kind of slot. + +This is a change from how slots are defined in the old C API. In the old API, +the kind of slot is only specified much lower down in ``Point_legacy_slots``. In +HPy the implementation and kind are defined in one place using a syntax +reminiscent of Python decorators. + +The implementation of traverse is now a bit simpler than in the old C API. +We no longer need to visit ``Py_TYPE(self)`` and need only ``HPy_VISIT`` +``self->obj``. HPy ensures that interpreter knows that the type of the instance +is still referenced. + +Only struct members of type ``HPyField`` can be visited with ``HPy_VISIT``, which +is why we needed to convert ``obj`` to an ``HPyField`` before we implemented the +HPy traverse. + +Next we must update ``Point_init`` to store the value of ``obj`` as an ``HPyField``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_SLOT(Point_init, HPy_tp_init) + :end-before: // this is the getter for the associated object + +There are a few new HPy constructs used here: + +- The kind of the slot passed to ``HPyDef_SLOT`` is ``HPy_tp_init``. + +- ``PointObject_AsStruct`` is defined by ``HPyType_LEGACY_HELPERS`` and returns + an instance of the ``PointObject`` struct. Because we still include + ``PyObject_HEAD`` at the start of the struct this is still a valid ``PyObject *`` + but once we finish the port the struct will no longer contain ``PyObject_HEAD`` + and this will just be an ordinary C struct with no memory overhead! + +- We use ``HPyTracker`` when parsing the arguments with ``HPyArg_ParseKeywords``. + The ``HPyTracker`` keeps track of open handles so that they can be closed + easily at the end with ``HPyTracker_Close``. + +- ``HPyArg_ParseKeywords`` is the equivalent of ``PyArg_ParseTupleAndKeywords``. + Note that the HPy version does not steal a reference like the Python + version. + +- ``HPyField_Store`` is used to store a reference to ``obj`` in the struct. The + arguments are the context (``ctx``), a handle to the object that owns the + reference (``self``), the address of the ``HPyField`` (``&p->obj``), and the + handle to the object (``obj``). + +.. note:: + + An ``HPyTracker`` is not strictly needed for ``HPyArg_ParseKeywords`` + in ``Point_init``. The arguments ``x`` and ``y`` are C floats (so there are no + handles to close) and the handle stored in ``obj`` was passed in to the + ``Point_init`` as an argument and so should not be closed. + + We showed the tracker here to demonstrate its use. You can read more + about argument parsing in the + :doc:`API docs `. + + If a tracker is needed and one is not provided, ``HPyArg_ParseKeywords`` + will return an error. + + +The last update we need to make for the change to ``HPyField`` is to migrate +``Point_obj_get`` which retrieves ``obj`` from the stored ``HPyField``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_GET(Point_obj, "obj", .doc="Associated object.") + :end-before: // an HPy method of Point + +Above we have used ``PointObject_AsStruct`` again, and then ``HPyField_Load`` to +retrieve the value of ``obj`` from the ``HPyField``. + +We've now finished all of the changes needed by introducing ``HPyField``. We +could stop here, but let's migrate one ordinary method, ``Point_norm``, to end +off this stage of the port: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") + :end-before: // this is an LEGACY function which casts a PyObject* into a PyPointObject* + +To define a method we use ``HPyDef_METH`` instead of ``HPyDef_SLOT``. ``HPyDef_METH`` +creates a small structure defining the method. The first argument is the name +to assign to the structure (``Point_norm``). The second is the Python name of +the method (``norm``). The third specifies the method signature (``HPyFunc_NOARGS`` +-- i.e. no additional arguments in this case). The last provides the docstring. +The macro then expects a function named ``Point_norm_impl`` implementing the +method. + +The rest of the implementation remains similar, except that we use +``HPyFloat_FromDouble`` to create a handle to a Python float containing the +result (i.e. the distance of the point from the origin). + +Now we are done and just have to remove the old implementations from +``Point_legacy_slots`` and add them to ``point_defines``: + +.. literalinclude:: steps/step_02_hpy_legacy.c + :lineno-match: + :start-at: static HPyDef *point_defines[] = { + :end-before: static HPyType_Spec Point_Type_spec = { + + +Step 03: Complete the port to HPy +--------------------------------- + +In this step we'll complete the port. We'll no longer include Python, remove +``PyObject_HEAD`` from the ``PointObject`` struct, and port the remaining methods. + +First, let's remove the import of ``Python.h``: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: // #include // disallow use of the old C API + :end-at: // #include // disallow use of the old C API + +And ``PyObject_HEAD`` from the struct: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: typedef struct { + :end-at: } PointObject; + +And the typedef of ``PointObject`` to ``PyPointObject``: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: // typedef PointObject PyPointObject; + :end-at: // typedef PointObject PyPointObject; + +Now any code that has not been ported should result in a compilation error. + +We must also change the type helpers from ``HPyType_LEGACY_HELPERS`` to +``HPyType_HELPERS`` so that ``PointObject_AsStruct`` knows that ``PyObject_HEAD`` +has been removed: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: HPyType_HELPERS(PointObject) + :end-at: HPyType_HELPERS(PointObject) + +There is one more method to port, the ``dot`` method which is a module method +that implements the dot product between two points: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: HPyDef_METH(dot, "dot", HPyFunc_VARARGS, .doc="Dot product.") + :end-before: // Method, type and module definitions. In this porting step all + +The changes are similar to those used in porting the ``norm`` method, except: + +- We use ``HPyArg_Parse`` instead of ``HPyArg_ParseKeywordsDict``. + +- We opted not to use an ``HPyTracker`` by passing ``NULL`` as the pointer to the + tracker when calling ``HPyArg_Parse``. There is no reason not to use a + tracker here, but the handles to the two points are passed in as arguments + to ``dot_impl`` and thus there is no need to close them (and they should not + be closed). + +We use ``PointObject_AsStruct`` and ``HPyFloat_FromDouble`` as before. + +Now that we have ported everything we can remove ``PointMethods``, +``Point_legacy_slots`` and ``PointModuleLegacyMethods``. The resulting +type definition is much cleaner: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: static HPyDef *point_defines[] = { + :end-before: // HPy module methods + +and the module definition is simpler too: + +.. literalinclude:: steps/step_03_hpy_final.c + :lineno-match: + :start-at: static HPyDef *module_defines[] = { + :end-before: HPy_MODINIT(step_03_hpy_final, moduledef) + +Now that the port is complete, when we compile our extension in HPy +universal mode, we obtain a built extension that depends only on the HPy ABI +and not on the CPython ABI at all! diff --git a/graalpython/hpy/docs/porting-example/steps/.gitignore b/graalpython/hpy/docs/porting-example/steps/.gitignore new file mode 100644 index 0000000000..36733d4937 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/.gitignore @@ -0,0 +1 @@ +step_03_hpy_final.py diff --git a/graalpython/hpy/docs/porting-example/steps/conftest.py b/graalpython/hpy/docs/porting-example/steps/conftest.py new file mode 100644 index 0000000000..06b7c319ee --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/conftest.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +""" Pytest configuration for the porting example tests. """ + +import glob +import os +import sys + +import pytest + + +sys.path.insert(0, os.path.join(os.path.dirname(__file__))) + + +class PortingStep: + def __init__(self, src): + self.name = os.path.splitext(os.path.basename(src))[0] + self.src = src + + def import_step(self): + return __import__(self.name) + + +PORTING_STEPS = [ + PortingStep(src) for src in sorted( + glob.glob(os.path.join(os.path.dirname(__file__), "step_*.c"))) +] + + +@pytest.fixture( + params=PORTING_STEPS, + ids=[step.name for step in PORTING_STEPS], +) +def step(request): + return request.param diff --git a/graalpython/hpy/docs/porting-example/steps/setup00.py b/graalpython/hpy/docs/porting-example/steps/setup00.py new file mode 100644 index 0000000000..b3ea47b7aa --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup00.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + ext_modules=[ + Extension("step_00_c_api", sources=["step_00_c_api.c"]) + ], + py_modules=["step_00_c_api"], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup01.py b/graalpython/hpy/docs/porting-example/steps/setup01.py new file mode 100644 index 0000000000..33803453a7 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup01.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_01_hpy_legacy", sources=["step_01_hpy_legacy.c"]) + ], + py_modules=["step_01_hpy_legacy"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup02.py b/graalpython/hpy/docs/porting-example/steps/setup02.py new file mode 100644 index 0000000000..6ab9b95e73 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup02.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + + +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_02_hpy_legacy", sources=["step_02_hpy_legacy.c"]) + ], + py_modules=["step_02_hpy_legacy"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/setup03.py b/graalpython/hpy/docs/porting-example/steps/setup03.py new file mode 100644 index 0000000000..c12ec40020 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/setup03.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- + +from setuptools import setup, Extension + +# now we can add --hpy-abi=universal to the invocation of setup.py to build a +# universal binary +setup( + name="hpy-porting-example", + hpy_ext_modules=[ + Extension("step_03_hpy_final", sources=["step_03_hpy_final.c"]) + ], + py_modules=["step_03_hpy_final"], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c new file mode 100644 index 0000000000..0b2742fd4d --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.c @@ -0,0 +1,165 @@ +#include +#include + +// Porting to HPy, Step 0: Original Python C API version +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains the original C API version that needs to be ported. +// +// HPy supports porting C extensions piece by piece. +// +// point_hpy_legacy_1.c illustrates a possible first step where all +// methods still receive PyObject arguments and may still cast them to +// PyPointObject if they are instances of Point. +// +// point_hpy_legacy_2.c shows how to transition some methods to HPy methods +// that receive HPy handles as arguments while still supporting legacy +// methods that receive PyObject arguments. +// +// point_hpy_final.c shows the completed port to HPy where all methods receive +// HPy handles and PyObject_HEAD has been removed. + +typedef struct { + PyObject_HEAD + double x; + double y; + PyObject *obj; +} PyPointObject; + +int Point_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyPointObject*)self)->obj); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +void Point_dealloc(PyObject *self) +{ + Py_CLEAR(((PyPointObject*)self)->obj); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +// this is a method for creating a Point +int Point_init(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *kwlist[] = {"x", "y", "obj", NULL}; + PyPointObject *p = (PyPointObject *)self; + p->x = 0.0; + p->y = 0.0; + p->obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|ddO", kwlist, + &p->x, &p->y, &p->obj)) + return -1; + if (p->obj == NULL) + p->obj = Py_None; + Py_INCREF(p->obj); + return 0; +} + +// this is a method of Point +PyObject* Point_norm(PyObject *self) +{ + PyPointObject *p = (PyPointObject *)self; + double norm; + PyObject *result; + norm = sqrt(p->x * p->x + p->y * p->y); + result = PyFloat_FromDouble(norm); + return result; +} + +// this is the getter for the associated object +PyObject* Point_obj_get(PyObject *self, void *context) +{ + PyPointObject *p = (PyPointObject *)self; + Py_INCREF(p->obj); + return p->obj; +} + +// this is an unrelated function which happens to cast a PyObject* into a +// PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + PyObject *result; + dp = p1->x * p2->x + p1->y * p2->y; + result = PyFloat_FromDouble(dp); + return result; +} + + +// Method, type and module definitions. These will be updated to add HPy +// module support in point_hpy_legacy_1.c. + +static PyMethodDef PointMethods[] = { + {"norm", (PyCFunction)Point_norm, METH_NOARGS, "Distance from origin."}, + {NULL, NULL, 0, NULL} +}; + +static PyGetSetDef PointGetSets[] = { + {"obj", (getter)Point_obj_get, NULL, "Associated object.", NULL}, + {NULL, NULL, 0, NULL} +}; + +static PyType_Slot Point_slots[] = { + {Py_tp_doc, "Point (Step 0; C API implementation)"}, + {Py_tp_init, Point_init}, + {Py_tp_methods, PointMethods}, + {Py_tp_getset, PointGetSets}, + {Py_tp_traverse, Point_traverse}, + {Py_tp_dealloc, Point_dealloc}, + {0, 0} +}; + +static PyType_Spec Point_Type_spec = { + .name = "point_capi.Point", + .basicsize = sizeof(PyPointObject), + .itemsize = 0, + .flags = Py_TPFLAGS_DEFAULT, + .slots = Point_slots +}; + +static PyMethodDef PointModuleMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "step_00_c_api", + "Point module (Step 0; C API implementation)", + -1, + PointModuleMethods, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_step_00_c_api(void) +{ + PyObject* m; + m = PyModule_Create(&moduledef); + if (m == NULL) + return NULL; + + PyObject *point_type = PyType_FromSpec(&Point_Type_spec); + if (point_type == NULL) + return NULL; + PyModule_AddObject(m, "Point", point_type); + + return m; +} diff --git a/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst new file mode 100644 index 0000000000..c9e9b76c18 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_00_c_api.rst @@ -0,0 +1,8 @@ +:orphan: + +step_00_c_api.c +=============== + +.. literalinclude:: ./step_00_c_api.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c new file mode 100644 index 0000000000..48137cc589 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.c @@ -0,0 +1,189 @@ +#include +#include +#include + +// Porting to HPy, Step 1: All legacy methods +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example first step of the port in which all methods +// still receive PyObject arguments and may still cast them to +// PyPointObject if they are instances of Point. + +typedef struct { + // PyObject_HEAD is required while legacy_slots are still used + // but can (and should) be removed once the port to HPy is completed. + PyObject_HEAD + double x; + double y; + PyObject *obj; +} PointObject; + +// This defines PyPointObject as an alias of PointObject so that existing +// code that still uses PyPointObject and expects PyObject_HEAD continues to +// compile and run. Once PyObject_HEAD has been removed, this alias should be +// removed so that code that still expects PyObject_HEAD will fail to compile. +typedef PointObject PyPointObject; + +// The legacy type helper macro defines an PointObject_AsStruct function allows +// non-legacy methods to convert HPy handles to PointObject structs. It is not +// used in this file, but is provided so that methods can start to be ported +// (see point_hpy_legacy_2.c). The legacy type helper macro is used because +// PyObject_HEAD is still present in PointObject. Once PyObject_HEAD has been +// removed (see point_hpy_final.c) we will use HPy_TYPE_HELPERS instead. +HPyType_LEGACY_HELPERS(PointObject) + +int Point_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyPointObject*)self)->obj); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +void Point_dealloc(PyObject *self) +{ + Py_CLEAR(((PyPointObject*)self)->obj); + PyTypeObject *tp = Py_TYPE(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +// this is a method for creating a Point +int Point_init(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *kwlist[] = {"x", "y", "obj", NULL}; + PyPointObject *p = (PyPointObject *)self; + p->x = 0.0; + p->y = 0.0; + p->obj = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|ddO", kwlist, + &p->x, &p->y, &p->obj)) + return -1; + if (p->obj == NULL) + p->obj = Py_None; + Py_INCREF(p->obj); + return 0; +} + +// this is a LEGACY method of Point +PyObject* Point_norm(PyObject *self) +{ + PyPointObject *p = (PyPointObject *)self; + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return PyFloat_FromDouble(norm); +} + +// this is the getter for the associated object +PyObject* Point_obj_get(PyObject *self, void *context) +{ + PyPointObject *p = (PyPointObject *)self; + Py_INCREF(p->obj); + return p->obj; +} + +// this is an LEGACY function which casts a PyObject* into a PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return PyFloat_FromDouble(dp); +} + + +// Method, type and module definitions. In this porting step, the module and +// type definitions have been ported to HPy, but the methods themselves +// remaining legacy methods. + +// Legacy methods (all methods are still legacy methods) +static PyMethodDef PointMethods[] = { + {"norm", (PyCFunction)Point_norm, METH_NOARGS, "Distance from origin."}, + {NULL, NULL, 0, NULL} +}; + +// Legacy getsets +static PyGetSetDef PointGetSets[] = { + {"obj", (getter)Point_obj_get, NULL, "Associated object.", NULL}, + {NULL, NULL, 0, NULL} +}; + +// Legacy slots (all slots are still legacy slots) +static PyType_Slot Point_legacy_slots[] = { + {Py_tp_doc, "Point (Step 1; All legacy methods)"}, + {Py_tp_init, Point_init}, + {Py_tp_methods, PointMethods}, + {Py_tp_getset, PointGetSets}, + {Py_tp_traverse, Point_traverse}, + {Py_tp_dealloc, Point_dealloc}, + {0, 0} +}; + +// HPy type methods and slots (no methods or slots have been ported yet) +static HPyDef *point_defines[] = { + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_legacy_1.Point", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_legacy_slots, + .defines = point_defines, +}; + +// HPy supports only multiphase module initialization, so we must migrate the +// single phase initialization by extracting the code that populates the module +// object with attributes into a separate 'exec' slot. The module is not +// created manually by calling API like PyModule_Create, but the runtime creates +// the module for us from the specification in HPyModuleDef, and we can provide +// additional slots to populate the module before its initialization is finalized +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// Legacy module methods (the "dot" method is still a PyCFunction) +static PyMethodDef PointModuleLegacyMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +// HPy module methods: no regular methods have been ported yet, +// but we add the module execute slot +static HPyDef *module_defines[] = { + &module_exec, + NULL +}; + +static HPyModuleDef moduledef = { + // .name = "step_01_hpy_legacy", + // ^-- .name is not needed for multiphase module initialization, + // it is always taken from the ModuleSpec + .doc = "Point module (Step 1; All legacy methods)", + .size = 0, + .legacy_methods = PointModuleLegacyMethods, + .defines = module_defines, +}; +// END-OF: HPyModuleDef + +// HPy_MODINIT takes the extension name, i.e., what would be XXX in PyInit_XXX, +// and the module definition. The module will be created by the runtime and +// passed to the HPy_mod_exec slots if any are defined +HPy_MODINIT(step_01_hpy_legacy, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst new file mode 100644 index 0000000000..7ba453ac10 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_01_hpy_legacy.rst @@ -0,0 +1,8 @@ +:orphan: + +step_01_hpy_legacy.c +==================== + +.. literalinclude:: ./step_01_hpy_legacy.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c new file mode 100644 index 0000000000..0d4ff7b04b --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.c @@ -0,0 +1,169 @@ +#include +#include +#include + +// Porting to HPy, Step 2: Porting some methods +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example second step of the port in which some methods +// have been converted to HPy methods that receive handles as arguments, but +// other methods are still legacy methods that receive PyObject arguments. + +typedef struct { + // PyObject_HEAD is required while legacy methods still access + // PointObject and should be removed once the port to HPy is completed. + PyObject_HEAD + double x; + double y; + // HPy handles are shortlived to support all GC strategies + // For that reason, PyObject* in C structs are replaced by HPyField + HPyField obj; +} PointObject; + +// This defines PyPointObject as an alias of PointObject so that existing +// code that still uses PyPointObject and expects PyObject_HEAD continues to +// compile and run. Once PyObject_HEAD has been removed, this alias should be +// removed so that code that still expects PyObject_HEAD will fail to compile. +typedef PointObject PyPointObject; + +// The legacy type helper macro defines an PointObject_AsStruct function allows +// non-legacy methods to convert HPy handles to PointObject structs. The legacy +// type helper macro is used because PyObject_HEAD is still present in +// PointObject. Once PyObject_HEAD has been removed (see point_hpy_final.c) we +// will use HPy_TYPE_HELPERS instead. +HPyType_LEGACY_HELPERS(PointObject) + +HPyDef_SLOT(Point_traverse, HPy_tp_traverse) +int Point_traverse_impl(void *self, HPyFunc_visitproc visit, void *arg) +{ + HPy_VISIT(&((PointObject*)self)->obj); + return 0; +} + +// this is a method for creating a Point +HPyDef_SLOT(Point_init, HPy_tp_init) +int Point_init_impl(HPyContext *ctx, HPy self, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + static const char *kwlist[] = {"x", "y", "obj", NULL}; + PointObject *p = PointObject_AsStruct(ctx, self); + p->x = 0.0; + p->y = 0.0; + HPy obj = HPy_NULL; + HPyTracker ht; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "|ddO", kwlist, + &p->x, &p->y, &obj)) + return -1; + if (HPy_IsNull(obj)) + obj = ctx->h_None; + /* INCREF not needed because HPyArg_ParseKeywordsDict does not steal a + reference */ + HPyField_Store(ctx, self, &p->obj, obj); + HPyTracker_Close(ctx, ht); + return 0; +} + +// this is the getter for the associated object +HPyDef_GET(Point_obj, "obj", .doc="Associated object.") +HPy Point_obj_get(HPyContext *ctx, HPy self, void* closure) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + return HPyField_Load(ctx, self, p->obj); +} + +// an HPy method of Point +HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") +HPy Point_norm_impl(HPyContext *ctx, HPy self) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return HPyFloat_FromDouble(ctx, norm); +} + +// this is an LEGACY function which casts a PyObject* into a PyPointObject* +PyObject* dot(PyObject *self, PyObject *args) +{ + PyObject *point1, *point2; + if (!PyArg_ParseTuple(args, "OO", &point1, &point2)) + return NULL; + + PyPointObject *p1 = (PyPointObject *)point1; + PyPointObject *p2 = (PyPointObject *)point2; + + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return PyFloat_FromDouble(dp); +} + +// Method, type and module definitions. In this porting step .norm() +// is ported to HPy, but dot(...) remains a legacy methods. +// Point.__init__ and Point.__doc__ are ported from legacy slots to +// HPy type defines. + +// Legacy methods (there are no legacy methods left now) +static PyMethodDef PointMethods[] = { + {NULL, NULL, 0, NULL} +}; + +// Legacy slots (all slots are still legacy slots) +static PyType_Slot Point_legacy_slots[] = { + {Py_tp_doc, "Point (Step 2; Porting some methods)"}, + {Py_tp_methods, PointMethods}, + {0, 0} +}; + +// HPy type methods and slots +static HPyDef *point_defines[] = { + &Point_init, + &Point_norm, + &Point_obj, + &Point_traverse, + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_legacy_2.Point", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .builtin_shape = SHAPE(PointObject), + .legacy_slots = Point_legacy_slots, + .defines = point_defines +}; + +// Legacy module methods (the "dot" method is still a PyCFunction) +static PyMethodDef PointModuleLegacyMethods[] = { + {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."}, + {NULL, NULL, 0, NULL} +}; + +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// HPy module methods: no regular methods have been ported yet, +// but we add the module execute slot +static HPyDef *module_defines[] = { + &module_exec, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "Point module (Step 2; Porting some methods)", + .size = 0, + .legacy_methods = PointModuleLegacyMethods, + .defines = module_defines, +}; + +HPy_MODINIT(step_02_hpy_legacy, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst new file mode 100644 index 0000000000..5523681465 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_02_hpy_legacy.rst @@ -0,0 +1,8 @@ +:orphan: + +step_02_hpy_legacy.c +==================== + +.. literalinclude:: ./step_02_hpy_legacy.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c new file mode 100644 index 0000000000..8f49ecf857 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.c @@ -0,0 +1,153 @@ +#include +// #include // disallow use of the old C API +#include + +// Porting to HPy, Step 3: All methods ported +// +// An example of porting a C extension that implements a Point type +// with a couple of simple methods (a norm and a dot product). It +// illustrates the steps needed to port types that contain additional +// C attributes (in this case, x and y). +// +// This file contains an example final step of the port in which all methods +// have been converted to HPy methods and PyObject_HEAD has been removed. + +typedef struct { + // PyObject_HEAD is no longer available in PointObject. In CPython, + // of course, it still exists but is inaccessible from HPy_AsStruct. In + // other Python implementations (e.g. PyPy) it might no longer exist at + // all. + double x; + double y; + HPyField obj; +} PointObject; + +// Code using PyPointObject relied on PyObject_HEAD and is no longer valid +// (PyObject_HEAD has been removed from the PointObject struct above). The +// typedef below has been deleted to ensure that such code is now generates +// an error during compilation. +// typedef PointObject PyPointObject; + +// The type helper macro defines an PointObject_AsStruct function allows +// converting HPy handles to PointObject structs. We no longer need to use +// the legacy type helper macro because PyObject_HEAD has been removed from +// PointObject. +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_traverse, HPy_tp_traverse) +int Point_traverse_impl(void *self, HPyFunc_visitproc visit, void *arg) +{ + HPy_VISIT(&((PointObject*)self)->obj); + return 0; +} + +// this is a method for creating a Point +HPyDef_SLOT(Point_init, HPy_tp_init) +int Point_init_impl(HPyContext *ctx, HPy self, const HPy *args, + HPy_ssize_t nargs, HPy kw) +{ + static const char *kwlist[] = {"x", "y", "obj", NULL}; + PointObject *p = PointObject_AsStruct(ctx, self); + p->x = 0.0; + p->y = 0.0; + HPy obj = HPy_NULL; + HPyTracker ht; + if (!HPyArg_ParseKeywordsDict(ctx, &ht, args, nargs, kw, "|ddO", kwlist, + &p->x, &p->y, &obj)) + return -1; + if (HPy_IsNull(obj)) + obj = ctx->h_None; + /* INCREF not needed because HPyArg_ParseKeywordsDict does not steal a + reference */ + HPyField_Store(ctx, self, &p->obj, obj); + HPyTracker_Close(ctx, ht); + return 0; +} + +// this is the getter for the associated object +HPyDef_GET(Point_obj, "obj", .doc="Associated object.") +HPy Point_obj_get(HPyContext *ctx, HPy self, void* closure) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + return HPyField_Load(ctx, self, p->obj); +} + +// an HPy method of Point +HPyDef_METH(Point_norm, "norm", HPyFunc_NOARGS, .doc="Distance from origin.") +HPy Point_norm_impl(HPyContext *ctx, HPy self) +{ + PointObject *p = PointObject_AsStruct(ctx, self); + double norm; + norm = sqrt(p->x * p->x + p->y * p->y); + return HPyFloat_FromDouble(ctx, norm); +} + +// this is an HPy function that uses Point +HPyDef_METH(dot, "dot", HPyFunc_VARARGS, .doc="Dot product.") +HPy dot_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy point1, point2; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "OO", &point1, &point2)) + return HPy_NULL; + PointObject *p1 = PointObject_AsStruct(ctx, point1); + PointObject *p2 = PointObject_AsStruct(ctx, point2); + double dp; + dp = p1->x * p2->x + p1->y * p2->y; + return HPyFloat_FromDouble(ctx, dp); +} + +// Method, type and module definitions. In this porting step all +// methods and slots have been ported to HPy and all legacy support +// has been removed. + +// Support for legacy methods and slots has been removed. It used to be: +/// +// static PyMethodDef PointMethods[] = { ... } +// static PyType_Slot Point_legacy_slots[] = { ... } +// static PyMethodDef PointModuleLegacyMethods[] = { ... } +// +// and .legacy_slots and .legacy_defines have been removed from HPyType_Spec +// HPyModuleDef respectively. + +// HPy type methods and slots +static HPyDef *point_defines[] = { + &Point_init, + &Point_norm, + &Point_obj, + &Point_traverse, + NULL +}; + +static HPyType_Spec Point_Type_spec = { + .name = "point_hpy_final.Point", + .doc = "Point (Step 03)", + .basicsize = sizeof(PointObject), + .itemsize = 0, + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_defines +}; + +HPyDef_SLOT(module_exec, HPy_mod_exec) +static int module_exec_impl(HPyContext *ctx, HPy mod) +{ + HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL); + if (HPy_IsNull(point_type)) + return -1; + HPy_SetAttr_s(ctx, mod, "Point", point_type); + return 0; +} + +// HPy module methods +static HPyDef *module_defines[] = { + &module_exec, + &dot, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "Point module (Step 3; Porting complete)", + .size = 0, + .defines = module_defines, +}; + +HPy_MODINIT(step_03_hpy_final, moduledef) diff --git a/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst new file mode 100644 index 0000000000..3b8f561fc7 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/step_03_hpy_final.rst @@ -0,0 +1,8 @@ +:orphan: + +step_03_hpy_final.c +=================== + +.. literalinclude:: ./step_03_hpy_final.c + :language: c + :linenos: diff --git a/graalpython/hpy/docs/porting-example/steps/test_porting_example.py b/graalpython/hpy/docs/porting-example/steps/test_porting_example.py new file mode 100644 index 0000000000..013b010c03 --- /dev/null +++ b/graalpython/hpy/docs/porting-example/steps/test_porting_example.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +""" Porting example tests. """ + +import pytest +import math +import types + + +class TestPorting: + def test_load_module(self, step): + mod = step.import_step() + assert isinstance(mod, types.ModuleType) + assert mod.__name__ == step.name + assert mod.__doc__.startswith("Point module (Step ") + assert type(mod.Point) == type + assert mod.Point.__doc__.startswith("Point (Step ") + assert isinstance(mod.dot, types.BuiltinFunctionType) + assert mod.dot.__doc__ == "Dot product." + + def test_create_point(self, step): + mod = step.import_step() + p = mod.Point(1, 2) + assert type(p) == mod.Point + + def test_norm(self, step): + mod = step.import_step() + assert mod.Point(1, 2).norm() == math.sqrt(5.0) + assert mod.Point(1.5).norm() == 1.5 + + def test_dot(self, step): + mod = step.import_step() + p1 = mod.Point(1, 2) + p2 = mod.Point(3, 2) + assert mod.dot(p1, p2) == 7.0 + + def test_object(self, step): + mod = step.import_step() + p1 = mod.Point(23, 42, ...) + assert p1.obj is ... + p2 = mod.Point(23, 42) + assert p2.obj is None + + def test_leak_checker(self, step): + if "hpy_final" not in step.name: + pytest.skip("Can only check for leaks in universal mode") + mod = step.import_step() + import hpy.debug + hpy.debug.set_handle_stack_trace_limit(10) + with hpy.debug.LeakDetector(): + p1 = mod.Point(1, 2, ...) + p2 = mod.Point(3, 2) + + assert p1.obj is ... + assert p2.obj is None + assert p1.norm() == math.sqrt(5.0) + assert mod.dot(p1, p2) == 7.0 + + del p1 + del p2 diff --git a/graalpython/hpy/docs/porting-guide.rst b/graalpython/hpy/docs/porting-guide.rst new file mode 100644 index 0000000000..c21961366d --- /dev/null +++ b/graalpython/hpy/docs/porting-guide.rst @@ -0,0 +1,569 @@ +Porting Guide +============= + +Porting ``PyObject *`` to HPy API constructs +-------------------------------------------- + +While in CPython one always uses ``PyObject *`` to reference to Python objects, +in HPy there are several types of handles that should be used depending on the +life-time of the handle: ``HPy``, ``HPyField``, and ``HPyGlobal``. + +- ``HPy`` represents short lived handles that live no longer than the duration of + one call from Python to HPy extension function. Rule of thumb: use for local + variables, arguments, and return values. + +- ``HPyField`` represents handles that are Python object struct fields, i.e., + live in native memory attached to some Python object. + +- ``HPyGlobal`` represents handles stored in C global variables. ``HPyGlobal`` + can provide isolation between subinterpreters. + +.. warning:: Never use a local variable of type ``HPyField``, for any reason! If + the GC kicks in, it might become invalid and become a dangling pointer. + +.. warning:: Never store `HPy` handles to a long-lived memory, for example: C + global variables or Python object structs. + +The ``HPy``/``HPyField`` dichotomy might seem arbitrary at first, but it is +needed to allow Python implementations to use a moving GC, such as PyPy. It is +easier to explain and understand the rules by thinking about how a moving GC +interacts with the C code inside an HPy extension. + +It is worth remembering that during the collection phase, a moving GC might +move an existing object to another memory location, and in that case it needs +to update all the places which store a pointer to it. In order to do so, it +needs to *know* where the pointers are. If there is a local C variable which is +unknown to the GC but contains a pointer to a GC-managed object, the variable +will point to invalid memory as soon as the object is moved. + +Back to ``HPy`` vs ``HPyField`` vs ``HPyGlobal``: + + * ``HPy`` handles must be used for all C local variables, function arguments + and function return values. They are supposed to be short-lived and closed + as soon as they are no longer needed. The debug mode will report a + long-lived ``HPy`` as a potential memory leak. + + * In PyPy and GraalPy, ``HPy`` handles are implemented using an + indirection: they are indexes inside a big list of GC-managed objects: this + big list is tracked by the GC, so when an object moves its pointer is + correctly updated. + + * ``HPyField`` is for long-lived references, and the GC must be aware of + their location in memory. In PyPy, an ``HPyField`` is implemented as a + direct pointer to the object, and thus we need a way to inform the GC + where it is in memory, so that it can update its value upon moving: this + job is done by ``tp_traverse``, as explained in the next section. + + * ``HPyGlobal`` is for long-lived references that are supposed to be closed + implicitly when the module is unloaded (once module unloading is actually + implemented). ``HPyGlobal`` provides indirection to isolate subinterpreters. + Implementation wise, ``HPyGlobal`` will usually contain an index to a table + with Python objects stored in the interpreter state. + + * On CPython without subinterpreters support, ``HPy``, ``HPyGlobal``, + and ``HPyField`` are implemented as ``PyObject *``. + + * On CPython with subinterpreters support, ``HPyGlobal`` will be implemented + by an indirection through the interpreter state. Note that thanks to the HPy + design, switching between this and the more efficient implementation without + subinterpreter support will not require rebuilding of the extension (in HPy + universal mode), nor rebuilding of CPython. + +.. note:: If you write a custom type using ``HPyField``, you **MUST** also write + a ``tp_traverse`` slot. Note that this is different than the old ``Python.h`` + API, where you need ``tp_traverse`` only under certain conditions. See the + next section for more details. + +.. note:: The contract of ``tp_traverse`` is that it must visit all members of + type ``HPyField`` contained within given struct, or more precisely *owned* by + given Python object (in the sense of the *owner* argument to + ``HPyField_Store``), and nothing more, nothing less. Some Python + implementations may choose to not call the provided ``tp_traverse`` if they + know how to visit all members of type ``HPyField`` by other means (for + example, when they track them internally already). The debug mode will check + this contract. + +``tp_traverse``, ``tp_clear``, ``Py_TPFLAGS_HAVE_GC`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's quote the ``Python.h`` documentation about `GC support +`_ + + Python's support for detecting and collecting garbage which involves + circular references requires support from object types which are + “containers” for other objects which may also be containers. Types which do + not store references to other objects, or which only store references to + atomic types (such as numbers or strings), do not need to provide any + explicit support for garbage collection. + +A good rule of thumb is that if your type contains ``PyObject *`` fields, you +need to: + + 1. provide a ``tp_traverse`` slot; + + 2. provide a ``tp_clear`` slot; + + 3. add the ``Py_TPFLAGS_GC`` to the ``tp_flags``. + + +However, if you know that your ``PyObject *`` fields will contain only +"atomic" types, you can avoid these steps. + +In HPy the rules are slightly different: + + 1. if you have a field of type ``HPyField``, you always **MUST** provide a + ``tp_traverse``. This is needed so that a moving GC can track the + relevant areas of memory. However, you **MUST NOT** rely on + ``tp_traverse`` to be called; + + 2. ``tp_clear`` does not exist. On CPython, ``HPy`` automatically generates + one for you, by using ``tp_traverse`` to know which are the fields to + clear. Other implementations are free to ignore it, if it's not needed; + + 3. ``HPy_TPFLAGS_GC`` is still needed, especially on CPython. If you don't + specify it, your type will not be tracked by CPython's GC and thus it + might cause memory leaks if it's part of a reference cycle. However, + other implementations are free to ignore the flag and track the objects + anyway, if their GC implementation allows it. + +``tp_dealloc`` and ``Py_DECREF`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Generally speaking, if you have one or more ``PyObject *`` fields in the old +``Python.h``, you must provide a ``tp_dealloc`` slot where you ``Py_DECREF`` all +of them. In HPy this is not needed and will be handled automatically by the +system. + +In particular, when running on top of CPython, HPy will automatically provide +a ``tp_dealloc`` which decrefs all the fields listed by ``tp_traverse``. + +See also, :ref:`dealloc`. + + +Direct C API to HPy mappings +---------------------------- + +In many cases, migrating to HPy is as easy as just replacing a certain C API +function by the appropriate HPy API function. Table :ref:`table-mapping` gives a +mapping between C API and HPy API functions. This mapping is generated together +with the code for the :term:`CPython ABI` mode, so it is guaranteed to be correct. + + +.. mark: BEGIN API MAPPING +.. _table-mapping: +.. table:: Safe API function mapping + :widths: auto + + ================================================================================================================================== ================================================ + C API function HPY API function + ================================================================================================================================== ================================================ + `PyBool_FromLong `_ :c:func:`HPyBool_FromLong` + `PyBytes_AS_STRING `_ :c:func:`HPyBytes_AS_STRING` + `PyBytes_AsString `_ :c:func:`HPyBytes_AsString` + `PyBytes_Check `_ :c:func:`HPyBytes_Check` + `PyBytes_FromString `_ :c:func:`HPyBytes_FromString` + `PyBytes_GET_SIZE `_ :c:func:`HPyBytes_GET_SIZE` + `PyBytes_Size `_ :c:func:`HPyBytes_Size` + `PyCallable_Check `_ :c:func:`HPyCallable_Check` + `PyCapsule_IsValid `_ :c:func:`HPyCapsule_IsValid` + `PyContextVar_Get `_ :c:func:`HPyContextVar_Get` + `PyContextVar_New `_ :c:func:`HPyContextVar_New` + `PyContextVar_Set `_ :c:func:`HPyContextVar_Set` + `PyDict_Check `_ :c:func:`HPyDict_Check` + `PyDict_Copy `_ :c:func:`HPyDict_Copy` + `PyDict_Keys `_ :c:func:`HPyDict_Keys` + `PyDict_New `_ :c:func:`HPyDict_New` + `PyErr_Clear `_ :c:func:`HPyErr_Clear` + `PyErr_ExceptionMatches `_ :c:func:`HPyErr_ExceptionMatches` + `PyErr_NewException `_ :c:func:`HPyErr_NewException` + `PyErr_NewExceptionWithDoc `_ :c:func:`HPyErr_NewExceptionWithDoc` + `PyErr_NoMemory `_ :c:func:`HPyErr_NoMemory` + `PyErr_SetFromErrnoWithFilename `_ :c:func:`HPyErr_SetFromErrnoWithFilename` + `PyErr_SetFromErrnoWithFilenameObjects `_ :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` + `PyErr_SetObject `_ :c:func:`HPyErr_SetObject` + `PyErr_SetString `_ :c:func:`HPyErr_SetString` + `PyErr_WarnEx `_ :c:func:`HPyErr_WarnEx` + `PyErr_WriteUnraisable `_ :c:func:`HPyErr_WriteUnraisable` + `PyEval_EvalCode `_ :c:func:`HPy_EvalCode` + `PyEval_RestoreThread `_ :c:func:`HPy_ReenterPythonExecution` + `PyEval_SaveThread `_ :c:func:`HPy_LeavePythonExecution` + `PyFloat_AsDouble `_ :c:func:`HPyFloat_AsDouble` + `PyFloat_FromDouble `_ :c:func:`HPyFloat_FromDouble` + `PyImport_ImportModule `_ :c:func:`HPyImport_ImportModule` + `PyIter_Check `_ :c:func:`HPyIter_Check` + `PyIter_Next `_ :c:func:`HPyIter_Next` + `PyList_Append `_ :c:func:`HPyList_Append` + `PyList_Check `_ :c:func:`HPyList_Check` + `PyList_Insert `_ :c:func:`HPyList_Insert` + `PyList_New `_ :c:func:`HPyList_New` + `PyLong_AsDouble `_ :c:func:`HPyLong_AsDouble` + `PyLong_AsLong `_ :c:func:`HPyLong_AsLong` + `PyLong_AsLongLong `_ :c:func:`HPyLong_AsLongLong` + `PyLong_AsSize_t `_ :c:func:`HPyLong_AsSize_t` + `PyLong_AsSsize_t `_ :c:func:`HPyLong_AsSsize_t` + `PyLong_AsUnsignedLong `_ :c:func:`HPyLong_AsUnsignedLong` + `PyLong_AsUnsignedLongLong `_ :c:func:`HPyLong_AsUnsignedLongLong` + `PyLong_AsUnsignedLongLongMask `_ :c:func:`HPyLong_AsUnsignedLongLongMask` + `PyLong_AsUnsignedLongMask `_ :c:func:`HPyLong_AsUnsignedLongMask` + `PyLong_AsVoidPtr `_ :c:func:`HPyLong_AsVoidPtr` + `PyLong_FromLong `_ :c:func:`HPyLong_FromLong` + `PyLong_FromLongLong `_ :c:func:`HPyLong_FromLongLong` + `PyLong_FromSize_t `_ :c:func:`HPyLong_FromSize_t` + `PyLong_FromSsize_t `_ :c:func:`HPyLong_FromSsize_t` + `PyLong_FromUnsignedLong `_ :c:func:`HPyLong_FromUnsignedLong` + `PyLong_FromUnsignedLongLong `_ :c:func:`HPyLong_FromUnsignedLongLong` + `PyNumber_Absolute `_ :c:func:`HPy_Absolute` + `PyNumber_Add `_ :c:func:`HPy_Add` + `PyNumber_And `_ :c:func:`HPy_And` + `PyNumber_Check `_ :c:func:`HPyNumber_Check` + `PyNumber_Divmod `_ :c:func:`HPy_Divmod` + `PyNumber_Float `_ :c:func:`HPy_Float` + `PyNumber_FloorDivide `_ :c:func:`HPy_FloorDivide` + `PyNumber_InPlaceAdd `_ :c:func:`HPy_InPlaceAdd` + `PyNumber_InPlaceAnd `_ :c:func:`HPy_InPlaceAnd` + `PyNumber_InPlaceFloorDivide `_ :c:func:`HPy_InPlaceFloorDivide` + `PyNumber_InPlaceLshift `_ :c:func:`HPy_InPlaceLshift` + `PyNumber_InPlaceMatrixMultiply `_ :c:func:`HPy_InPlaceMatrixMultiply` + `PyNumber_InPlaceMultiply `_ :c:func:`HPy_InPlaceMultiply` + `PyNumber_InPlaceOr `_ :c:func:`HPy_InPlaceOr` + `PyNumber_InPlacePower `_ :c:func:`HPy_InPlacePower` + `PyNumber_InPlaceRemainder `_ :c:func:`HPy_InPlaceRemainder` + `PyNumber_InPlaceRshift `_ :c:func:`HPy_InPlaceRshift` + `PyNumber_InPlaceSubtract `_ :c:func:`HPy_InPlaceSubtract` + `PyNumber_InPlaceTrueDivide `_ :c:func:`HPy_InPlaceTrueDivide` + `PyNumber_InPlaceXor `_ :c:func:`HPy_InPlaceXor` + `PyNumber_Index `_ :c:func:`HPy_Index` + `PyNumber_Invert `_ :c:func:`HPy_Invert` + `PyNumber_Long `_ :c:func:`HPy_Long` + `PyNumber_Lshift `_ :c:func:`HPy_Lshift` + `PyNumber_MatrixMultiply `_ :c:func:`HPy_MatrixMultiply` + `PyNumber_Multiply `_ :c:func:`HPy_Multiply` + `PyNumber_Negative `_ :c:func:`HPy_Negative` + `PyNumber_Or `_ :c:func:`HPy_Or` + `PyNumber_Positive `_ :c:func:`HPy_Positive` + `PyNumber_Power `_ :c:func:`HPy_Power` + `PyNumber_Remainder `_ :c:func:`HPy_Remainder` + `PyNumber_Rshift `_ :c:func:`HPy_Rshift` + `PyNumber_Subtract `_ :c:func:`HPy_Subtract` + `PyNumber_TrueDivide `_ :c:func:`HPy_TrueDivide` + `PyNumber_Xor `_ :c:func:`HPy_Xor` + `PyObject_ASCII `_ :c:func:`HPy_ASCII` + `PyObject_Bytes `_ :c:func:`HPy_Bytes` + `PyObject_Call `_ :c:func:`HPy_CallTupleDict` + `PyObject_DelItem `_ :c:func:`HPy_DelItem` + `PyObject_GetAttr `_ :c:func:`HPy_GetAttr` + `PyObject_GetAttrString `_ :c:func:`HPy_GetAttr_s` + `PyObject_GetItem `_ :c:func:`HPy_GetItem` + `PyObject_GetIter `_ :c:func:`HPy_GetIter` + `PyObject_HasAttr `_ :c:func:`HPy_HasAttr` + `PyObject_HasAttrString `_ :c:func:`HPy_HasAttr_s` + `PyObject_Hash `_ :c:func:`HPy_Hash` + `PyObject_IsTrue `_ :c:func:`HPy_IsTrue` + `PyObject_Length `_ :c:func:`HPy_Length` + `PyObject_Repr `_ :c:func:`HPy_Repr` + `PyObject_RichCompare `_ :c:func:`HPy_RichCompare` + `PyObject_RichCompareBool `_ :c:func:`HPy_RichCompareBool` + `PyObject_SetAttr `_ :c:func:`HPy_SetAttr` + `PyObject_SetAttrString `_ :c:func:`HPy_SetAttr_s` + `PyObject_SetItem `_ :c:func:`HPy_SetItem` + `PyObject_Str `_ :c:func:`HPy_Str` + `PyObject_Type `_ :c:func:`HPy_Type` + `PyObject_TypeCheck `_ :c:func:`HPy_TypeCheck` + `PyObject_Vectorcall `_ :c:func:`HPy_Call` + `PyObject_VectorcallMethod `_ :c:func:`HPy_CallMethod` + `PySequence_Contains `_ :c:func:`HPy_Contains` + `PySequence_DelSlice `_ :c:func:`HPy_DelSlice` + `PySequence_GetSlice `_ :c:func:`HPy_GetSlice` + `PySequence_SetSlice `_ :c:func:`HPy_SetSlice` + `PySlice_AdjustIndices `_ :c:func:`HPySlice_AdjustIndices` + `PySlice_New `_ :c:func:`HPySlice_New` + `PySlice_Unpack `_ :c:func:`HPySlice_Unpack` + `PyTuple_Check `_ :c:func:`HPyTuple_Check` + `PyType_IsSubtype `_ :c:func:`HPyType_IsSubtype` + `PyUnicode_AsASCIIString `_ :c:func:`HPyUnicode_AsASCIIString` + `PyUnicode_AsLatin1String `_ :c:func:`HPyUnicode_AsLatin1String` + `PyUnicode_AsUTF8AndSize `_ :c:func:`HPyUnicode_AsUTF8AndSize` + `PyUnicode_AsUTF8String `_ :c:func:`HPyUnicode_AsUTF8String` + `PyUnicode_Check `_ :c:func:`HPyUnicode_Check` + `PyUnicode_DecodeASCII `_ :c:func:`HPyUnicode_DecodeASCII` + `PyUnicode_DecodeFSDefault `_ :c:func:`HPyUnicode_DecodeFSDefault` + `PyUnicode_DecodeFSDefaultAndSize `_ :c:func:`HPyUnicode_DecodeFSDefaultAndSize` + `PyUnicode_DecodeLatin1 `_ :c:func:`HPyUnicode_DecodeLatin1` + `PyUnicode_EncodeFSDefault `_ :c:func:`HPyUnicode_EncodeFSDefault` + `PyUnicode_FromEncodedObject `_ :c:func:`HPyUnicode_FromEncodedObject` + `PyUnicode_FromString `_ :c:func:`HPyUnicode_FromString` + `PyUnicode_FromWideChar `_ :c:func:`HPyUnicode_FromWideChar` + `PyUnicode_ReadChar `_ :c:func:`HPyUnicode_ReadChar` + `PyUnicode_Substring `_ :c:func:`HPyUnicode_Substring` + `Py_FatalError `_ :c:func:`HPy_FatalError` + ================================================================================================================================== ================================================ +.. mark: END API MAPPING + + +.. note: There are, of course, also cases where it is not possible to map directly and safely from a C API function (or concept) to an HPy API function (or concept). + +Reference Counting ``Py_INCREF`` and ``Py_DECREF`` +-------------------------------------------------- + +The equivalents of ``Py_INCREF`` and ``Py_DECREF`` are essentially +:c:func:`HPy_Dup` and :c:func:`HPy_Close`, respectively. The main difference is +that :c:func:`HPy_Dup` gives you a *new handle* to the same object which means +that the two handles may be different if comparing them with ``memcmp`` but +still reference the same object. As a consequence, you may close a handle only +once, i.e., you cannot call :c:func:`HPy_Close` twice on the same ``HPy`` +handle, even if returned from ``HPy_Dup``. For examples, see also sections +:ref:`api:handles` and :ref:`api:handles vs ``pyobject *``` + +Calling functions ``PyObject_Call`` and ``PyObject_CallObject`` +--------------------------------------------------------------- + +Both ``PyObject_Call`` and ``PyObject_CallObject`` are replaced by +``HPy_CallTupleDict(callable, args, kwargs)`` in which either or both of +``args`` and ``kwargs`` may be null handles. + +``PyObject_Call(callable, args, kwargs)`` becomes:: + + HPy result = HPy_CallTupleDict(ctx, callable, args, kwargs); + +``PyObject_CallObject(callable, args)`` becomes:: + + HPy result = HPy_CallTupleDict(ctx, callable, args, HPy_NULL); + +If ``args`` is not a handle to a tuple or ``kwargs`` is not a handle to a +dictionary, ``HPy_CallTupleDict`` will return ``HPy_NULL`` and raise a +``TypeError``. This is different to ``PyObject_Call`` and +``PyObject_CallObject`` which may segfault instead. + +Calling Protocol +---------------- + +Both the *tp_call* and *vectorcall* calling protocols are replaced by HPy's +calling protocol. This is done by defining slot ``HPy_tp_call``. HPy uses only +one calling convention which is similar to the vectorcall calling convention. +In the following example, we implement a call function for a simple Euclidean +vector type. The function computes the dot product of two vectors. + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN EuclideanVectorObject + :end-before: // END EuclideanVectorObject + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN HPy_tp_call + :end-before: // END HPy_tp_call + +Positional and keyword arguments are passed as C array ``args``. Argument +``nargs`` specifies the number of positional arguments. Argument ``kwnames`` is +a tuple containing the names of the keyword arguments. The keyword argument +values are appended to positional arguments and start at ``args[nargs]`` (if +there are any). + +In the above example, function ``call_impl`` will be used by default to call all +instances of the corresponding type. It is also possible to install (maybe +specialized) call function implementations per instances by using function +:c:func:`HPy_SetCallFunction`. This needs to be done in the constructor of an +object. For example: + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN HPy_SetCallFunction + :end-before: // END HPy_SetCallFunction + +Limitations +~~~~~~~~~~~ + + 1. It is not possible to use slot ``HPy_tp_call`` for a *var object* (i.e. if + :c:member:`HPyType_Spec.itemsize` is greater ``0``). Reason: HPy installs + a hidden field in the object's data to store the call function pointer + which is appended to everything else. In case of ``EuclideanVectorObject``, + a field is implicitly appended after member ``y``. This is not possible for + var objects because the variable part will also start after the fixed + members. + + 2. It is also not possible to use slot ``HPy_tp_call`` with a legacy type that + inherits the basicsize (i.e. if :c:member:`HPyType_Spec.basicsize` is + ``0``) for the same reason as above. + +To overcome these limitations, it is still possible to manually embed a field +for the call function pointer in a type's C struct and tell HPy where this field +is. In this case, it is always necessary to set the call function pointer using +:c:func:`HPy_SetCallFunction` in the object's constructor. This procedure is +less convenient than just using slot ``HPy_tp_cal`` but still not hard to use. +Consider following example. We define a struct ``FooObject`` and declare field +``HPyCallFunction call_func`` which will be used to store the call function's +pointer. We need to register the offset of that field with member +``__vectorcalloffset__`` and in the constructor ``Foo_new``, we assign the call +function ``Foo_call_func``. + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN FooObject + :end-before: // END FooObject + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN vectorcalloffset + :end-before: // END vectorcalloffset + +.. note:: + + In contrast to CPython's vectorcall protocol, ``nargs`` will never have flag + ``PY_VECTORCALL_ARGUMENTS_OFFSET`` set. It will **only** be the positional + argument count. + +.. _call-migration: + +Incremental Migration to HPy's Calling Protocol +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to support incremental migration, HPy provides helper function +:c:func:`HPyHelpers_PackArgsAndKeywords` that converts from HPy's calling +convention to CPython's *tp_call* calling convention. Consider following +example: + +.. literalinclude:: examples/snippets/hpycall.c + :start-after: // BEGIN pack_args + :end-before: // END pack_args + +In this example, ``args``, ``nargs``, and ``kwnames`` are used to create a tuple +of positional arguments ``args_tuple`` and a keyword arguments dictionary +``kwd``. + +PyModule_AddObject +------------------ + +``PyModule_AddObject`` is replaced with a regular :c:func:`HPy_SetAttr_s`. There +is no ``HPyModule_AddObject`` function because it has an unusual refcount +behavior (stealing a reference but only when it returns ``0``). + + +.. _dealloc: + +Deallocator slot ``Py_tp_dealloc`` +---------------------------------- + +``Py_tp_dealloc`` essentially becomes ``HPy_tp_destroy``. The name intentionally +differs because there are major differences: while the slot function of +``Py_tp_dealloc`` receives the full object (which makes it possible to resurrect +it) and while there are no restrictions on what you may call in the C API +deallocator, you must not do that in HPy's deallocator. + +The two major restrictions apply to the slot function of ``HPy_tp_destroy``: + +1. The function must be **thread-safe**. +2. The function **must not** call into the interpreter. + +The idea is, that ``HPy_tp_destroy`` just releases native resources (e.g. by +using C lib's ``free`` function). Therefore, it only receives a pointer to the +object's native data (and not a handle to the object) and it does not receive an +``HPyContext`` pointer argument. + +For the time being, HPy will support the ``HPy_tp_finalize`` slot where those +tight restrictions do not apply at the (significant) cost of performance. + +Special slots ``Py_tp_methods``, ``Py_tp_members``, and ``Py_tp_getset`` +------------------------------------------------------------------------ + +There is no direct replacement for C API slots ``Py_tp_methods``, +``Py_tp_members``, and ``Py_tp_getset`` because they are no longer needed. +Methods, members, and get/set descriptors are specified *flatly* together with +the other slots, using the standard mechanisms of :c:macro:`HPyDef_METH`, +:c:macro:`HPyDef_MEMBER`, and :c:macro:`HPyDef_GETSET`. The resulting ``HPyDef`` +structures are then accumulated in :c:member:`HPyType_Spec.defines`. + +Creating lists and tuples +------------------------- + +The C API way of creating lists and tuples is to create an empty list or tuple +object using ``PyList_New(n)`` or ``PyTuple_New(n)``, respectively, and then to +fill the empty object using ``PyList_SetItem / PyList_SET_ITEM`` or +``PyTuple_SetItem / PyTuple_SET_ITEM``, respectively. + +This is in particular problematic for tuples because they are actually +immutable. HPy goes a different way and provides a dedicated *builder* API to +avoid the (temporary) inconsistent state during object initialization. + +Long story short, doing the same in HPy with builders is still very simple and +straight forward. Following an example for creating a list: + +.. code-block:: c + + PyObject *list = PyList_New(5); + if (list == NULL) + return NULL; /* error */ + PyList_SET_ITEM(list, 0, item0); + PyList_SET_ITEM(list, 1, item0); + ... + PyList_SET_ITEM(list, 4, item0); + /* now 'list' is ready to use */ + +becomes + +.. code-block:: c + + HPyListBuilder builder = HPyListBuilder_New(ctx, 5); + HPyListBuilder_Set(ctx, builder, 0, h_item0); + HPyListBuilder_Set(ctx, builder, 1, h_item1); + ... + HPyListBuilder_Set(ctx, builder, 4, h_item4); + HPy h_list = HPyListBuilder_Build(ctx, builder); + if (HPy_IsNull(h_list)) + return HPy_NULL; /* error */ + +.. note:: In contrast to ``PyList_SetItem``, ``PyList_SET_ITEM``, + ``PyTuple_SetItem``, and ``PyTuple_SET_ITEM``, the builder functions + :c:func:`HPyListBuilder_Set` and :c:func:`HPyTupleBuilder_Set` are **NOT** + stealing references. It is necessary to close the passed item handles (e.g. + ``h_item0`` in the above example) if they are no longer needed. + +If an error occurs during building the list or tuple, it is necessary to call +:c:func:`HPyListBuilder_Cancel` or :c:func:`HPyTupleBuilder_Cancel`, +respectively, to avoid memory leaks. + +For details, see the API reference documentation +:doc:`api-reference/hpy-sequence`. + +Buffers +------- + +The buffer API in HPy is implemented using the ``HPy_buffer`` struct, which looks +very similar to ``Py_buffer`` (refer to the `CPython documentation +`_ for the +meaning of the fields):: + + typedef struct { + void *buf; + HPy obj; + HPy_ssize_t len; + HPy_ssize_t itemsize; + int readonly; + int ndim; + char *format; + HPy_ssize_t *shape; + HPy_ssize_t *strides; + HPy_ssize_t *suboffsets; + void *internal; + } HPy_buffer; + +Buffer slots for HPy types are specified using slots ``HPy_bf_getbuffer`` and +``HPy_bf_releasebuffer`` on all supported Python versions, even though the +matching PyType_Spec slots, ``Py_bf_getbuffer`` and ``Py_bf_releasebuffer``, are +only available starting from CPython 3.9. + +Multi-phase Module Initialization +--------------------------------- + +HPy supports only multi-phase module initialization (PEP 451). This means that +the module object is typically created by interpreter from the ``HPyModuleDef`` +specification and there is no "init" function. However, the module can define +one or more ``HPy_mod_exec`` slots, which will be executed just after the module +object is created. Inside the code of those slots, one can usually perform the same +initialization as before. + +Example of legacy single phase module initialization that uses Python/C API: + +.. literalinclude:: examples/snippets/legacyinit.c + :start-after: // BEGIN + :end-before: // END + +The same code structure ported to HPy and multi-phase module initialization: + +.. literalinclude:: examples/snippets/hpyinit.c + :start-after: // BEGIN + :end-before: // END diff --git a/graalpython/hpy/docs/quickstart.rst b/graalpython/hpy/docs/quickstart.rst new file mode 100644 index 0000000000..f57063cfa8 --- /dev/null +++ b/graalpython/hpy/docs/quickstart.rst @@ -0,0 +1,55 @@ +HPy Quickstart +============== + +This section shows how to quickly get started with HPy by creating +a simple HPy extension from scratch. + +Install latest HPy release: + +.. code-block:: console + + python3 -m pip install hpy + +Alternatively, you can also install HPy from the development repository: + +.. code-block:: console + + python3 -m pip install git+https://github.com/hpyproject/hpy.git#egg=hpy + +Create a new directory for the new HPy extension. Location and name of the directory +do not matter. Add the following two files: + +.. literalinclude:: examples/quickstart/quickstart.c + +.. literalinclude:: examples/quickstart/setup.py + :language: python + +Build the extension: + +.. code-block:: console + + python3 setup.py --hpy-abi=universal develop + +Try it out -- start Python console in the same directory and type: + +.. literalinclude:: examples/tests.py + :start-after: test_quickstart + :end-before: # END: test_quickstart + +Notice the shared library that was created by running ``setup.py``: + +.. code-block:: console + + > ls *.so + quickstart.hpy0.so + +It does not have Python version encoded in it. If you happen to have +GraalPy or PyPy installation that supports given HPy version, you can +try running the same extension on it. For example, start +``$GRAALVM_HOME/bin/graalpy`` in the same directory and type the same +Python code: the extension should load and work just fine. + +Where to go next? + + - :ref:`Simple documented HPy extension example` + - :doc:`Tutorial: porting Python/C API extension to HPy` diff --git a/graalpython/hpy/docs/requirements.txt b/graalpython/hpy/docs/requirements.txt new file mode 100644 index 0000000000..bb561e010f --- /dev/null +++ b/graalpython/hpy/docs/requirements.txt @@ -0,0 +1,8 @@ +# Requirements for building the documentation +Jinja2==3.1.3 +sphinx==5.0.2 +sphinx-rtd-theme==2.0.0 +sphinx-autobuild==2021.3.14 +sphinx-c-autodoc==1.3.0 +clang==17.0.6 +docutils==0.16 # docutils >= 0.17 fails to render bullet lists :/ diff --git a/graalpython/hpy/docs/trace-mode.rst b/graalpython/hpy/docs/trace-mode.rst new file mode 100644 index 0000000000..b8763b4220 --- /dev/null +++ b/graalpython/hpy/docs/trace-mode.rst @@ -0,0 +1,64 @@ +Trace Mode +========== + +HPy's trace mode allows you to analyze the usage of the HPy API. The two +fundamental metrics are ``call count`` and ``duration``. As the name already +suggests, ``call count`` tells you how often a certain HPy API function was called +and ``duration`` uses a monotonic clock to measure how much (accumulated) time was +spent in a certain HPy API function. It is further possible to register custom +*on-enter* and *on-exit* Python functions. + +As with the debug mode, the trace mode can be activated at *import time*, so no +recompilation is required. + + +Activating Trace Mode +--------------------- + +Similar to how the +:ref:`debug mode is activated `, use +environment variable ``HPY``. If ``HPY=trace``, then all HPy modules are loaded +with the trace context. Alternatively, it is also possible to specify the mode +per module like this: ``HPY=modA:trace,modB:trace``. +Environment variable ``HPY_LOG`` also works. + + +Using Trace Mode +---------------- + +The trace mode can be accessed via the shipped module ``hpy.trace``. It provides +following functions: + +* ``get_call_counts()`` returns a dict. The HPy API function names are used as + keys and the corresponding call count is the value. +* ``get_durations()`` also returns a dict similar to ``get_call_counts`` but + the value is the accumulated time spent in the corresponding HPy API + function (in nanoseconds). Note, the used clock does not necessarily have a + nanosecond resolution which means that the least significant digits may not be + accurate. +* ``set_trace_functions(on_enter=None, on_exit=None)`` allows the user to + register custom trace functions. The function provided for ``on_enter`` and + ``on_exit`` functions will be executed before and after and HPy API function + is and was executed, respectively. Passing ``None`` to any of the two + arguments or omitting one will clear the corresponding function. +* ``get_frequency()`` returns the resolution of the used clock to measure the + time in Hertz. For example, a value of ``10000000`` corresponds to + ``10 MHz``. In that case, the two least significant digits of the durations + are inaccurate. + + +Example +------- + +Following HPy function uses ``HPy_Add``: + +.. literalinclude:: examples/snippets/snippets.c + :start-after: // BEGIN: add + :end-before: // END: add + +When this script is executed in trace mode: + +.. literalinclude:: examples/trace-example.py + :language: python + +The output is ``get_call_counts()["ctx_Add"] == 1``. diff --git a/graalpython/hpy/docs/xxx-unsorted-notes.txt b/graalpython/hpy/docs/xxx-unsorted-notes.txt new file mode 100644 index 0000000000..159b6e315c --- /dev/null +++ b/graalpython/hpy/docs/xxx-unsorted-notes.txt @@ -0,0 +1,91 @@ +Goal: better C API for CPython and 1st-class citizen on PyPy + +- everything should be opaque by default + + - should we have an API to "query" if an object supports a certain low-level layout? E.g list-of-integer + + - should we have an API to e.g. ask "give me the nth C-level long in the + list? And then rely on the compiler to turn this into an efficient loop: + long PyObject_GetLongItem(PyHandle h, long index) ? + + +- we need PyObject_GetItem(handle) and PyObject_GetItem(int_index) + +- PyHandle PyObject_GetMappingProtocol(PyHandle o): ask a python object if it + has the "mapping" interface; then you can close it when you are done with it + and e.g. PyPy can release when it's no longer needed + +- should we write a tool to convert from the old API to the new API? + +- how we do deploy it? Should we have a single PyHandle.h file which is enough + to include? Or do like numpy.get_include_dirs()? + + - we need to do what cffi does (and ship our version on PyPy) + + - cython might need/want to ship its own vendored version of PyHandle + +- we need a versioning system which is possible to query at runtime? (to check + that it was compiled with the "correct/expected" PyHandle version + +- what to do with existing code which actively check whether the refcount is 1? E.g. PyString_Resize? + +- fast c-to-c calls: should we use argument clinic or something similar? + + + +Protocol sketch +---------------- + +HPySequence_long x = HPy_AsSequence_long(obj); /* it is possible to fail and you should be ready to handle the fallback */ +int len = HPy_Sequence_Len_long(x, obj); +for(int i=0; i sequence index access +* comparisons (<, <=, ==, !=, >=, >) + * rich comparisons + * boolean comparisons (0/1) +* call (call/vectorcall/method/special-method) + * e.g. PyCall_SpecialMethodOneArg(obj, __add__, arg) -> call specific macro + * … +* context manager (enter/exit) + * -> call special method +* async (await/aiter/anext) + * -> call special method diff --git a/graalpython/hpy/gdb-py.test b/graalpython/hpy/gdb-py.test new file mode 100755 index 0000000000..7412a06214 --- /dev/null +++ b/graalpython/hpy/gdb-py.test @@ -0,0 +1,3 @@ +#!/bin/bash + +gdb --eval-command run --args python3 -m pytest -s "$@" diff --git a/graalpython/lib-graalpython/modules/hpy/debug/__init__.py b/graalpython/hpy/hpy/debug/__init__.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/__init__.py rename to graalpython/hpy/hpy/debug/__init__.py diff --git a/graalpython/lib-graalpython/modules/hpy/debug/leakdetector.py b/graalpython/hpy/hpy/debug/leakdetector.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/leakdetector.py rename to graalpython/hpy/hpy/debug/leakdetector.py diff --git a/graalpython/lib-graalpython/modules/hpy/debug/pytest.py b/graalpython/hpy/hpy/debug/pytest.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/debug/pytest.py rename to graalpython/hpy/hpy/debug/pytest.py diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c b/graalpython/hpy/hpy/debug/src/_debugmod.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/_debugmod.c rename to graalpython/hpy/hpy/debug/src/_debugmod.c diff --git a/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i new file mode 100644 index 0000000000..0cd23a98aa --- /dev/null +++ b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_call.i @@ -0,0 +1,468 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.debug.autogen_debug_ctx_call_i + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + + case HPyFunc_NOARGS: { + HPyFunc_noargs f = (HPyFunc_noargs)func; + _HPyFunc_args_NOARGS *a = (_HPyFunc_args_NOARGS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_self); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_self); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_O: { + HPyFunc_o f = (HPyFunc_o)func; + _HPyFunc_args_O *a = (_HPyFunc_args_O*)args; + DHPy dh_self = _py2dh(dctx, a->self); + DHPy dh_arg = _py2dh(dctx, a->arg); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_self, dh_arg); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_self); + DHPy_close_and_check(dctx, dh_arg); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_UNARYFUNC: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_BINARYFUNC: { + HPyFunc_binaryfunc f = (HPyFunc_binaryfunc)func; + _HPyFunc_args_BINARYFUNC *a = (_HPyFunc_args_BINARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_TERNARYFUNC: { + HPyFunc_ternaryfunc f = (HPyFunc_ternaryfunc)func; + _HPyFunc_args_TERNARYFUNC *a = (_HPyFunc_args_TERNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_INQUIRY: { + HPyFunc_inquiry f = (HPyFunc_inquiry)func; + _HPyFunc_args_INQUIRY *a = (_HPyFunc_args_INQUIRY*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_LENFUNC: { + HPyFunc_lenfunc f = (HPyFunc_lenfunc)func; + _HPyFunc_args_LENFUNC *a = (_HPyFunc_args_LENFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_SSIZEARGFUNC: { + HPyFunc_ssizeargfunc f = (HPyFunc_ssizeargfunc)func; + _HPyFunc_args_SSIZEARGFUNC *a = (_HPyFunc_args_SSIZEARGFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SSIZESSIZEARGFUNC: { + HPyFunc_ssizessizeargfunc f = (HPyFunc_ssizessizeargfunc)func; + _HPyFunc_args_SSIZESSIZEARGFUNC *a = (_HPyFunc_args_SSIZESSIZEARGFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SSIZEOBJARGPROC: { + HPyFunc_ssizeobjargproc f = (HPyFunc_ssizeobjargproc)func; + _HPyFunc_args_SSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZEOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_SSIZESSIZEOBJARGPROC: { + HPyFunc_ssizessizeobjargproc f = (HPyFunc_ssizessizeobjargproc)func; + _HPyFunc_args_SSIZESSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZESSIZEOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg3 = _py2dh(dctx, a->arg3); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, a->arg2, dh_arg3); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg3); + return; + } + case HPyFunc_OBJOBJARGPROC: { + HPyFunc_objobjargproc f = (HPyFunc_objobjargproc)func; + _HPyFunc_args_OBJOBJARGPROC *a = (_HPyFunc_args_OBJOBJARGPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_FREEFUNC: { + HPyFunc_freefunc f = (HPyFunc_freefunc)func; + _HPyFunc_args_FREEFUNC *a = (_HPyFunc_args_FREEFUNC*)args; + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + f(next_dctx, a->arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + return; + } + case HPyFunc_GETATTRFUNC: { + HPyFunc_getattrfunc f = (HPyFunc_getattrfunc)func; + _HPyFunc_args_GETATTRFUNC *a = (_HPyFunc_args_GETATTRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETATTROFUNC: { + HPyFunc_getattrofunc f = (HPyFunc_getattrofunc)func; + _HPyFunc_args_GETATTROFUNC *a = (_HPyFunc_args_GETATTROFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SETATTRFUNC: { + HPyFunc_setattrfunc f = (HPyFunc_setattrfunc)func; + _HPyFunc_args_SETATTRFUNC *a = (_HPyFunc_args_SETATTRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, a->arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_SETATTROFUNC: { + HPyFunc_setattrofunc f = (HPyFunc_setattrofunc)func; + _HPyFunc_args_SETATTROFUNC *a = (_HPyFunc_args_SETATTROFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_REPRFUNC: { + HPyFunc_reprfunc f = (HPyFunc_reprfunc)func; + _HPyFunc_args_REPRFUNC *a = (_HPyFunc_args_REPRFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_HASHFUNC: { + HPyFunc_hashfunc f = (HPyFunc_hashfunc)func; + _HPyFunc_args_HASHFUNC *a = (_HPyFunc_args_HASHFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } + case HPyFunc_RICHCMPFUNC: { + HPyFunc_richcmpfunc f = (HPyFunc_richcmpfunc)func; + _HPyFunc_args_RICHCMPFUNC *a = (_HPyFunc_args_RICHCMPFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETITERFUNC: { + HPyFunc_getiterfunc f = (HPyFunc_getiterfunc)func; + _HPyFunc_args_GETITERFUNC *a = (_HPyFunc_args_GETITERFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_ITERNEXTFUNC: { + HPyFunc_iternextfunc f = (HPyFunc_iternextfunc)func; + _HPyFunc_args_ITERNEXTFUNC *a = (_HPyFunc_args_ITERNEXTFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_DESCRGETFUNC: { + HPyFunc_descrgetfunc f = (HPyFunc_descrgetfunc)func; + _HPyFunc_args_DESCRGETFUNC *a = (_HPyFunc_args_DESCRGETFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_DESCRSETFUNC: { + HPyFunc_descrsetfunc f = (HPyFunc_descrsetfunc)func; + _HPyFunc_args_DESCRSETFUNC *a = (_HPyFunc_args_DESCRSETFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + DHPy dh_arg2 = _py2dh(dctx, a->arg2); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, dh_arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + DHPy_close_and_check(dctx, dh_arg2); + return; + } + case HPyFunc_GETTER: { + HPyFunc_getter f = (HPyFunc_getter)func; + _HPyFunc_args_GETTER *a = (_HPyFunc_args_GETTER*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + DHPy dh_result = f(next_dctx, dh_arg0, a->arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_SETTER: { + HPyFunc_setter f = (HPyFunc_setter)func; + _HPyFunc_args_SETTER *a = (_HPyFunc_args_SETTER*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1, a->arg2); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + return; + } + case HPyFunc_OBJOBJPROC: { + HPyFunc_objobjproc f = (HPyFunc_objobjproc)func; + _HPyFunc_args_OBJOBJPROC *a = (_HPyFunc_args_OBJOBJPROC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_arg1 = _py2dh(dctx, a->arg1); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + a->result = f(next_dctx, dh_arg0, dh_arg1); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg1); + return; + } + case HPyFunc_DESTRUCTOR: { + HPyFunc_destructor f = (HPyFunc_destructor)func; + _HPyFunc_args_DESTRUCTOR *a = (_HPyFunc_args_DESTRUCTOR*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + f(next_dctx, dh_arg0); + _switch_back_to_original_dctx(dctx, next_dctx); + DHPy_close_and_check(dctx, dh_arg0); + return; + } diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h b/graalpython/hpy/hpy/debug/src/autogen_debug_ctx_init.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_ctx_init.h rename to graalpython/hpy/hpy/debug/src/autogen_debug_ctx_init.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c b/graalpython/hpy/hpy/debug/src/autogen_debug_wrappers.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/autogen_debug_wrappers.c rename to graalpython/hpy/hpy/debug/src/autogen_debug_wrappers.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c b/graalpython/hpy/hpy/debug/src/debug_ctx.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx.c rename to graalpython/hpy/hpy/debug/src/debug_ctx.c diff --git a/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c b/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c new file mode 100644 index 0000000000..01d9c64788 --- /dev/null +++ b/graalpython/hpy/hpy/debug/src/debug_ctx_cpython.c @@ -0,0 +1,299 @@ +/* =========== CPython-ONLY =========== + In the following code, we use _py2h and _h2py and we assumed they are the + ones defined by CPython's version of hpy.universal. + + DO NOT COMPILE THIS FILE UNLESS YOU ARE BUILDING CPython's hpy.universal. + + If you want to compile the debug mode into your own non-CPython version of + hpy.universal, you should include debug_ctx_not_cpython.c. + ==================================== + + In theory, the debug mode is completely generic and can wrap a generic + uctx. However, CPython is special because it does not have native support + for HPy, so uctx contains the logic to call HPy functions from CPython, by + using _HPy_CallRealFunctionFromTrampoline. + + uctx->ctx_CallRealFunctionFromTrampoline converts PyObject* into UHPy. So + for the debug mode we need to: + + 1. convert the PyObject* args into UHPys. + 2. wrap the UHPys into DHPys. + 3. unwrap the resulting DHPy and convert to PyObject*. +*/ + +#include +#include "debug_internal.h" +#include "hpy/runtime/ctx_type.h" // for call_traverseproc_from_trampoline +#include "hpy/runtime/ctx_module.h" +#include "handles.h" // for _py2h and _h2py + +#if defined(_MSC_VER) +# include /* for alloca() */ +#endif + +static inline DHPy _py2dh(HPyContext *dctx, PyObject *obj) +{ + return DHPy_open(dctx, _py2h(obj)); +} + +static inline PyObject *_dh2py(HPyContext *dctx, DHPy dh) +{ + return _h2py(DHPy_unwrap(dctx, dh)); +} + +static void _buffer_h2py(HPyContext *dctx, const HPy_buffer *src, Py_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_AsPyObject(dctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static void _buffer_py2h(HPyContext *dctx, const Py_buffer *src, HPy_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_FromPyObject(dctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static HPyContext* _switch_to_next_dctx_from_cache(HPyContext *current_dctx) { + HPyContext *next_dctx = hpy_debug_get_next_dctx_from_cache(current_dctx); + if (next_dctx == NULL) { + HPyErr_NoMemory(current_dctx); + get_ctx_info(current_dctx)->is_valid = false; + get_ctx_info(next_dctx)->is_valid = true; + return NULL; + } + get_ctx_info(current_dctx)->is_valid = false; + get_ctx_info(next_dctx)->is_valid = true; + return next_dctx; +} + +static void _switch_back_to_original_dctx(HPyContext *original_dctx, HPyContext *next_dctx) { + get_ctx_info(next_dctx)->is_valid = false; + get_ctx_info(original_dctx)->is_valid = true; +} + +void debug_ctx_CallRealFunctionFromTrampoline(HPyContext *dctx, + HPyFunc_Signature sig, + void *func, void *args) +{ + switch (sig) { + case HPyFunc_VARARGS: { + HPyFunc_varargs f = (HPyFunc_varargs)func; + _HPyFunc_args_VARARGS *a = (_HPyFunc_args_VARARGS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + DHPy *dh_args = (DHPy *)alloca(a->nargs * sizeof(DHPy)); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + dh_args[i] = _py2dh(dctx, a->args[i]); + } + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, a->nargs); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_KEYWORDS: { + HPyFunc_keywords f = (HPyFunc_keywords)func; + _HPyFunc_args_KEYWORDS *a = (_HPyFunc_args_KEYWORDS*)args; + DHPy dh_self = _py2dh(dctx, a->self); + size_t n_kwnames = a->kwnames != NULL ? PyTuple_GET_SIZE(a->kwnames) : 0; + size_t nargs = PyVectorcall_NARGS(a->nargsf); + size_t nargs_with_kw = nargs + n_kwnames; + DHPy *dh_args = (DHPy *)alloca(nargs_with_kw * sizeof(DHPy)); + for (size_t i = 0; i < nargs_with_kw; i++) { + dh_args[i] = _py2dh(dctx, a->args[i]); + } + DHPy dh_kwnames = _py2dh(dctx, a->kwnames); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, nargs, dh_kwnames); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (size_t i = 0; i < nargs_with_kw; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kwnames); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_INITPROC: { + HPyFunc_initproc f = (HPyFunc_initproc)func; + _HPyFunc_args_INITPROC *a = (_HPyFunc_args_INITPROC*)args; + DHPy dh_self = _py2dh(dctx, a->self); + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + DHPy *dh_args = (DHPy *)alloca(nargs * sizeof(DHPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + dh_args[i] = _py2dh(dctx, PyTuple_GET_ITEM(a->args, i)); + } + DHPy dh_kw = _py2dh(dctx, a->kw); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = f(next_dctx, dh_self, dh_args, nargs, dh_kw); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (Py_ssize_t i = 0; i < nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kw); + return; + } + case HPyFunc_NEWFUNC: { + HPyFunc_newfunc f = (HPyFunc_newfunc)func; + _HPyFunc_args_NEWFUNC *a = (_HPyFunc_args_NEWFUNC*)args; + DHPy dh_self = _py2dh(dctx, a->self); + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + DHPy *dh_args = (DHPy *)alloca(nargs * sizeof(DHPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + dh_args[i] = _py2dh(dctx, PyTuple_GET_ITEM(a->args, i)); + } + DHPy dh_kw = _py2dh(dctx, a->kw); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = NULL; + return; + } + + DHPy dh_result = f(next_dctx, dh_self, dh_args, nargs, dh_kw); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + for (Py_ssize_t i = 0; i < nargs; i++) { + DHPy_close_and_check(dctx, dh_args[i]); + } + DHPy_close_and_check(dctx, dh_kw); + a->result = _dh2py(dctx, dh_result); + DHPy_close(dctx, dh_result); + return; + } + case HPyFunc_GETBUFFERPROC: { + HPyFunc_getbufferproc f = (HPyFunc_getbufferproc)func; + _HPyFunc_args_GETBUFFERPROC *a = (_HPyFunc_args_GETBUFFERPROC*)args; + HPy_buffer hbuf; + DHPy dh_self = _py2dh(dctx, a->self); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = f(next_dctx, dh_self, &hbuf, a->flags); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + if (a->result < 0) { + a->view->obj = NULL; + return; + } + _buffer_h2py(dctx, &hbuf, a->view); + HPy_Close(dctx, hbuf.obj); + return; + } + case HPyFunc_RELEASEBUFFERPROC: { + HPyFunc_releasebufferproc f = (HPyFunc_releasebufferproc)func; + _HPyFunc_args_RELEASEBUFFERPROC *a = (_HPyFunc_args_RELEASEBUFFERPROC*)args; + HPy_buffer hbuf; + _buffer_py2h(dctx, a->view, &hbuf); + DHPy dh_self = _py2dh(dctx, a->self); + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + return; + } + + f(next_dctx, dh_self, &hbuf); + + _switch_back_to_original_dctx(dctx, next_dctx); + + DHPy_close_and_check(dctx, dh_self); + // XXX: copy back from hbuf? + HPy_Close(dctx, hbuf.obj); + return; + } + case HPyFunc_TRAVERSEPROC: { + HPyFunc_traverseproc f = (HPyFunc_traverseproc)func; + _HPyFunc_args_TRAVERSEPROC *a = (_HPyFunc_args_TRAVERSEPROC*)args; + + HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx); + if (next_dctx == NULL) { + a->result = -1; + return; + } + + a->result = call_traverseproc_from_trampoline(f, a->self, + a->visit, a->arg); + + _switch_back_to_original_dctx(dctx, next_dctx); + return; + } + case HPyFunc_CAPSULE_DESTRUCTOR: { + HPyFunc_Capsule_Destructor f = (HPyFunc_Capsule_Destructor)func; + PyObject *capsule = (PyObject *)args; + const char *name = PyCapsule_GetName(capsule); + f(name, PyCapsule_GetPointer(capsule, name), + PyCapsule_GetContext(capsule)); + return; + } + case HPyFunc_MOD_CREATE: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + DHPy dh_arg0 = _py2dh(dctx, a->arg0); + DHPy dh_result = f(dctx, dh_arg0); + DHPy_close_and_check(dctx, dh_arg0); + a->result = _dh2py(dctx, dh_result); + _HPyModule_CheckCreateSlotResult(&a->result); + DHPy_close(dctx, dh_result); + return; + } +#include "autogen_debug_ctx_call.i" + default: + Py_FatalError("Unsupported HPyFunc_Signature in debug_ctx_cpython.c"); + } +} diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx_not_cpython.c b/graalpython/hpy/hpy/debug/src/debug_ctx_not_cpython.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_ctx_not_cpython.c rename to graalpython/hpy/hpy/debug/src/debug_ctx_not_cpython.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_handles.c b/graalpython/hpy/hpy/debug/src/debug_handles.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_handles.c rename to graalpython/hpy/hpy/debug/src/debug_handles.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/debug_internal.h b/graalpython/hpy/hpy/debug/src/debug_internal.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/debug_internal.h rename to graalpython/hpy/hpy/debug/src/debug_internal.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/dhqueue.c b/graalpython/hpy/hpy/debug/src/dhqueue.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/dhqueue.c rename to graalpython/hpy/hpy/debug/src/dhqueue.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/hpy_debug.h b/graalpython/hpy/hpy/debug/src/include/hpy_debug.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/hpy_debug.h rename to graalpython/hpy/hpy/debug/src/include/hpy_debug.h diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/memprotect.c b/graalpython/hpy/hpy/debug/src/memprotect.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/memprotect.c rename to graalpython/hpy/hpy/debug/src/memprotect.c diff --git a/graalpython/com.oracle.graal.python.jni/src/debug/stacktrace.c b/graalpython/hpy/hpy/debug/src/stacktrace.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/debug/stacktrace.c rename to graalpython/hpy/hpy/debug/src/stacktrace.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/__init__.py b/graalpython/hpy/hpy/devel/__init__.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/__init__.py rename to graalpython/hpy/hpy/devel/__init__.py diff --git a/graalpython/lib-graalpython/modules/hpy/devel/abitag.py b/graalpython/hpy/hpy/devel/abitag.py similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/abitag.py rename to graalpython/hpy/hpy/devel/abitag.py diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy.h b/graalpython/hpy/hpy/devel/include/hpy.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy.h rename to graalpython/hpy/hpy/devel/include/hpy.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h b/graalpython/hpy/hpy/devel/include/hpy/autogen_hpyfunc_declare.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyfunc_declare.h rename to graalpython/hpy/hpy/devel/include/hpy/autogen_hpyfunc_declare.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h b/graalpython/hpy/hpy/devel/include/hpy/autogen_hpyslot.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/autogen_hpyslot.h rename to graalpython/hpy/hpy/devel/include/hpy/autogen_hpyslot.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h b/graalpython/hpy/hpy/devel/include/hpy/cpy_types.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpy_types.h rename to graalpython/hpy/hpy/devel/include/hpy/cpy_types.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_api_impl.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_api_impl.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_api_impl.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_ctx.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_ctx.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_ctx.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/autogen_hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h b/graalpython/hpy/hpy/devel/include/hpy/cpython/misc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/cpython/misc.h rename to graalpython/hpy/hpy/devel/include/hpy/cpython/misc.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h b/graalpython/hpy/hpy/devel/include/hpy/forbid_python_h/Python.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/forbid_python_h/Python.h rename to graalpython/hpy/hpy/devel/include/hpy/forbid_python_h/Python.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h b/graalpython/hpy/hpy/devel/include/hpy/hpydef.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpydef.h rename to graalpython/hpy/hpy/devel/include/hpy/hpydef.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h b/graalpython/hpy/hpy/devel/include/hpy/hpyexports.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpyexports.h rename to graalpython/hpy/hpy/devel/include/hpy/hpyexports.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h b/graalpython/hpy/hpy/devel/include/hpy/hpyfunc.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpyfunc.h rename to graalpython/hpy/hpy/devel/include/hpy/hpyfunc.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h b/graalpython/hpy/hpy/devel/include/hpy/hpymodule.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpymodule.h rename to graalpython/hpy/hpy/devel/include/hpy/hpymodule.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h b/graalpython/hpy/hpy/devel/include/hpy/hpytype.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/hpytype.h rename to graalpython/hpy/hpy/devel/include/hpy/hpytype.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h b/graalpython/hpy/hpy/devel/include/hpy/inline_helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/inline_helpers.h rename to graalpython/hpy/hpy/devel/include/hpy/inline_helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h b/graalpython/hpy/hpy/devel/include/hpy/macros.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/macros.h rename to graalpython/hpy/hpy/devel/include/hpy/macros.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/argparse.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/argparse.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/argparse.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/buildvalue.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/buildvalue.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/buildvalue.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_funcs.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_funcs.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_funcs.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_module.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_module.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_module.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_type.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/ctx_type.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/ctx_type.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/format.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/format.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/format.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/helpers.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/helpers.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/helpers.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h b/graalpython/hpy/hpy/devel/include/hpy/runtime/structseq.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/runtime/structseq.h rename to graalpython/hpy/hpy/devel/include/hpy/runtime/structseq.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_ctx.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_ctx.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_ctx.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/autogen_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/autogen_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/autogen_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/hpyfunc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/hpyfunc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/hpyfunc_trampolines.h diff --git a/graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h b/graalpython/hpy/hpy/devel/include/hpy/universal/misc_trampolines.h similarity index 100% rename from graalpython/com.oracle.graal.python.hpy/include/hpy/universal/misc_trampolines.h rename to graalpython/hpy/hpy/devel/include/hpy/universal/misc_trampolines.h diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/argparse.c b/graalpython/hpy/hpy/devel/src/runtime/argparse.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/argparse.c rename to graalpython/hpy/hpy/devel/src/runtime/argparse.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/buildvalue.c b/graalpython/hpy/hpy/devel/src/runtime/buildvalue.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/buildvalue.c rename to graalpython/hpy/hpy/devel/src/runtime/buildvalue.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_bytes.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_bytes.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_bytes.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_bytes.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_call.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_call.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_call.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_call.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_capsule.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_capsule.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_capsule.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_capsule.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_contextvar.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_contextvar.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_contextvar.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_contextvar.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_err.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_err.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_err.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_err.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_eval.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_eval.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_eval.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_eval.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_listbuilder.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_listbuilder.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_listbuilder.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_listbuilder.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_long.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_long.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_long.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_long.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_module.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_module.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_module.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_module.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_object.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_object.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_object.c diff --git a/graalpython/com.oracle.graal.python.jni/src/ctx_tracker.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tracker.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/ctx_tracker.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tracker.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tuple.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuple.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tuple.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuplebuilder.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_tuplebuilder.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tuplebuilder.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_tuplebuilder.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c b/graalpython/hpy/hpy/devel/src/runtime/ctx_type.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_type.c rename to graalpython/hpy/hpy/devel/src/runtime/ctx_type.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/format.c b/graalpython/hpy/hpy/devel/src/runtime/format.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/format.c rename to graalpython/hpy/hpy/devel/src/runtime/format.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/helpers.c b/graalpython/hpy/hpy/devel/src/runtime/helpers.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/helpers.c rename to graalpython/hpy/hpy/devel/src/runtime/helpers.c diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/structseq.c b/graalpython/hpy/hpy/devel/src/runtime/structseq.c similarity index 100% rename from graalpython/lib-graalpython/modules/hpy/devel/src/runtime/structseq.c rename to graalpython/hpy/hpy/devel/src/runtime/structseq.c diff --git a/graalpython/hpy/hpy/tools/autogen/__init__.py b/graalpython/hpy/hpy/tools/autogen/__init__.py new file mode 100644 index 0000000000..81eb61e746 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/__init__.py @@ -0,0 +1,21 @@ +import pycparser +from packaging import version + +if version.parse(pycparser.__version__) < version.parse('2.21'): + raise ImportError('You need pycparser>=2.21 to run autogen') + +from .parse import HPyAPI, AUTOGEN_H + + +def generate(generators, *gen_args): + """ + Takes a sequence of autogen generators that will have access to the parse + tree of 'public_api.c' and can then generate files or whatever. + :param generators: A sequence (e.g. tuple) of classes or callables that + will produce objects with a 'write' method. The 'gen_args' will be passed + to the 'write' method on invocation. + :param gen_args: Arguments for the autogen generator instances. + """ + api = HPyAPI.parse(AUTOGEN_H) + for cls in generators: + cls(api).write(*gen_args) diff --git a/graalpython/hpy/hpy/tools/autogen/__main__.py b/graalpython/hpy/hpy/tools/autogen/__main__.py new file mode 100644 index 0000000000..59cdf04872 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/__main__.py @@ -0,0 +1,66 @@ +""" +Parse public_api.h and generate various stubs around +""" +import sys +import py +import pycparser +from packaging import version + +if version.parse(pycparser.__version__) < version.parse('2.21'): + raise ImportError('You need pycparser>=2.21 to run autogen') + +from . import generate +from .ctx import (autogen_ctx_h, + autogen_ctx_def_h, + cpython_autogen_ctx_h) +from .trampolines import (autogen_trampolines_h, + cpython_autogen_api_impl_h, + universal_autogen_ctx_impl_h) +from .hpyfunc import autogen_hpyfunc_declare_h +from .hpyfunc import autogen_hpyfunc_trampoline_h +from .hpyfunc import autogen_ctx_call_i +from .hpyfunc import autogen_cpython_hpyfunc_trampoline_h +from .hpyslot import autogen_hpyslot_h +from .debug import (autogen_debug_ctx_init_h, + autogen_debug_wrappers, + autogen_debug_ctx_call_i) +from .trace import (autogen_tracer_ctx_init_h, + autogen_tracer_wrappers, + autogen_trace_func_table_c) +from .pypy import autogen_pypy_txt +from .doc import (autogen_function_index, + autogen_doc_api_mapping) + +DEFAULT_GENERATORS = (autogen_ctx_h, + autogen_ctx_def_h, + cpython_autogen_ctx_h, + autogen_trampolines_h, + cpython_autogen_api_impl_h, + universal_autogen_ctx_impl_h, + autogen_hpyfunc_declare_h, + autogen_hpyfunc_trampoline_h, + autogen_ctx_call_i, + autogen_cpython_hpyfunc_trampoline_h, + autogen_hpyslot_h, + autogen_debug_ctx_init_h, + autogen_debug_wrappers, + autogen_debug_ctx_call_i, + autogen_tracer_ctx_init_h, + autogen_tracer_wrappers, + autogen_trace_func_table_c, + autogen_pypy_txt, + autogen_function_index, + autogen_doc_api_mapping) + + +def main(): + if len(sys.argv) != 2: + print('Usage: python -m hpy.tools.autogen OUTDIR') + sys.exit(1) + outdir = py.path.local(sys.argv[1]) + + generate(DEFAULT_GENERATORS, outdir) + + +if __name__ == '__main__': + main() diff --git a/graalpython/hpy/hpy/tools/autogen/autogen.h b/graalpython/hpy/hpy/tools/autogen/autogen.h new file mode 100644 index 0000000000..78bae31617 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/autogen.h @@ -0,0 +1,40 @@ +#define STRINGIFY(X) #X +#define HPy_ID(X) _Pragma(STRINGIFY(id=X)) \ + +#define AUTOGEN 1 + +// These are not real typedefs: they are there only to make pycparser happy +typedef int HPy; +typedef int HPyContext; +typedef int HPyModuleDef; +typedef int HPyType_Spec; +typedef int HPyType_SpecParam; +typedef int HPyCFunction; +typedef int HPy_ssize_t; +typedef int HPy_hash_t; +typedef int wchar_t; +typedef int size_t; +typedef int HPyFunc_Signature; +typedef int cpy_PyObject; +typedef int HPyField; +typedef int HPyGlobal; +typedef int HPyListBuilder; +typedef int HPyTupleBuilder; +typedef int HPyTracker; +typedef int HPy_RichCmpOp; +typedef int HPy_buffer; +typedef int HPyFunc_visitproc; +typedef int HPy_UCS4; +typedef int HPyThreadState; +typedef int HPyType_BuiltinShape; +typedef int _HPyCapsule_key; +typedef int HPyCapsule_Destructor; +typedef int int32_t; +typedef int uint32_t; +typedef int int64_t; +typedef int uint64_t; +typedef int bool; +typedef int HPy_SourceKind; +typedef int HPyCallFunction; + +#include "public_api.h" diff --git a/graalpython/hpy/hpy/tools/autogen/autogenfile.py b/graalpython/hpy/hpy/tools/autogen/autogenfile.py new file mode 100644 index 0000000000..d4fddb74c7 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/autogenfile.py @@ -0,0 +1,35 @@ +C_DISCLAIMER = """ +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by {clsname} + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ +""" + +class AutoGenFile: + LANGUAGE = 'C' + PATH = None + DISCLAIMER = None + + def __init__(self, api): + if self.DISCLAIMER is None and self.LANGUAGE == 'C': + self.DISCLAIMER = C_DISCLAIMER + self.api = api + + def generate(self): + raise NotImplementedError + + def write(self, root): + cls = self.__class__ + clsname = '%s.%s' % (cls.__module__, cls.__name__) + with root.join(self.PATH).open('w', encoding='utf-8') as f: + if self.DISCLAIMER is not None: + f.write(self.DISCLAIMER.format(clsname=clsname)) + f.write('\n') + f.write(self.generate()) + f.write('\n') diff --git a/graalpython/hpy/hpy/tools/autogen/conf.py b/graalpython/hpy/hpy/tools/autogen/conf.py new file mode 100644 index 0000000000..5ebf345783 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/conf.py @@ -0,0 +1,211 @@ +# Defines parameters for the code generation + +# Handwritten trampolines: +NO_TRAMPOLINES = { + '_HPy_New', + 'HPy_FatalError', +} + +# Generated trampoline returns given constant, +# but the context function is void +RETURN_CONSTANT = { + 'HPyErr_SetString': 'HPy_NULL', + 'HPyErr_SetObject': 'HPy_NULL', + 'HPyErr_SetFromErrnoWithFilenameObjects': 'HPy_NULL', + 'HPyErr_NoMemory': 'HPy_NULL' +} + +# If the HPy function delegates to C Python API of a different name or, in the +# case of None, if the HPy function is implemented by hand +SPECIAL_CASES = { + 'HPy_Dup': None, + 'HPy_Close': None, + 'HPyField_Load': None, + 'HPyField_Store': None, + 'HPyModule_Create': None, + 'HPy_GetAttr': 'PyObject_GetAttr', + 'HPy_GetAttr_s': 'PyObject_GetAttrString', + 'HPy_HasAttr': 'PyObject_HasAttr', + 'HPy_HasAttr_s': 'PyObject_HasAttrString', + 'HPy_SetAttr': 'PyObject_SetAttr', + 'HPy_SetAttr_s': 'PyObject_SetAttrString', + 'HPy_GetIter': 'PyObject_GetIter', + 'HPy_GetItem': 'PyObject_GetItem', + 'HPy_GetItem_i': None, + 'HPy_GetItem_s': None, + 'HPy_GetSlice': 'PySequence_GetSlice', + 'HPy_SetItem': 'PyObject_SetItem', + 'HPy_SetItem_i': None, + 'HPy_SetItem_s': None, + 'HPy_SetSlice': 'PySequence_SetSlice', + 'HPy_DelItem': 'PyObject_DelItem', + 'HPy_DelItem_i': None, + 'HPy_DelItem_s': None, + 'HPy_DelSlice': 'PySequence_DelSlice', + 'HPy_Contains': 'PySequence_Contains', + 'HPy_Length': 'PyObject_Length', + 'HPy_CallTupleDict': None, + 'HPy_Call': None, # 'PyObject_Vectorcall', no auto arg conversion + 'HPy_CallMethod': None, # 'PyObject_VectorcallMethod',no auto arg conversion + 'HPy_FromPyObject': None, + 'HPy_AsPyObject': None, + '_HPy_AsStruct_Object': None, + '_HPy_AsStruct_Type': None, + '_HPy_AsStruct_Long': None, + '_HPy_AsStruct_Float': None, + '_HPy_AsStruct_Unicode': None, + '_HPy_AsStruct_Tuple': None, + '_HPy_AsStruct_List': None, + '_HPy_AsStruct_Dict': None, + '_HPy_AsStruct_Legacy': None, + '_HPyType_GetBuiltinShape': None, + '_HPy_CallRealFunctionFromTrampoline': None, + '_HPy_CallDestroyAndThenDealloc': None, + 'HPyErr_Occurred': None, + 'HPy_FatalError': None, + 'HPy_Add': 'PyNumber_Add', + 'HPy_Subtract': 'PyNumber_Subtract', + 'HPy_Multiply': 'PyNumber_Multiply', + 'HPy_MatrixMultiply': 'PyNumber_MatrixMultiply', + 'HPy_FloorDivide': 'PyNumber_FloorDivide', + 'HPy_TrueDivide': 'PyNumber_TrueDivide', + 'HPy_Remainder': 'PyNumber_Remainder', + 'HPy_Divmod': 'PyNumber_Divmod', + 'HPy_Power': 'PyNumber_Power', + 'HPy_Negative': 'PyNumber_Negative', + 'HPy_Positive': 'PyNumber_Positive', + 'HPy_Absolute': 'PyNumber_Absolute', + 'HPy_Invert': 'PyNumber_Invert', + 'HPy_Lshift': 'PyNumber_Lshift', + 'HPy_Rshift': 'PyNumber_Rshift', + 'HPy_And': 'PyNumber_And', + 'HPy_Xor': 'PyNumber_Xor', + 'HPy_Or': 'PyNumber_Or', + 'HPy_Index': 'PyNumber_Index', + 'HPy_Long': 'PyNumber_Long', + 'HPy_Float': 'PyNumber_Float', + 'HPy_InPlaceAdd': 'PyNumber_InPlaceAdd', + 'HPy_InPlaceSubtract': 'PyNumber_InPlaceSubtract', + 'HPy_InPlaceMultiply': 'PyNumber_InPlaceMultiply', + 'HPy_InPlaceMatrixMultiply': 'PyNumber_InPlaceMatrixMultiply', + 'HPy_InPlaceFloorDivide': 'PyNumber_InPlaceFloorDivide', + 'HPy_InPlaceTrueDivide': 'PyNumber_InPlaceTrueDivide', + 'HPy_InPlaceRemainder': 'PyNumber_InPlaceRemainder', + 'HPy_InPlacePower': 'PyNumber_InPlacePower', + 'HPy_InPlaceLshift': 'PyNumber_InPlaceLshift', + 'HPy_InPlaceRshift': 'PyNumber_InPlaceRshift', + 'HPy_InPlaceAnd': 'PyNumber_InPlaceAnd', + 'HPy_InPlaceXor': 'PyNumber_InPlaceXor', + 'HPy_InPlaceOr': 'PyNumber_InPlaceOr', + '_HPy_New': None, + 'HPyType_FromSpec': None, + 'HPyType_GenericNew': None, + 'HPy_Repr': 'PyObject_Repr', + 'HPy_Str': 'PyObject_Str', + 'HPy_ASCII': 'PyObject_ASCII', + 'HPy_Bytes': 'PyObject_Bytes', + 'HPy_IsTrue': 'PyObject_IsTrue', + 'HPy_RichCompare': 'PyObject_RichCompare', + 'HPy_RichCompareBool': 'PyObject_RichCompareBool', + 'HPy_Hash': 'PyObject_Hash', + 'HPyIter_Next': 'PyIter_Next', + 'HPyIter_Check': 'PyIter_Check', + 'HPyListBuilder_New': None, + 'HPyListBuilder_Set': None, + 'HPyListBuilder_Build': None, + 'HPyListBuilder_Cancel': None, + 'HPyTuple_FromArray': None, + 'HPyTupleBuilder_New': None, + 'HPyTupleBuilder_Set': None, + 'HPyTupleBuilder_Build': None, + 'HPyTupleBuilder_Cancel': None, + 'HPyTracker_New': None, + 'HPyTracker_Add': None, + 'HPyTracker_ForgetAll': None, + 'HPyTracker_Close': None, + '_HPy_Dump': None, + 'HPy_Type': None, + 'HPy_TypeCheck': None, + 'HPy_Is': None, + 'HPyBytes_FromStringAndSize': None, + 'HPy_LeavePythonExecution': 'PyEval_SaveThread', + 'HPy_ReenterPythonExecution': 'PyEval_RestoreThread', + 'HPyGlobal_Load': None, + 'HPyGlobal_Store': None, + 'HPyCapsule_New': None, + 'HPyCapsule_Get': None, + 'HPyCapsule_Set': None, + 'HPyLong_FromInt32_t': None, + 'HPyLong_FromUInt32_t': None, + 'HPyLong_FromInt64_t': None, + 'HPyLong_FromUInt64_t': None, + 'HPyLong_AsInt32_t': None, + 'HPyLong_AsUInt32_t': None, + 'HPyLong_AsUInt32_tMask': None, + 'HPyLong_AsInt64_t': None, + 'HPyLong_AsUInt64_t': None, + 'HPyLong_AsUInt64_tMask': None, + 'HPyBool_FromBool': 'PyBool_FromLong', + 'HPy_Compile_s': None, + 'HPy_EvalCode': 'PyEval_EvalCode', + 'HPyContextVar_Get': None, + 'HPyType_GetName': None, + 'HPyType_IsSubtype': None, + 'HPy_SetCallFunction': None, +} + +################################################################################ +# Configuration for auto-generating docs # +################################################################################ + +# A manual mapping of between CPython C API functions and HPy API functions. +# Most of the mapping will be generated automatically from 'public_api.h' if an +# HPy API function is not a special case (see 'conf.py'). However, in some +# cases, it might be that we have inline helper functions or something similar +# that map to a CPython C API function which cannot be determined automatically. +# In those cases, the mapping can be manually specified here. Also, manual +# mapping will always take precedence over automatically derived mappings. +DOC_MANUAL_API_MAPPING = { + # key = C API function name + # value = HPy API function name + 'Py_FatalError': 'HPy_FatalError', + 'PyContextVar_Get': 'HPyContextVar_Get', + 'PyLong_FromLong': 'HPyLong_FromLong', + 'PyLong_FromLongLong': 'HPyLong_FromLongLong', + 'PyLong_FromUnsignedLong': 'HPyLong_FromUnsignedLong', + 'PyLong_FromUnsignedLongLong': 'HPyLong_FromUnsignedLongLong', + 'PyLong_AsLong': 'HPyLong_AsLong', + 'PyLong_AsLongLong': 'HPyLong_AsLongLong', + 'PyLong_AsUnsignedLong': 'HPyLong_AsUnsignedLong', + 'PyLong_AsUnsignedLongMask': 'HPyLong_AsUnsignedLongMask', + 'PyLong_AsUnsignedLongLong': 'HPyLong_AsUnsignedLongLong', + 'PyLong_AsUnsignedLongLongMask': 'HPyLong_AsUnsignedLongLongMask', + 'PyBool_FromLong': 'HPyBool_FromLong', + 'PyObject_TypeCheck': 'HPy_TypeCheck', + 'PySlice_AdjustIndices': 'HPySlice_AdjustIndices', + 'PyType_IsSubtype': 'HPyType_IsSubtype', + 'PyObject_Call': 'HPy_CallTupleDict', + 'PyObject_Type': 'HPy_Type', + 'PyObject_Vectorcall': 'HPy_Call', + 'PyObject_VectorcallMethod': 'HPy_CallMethod', +} + +# Some C API functions are documented in very different pages. +DOC_C_API_PAGES_SPECIAL_CASES = { + 'Py_FatalError': 'sys', + 'PyEval_SaveThread': 'init', + 'PyEval_RestoreThread': 'init', + 'PyEval_EvalCode': 'veryhigh', + 'PyObject_Call': 'call', + 'PyObject_Vectorcall': 'call', + 'PyObject_VectorcallMethod': 'call', +} + +# We assume that, e.g., prefix 'PyLong_Something' belongs to 'longobject.c' and +# its documentation is in '.../3/c-api/long.html'. In some cases, the prefix +# maps to a different page and this can be specified here. E.g. +# 'PyErr_Something' is documented in page '.../3/c-api/exceptions.html' +DOC_PREFIX_TABLE = { + 'err': 'exceptions', + 'contextvar': 'contextvars' +} \ No newline at end of file diff --git a/graalpython/hpy/hpy/tools/autogen/ctx.py b/graalpython/hpy/hpy/tools/autogen/ctx.py new file mode 100644 index 0000000000..4f569f1f35 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/ctx.py @@ -0,0 +1,106 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, maybe_make_void + + +class autogen_ctx_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_ctx.h' + + ## struct _HPyContext_s { + ## const char *name; + ## void *_private; + ## int abi_version; + ## HPy h_None; + ## ... + ## HPy (*ctx_Module_Create)(HPyContext *ctx, HPyModuleDef *def); + ## ... + ## } + + def generate(self): + # Put all declarations (variables and functions) into one list in order + # to be able to sort them by their given context index. + # We need to remember the output function per item (either + # 'declare_var' or 'declare_func'). So, we create tuples with structure + # '(decl, declare_*)'. + all_decls = [(x, self.declare_var) for x in self.api.variables] + all_decls += [(x, self.declare_func) for x in self.api.functions] + + # sort the list of all declaration by 'decl.ctx_index' + all_decls.sort(key=lambda x: x[0].ctx_index) + + lines = [] + w = lines.append + w('struct _HPyContext_s {') + w(' const char *name; // used just to make debugging and testing easier') + w(' void *_private; // used by implementations to store custom data') + w(' int abi_version;') + for var, cons in all_decls: + w(' %s;' % cons(var)) + w('};') + return '\n'.join(lines) + + def declare_var(self, var): + return toC(var.node) + + def declare_func(self, func): + # e.g. "HPy (*ctx_Module_Create)(HPyContext *ctx, HPyModuleDef *def)" + # + # turn the function declaration into a function POINTER declaration + newnode = deepcopy(func.node) + newnode.type = c_ast.PtrDecl(type=newnode.type, quals=[]) + maybe_make_void(func, newnode) + + # fix the name of the function pointer + typedecl = find_typedecl(newnode) + typedecl.declname = func.ctx_name() + return toC(newnode) + + +class autogen_ctx_def_h(AutoGenFile): + PATH = 'hpy/universal/src/autogen_ctx_def.h' + + ## struct _HPyContext_s g_universal_ctx = { + ## .name = "...", + ## ._private = NULL, + ## .abi_version = HPY_ABI_VERSION, + ## .h_None = {CONSTANT_H_NONE}, + ## ... + ## .ctx_Module_Create = &ctx_Module_Create, + ## ... + ## } + + def generate(self): + lines = [] + w = lines.append + w('struct _HPyContext_s g_universal_ctx = {') + w(' .name = "HPy Universal ABI (CPython backend)",') + w(' ._private = NULL,') + w(' .abi_version = HPY_ABI_VERSION,') + w(' /* h_None & co. are initialized by init_universal_ctx() */') + for func in self.api.functions: + w(' .%s = &%s,' % (func.ctx_name(), func.ctx_name())) + w('};') + return '\n'.join(lines) + + +class cpython_autogen_ctx_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_ctx.h' + + def generate(self): + # Put all variable declarations into a list in order + # to be able to sort them by their given context index. + var_decls = list(self.api.variables) + + # sort the list of var declaration by 'decl.ctx_index' + var_decls.sort(key=lambda x: x.ctx_index) + + lines = [] + w = lines.append + w('struct _HPyContext_s {') + w(' const char *name;') + w(' int abi_version;') + for var in var_decls: + w(' %s;' % toC(var.node)) + w('};') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/debug.py b/graalpython/hpy/hpy/tools/autogen/debug.py new file mode 100644 index 0000000000..eb8783ba0a --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/debug.py @@ -0,0 +1,228 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, get_context_return_type, \ + maybe_make_void, make_void, get_return_constant +from .hpyfunc import NO_CALL + + +class HPy_2_DHPy_Visitor(c_ast.NodeVisitor): + "Visitor which renames all HPy types to DHPy" + + def visit_IdentifierType(self, node): + if node.names == ['HPy']: + node.names = ['DHPy'] + + def visit_TypeDecl(self, node): + if node.declname == 'ctx': + node.declname = 'dctx' + self.generic_visit(node) + +def funcnode_with_new_name(node, name): + newnode = deepcopy(node) + typedecl = find_typedecl(newnode) + typedecl.declname = name + return newnode + +def get_debug_wrapper_node(func): + newnode = funcnode_with_new_name(func.node, 'debug_%s' % func.ctx_name()) + maybe_make_void(func, newnode) + # fix all the types + visitor = HPy_2_DHPy_Visitor() + visitor.visit(newnode) + return newnode + + +class autogen_debug_ctx_init_h(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_ctx_init.h' + + def generate(self): + lines = [] + w = lines.append + # emit the declarations for all the debug_ctx_* functions + for func in self.api.functions: + w(toC(get_debug_wrapper_node(func)) + ';') + w('') + w('static inline void debug_ctx_init_fields(HPyContext *dctx, HPyContext *uctx)') + w('{') + for var in self.api.variables: + name = var.name + w(f' dctx->{name} = DHPy_open_immortal(dctx, uctx->{name});') + for func in self.api.functions: + name = func.ctx_name() + w(f' dctx->{name} = &debug_{name};') + + w('}') + return '\n'.join(lines) + + +class autogen_debug_wrappers(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_wrappers.c' + + NO_WRAPPER = { + '_HPy_CallRealFunctionFromTrampoline', + 'HPy_Close', + 'HPyUnicode_AsUTF8AndSize', + 'HPyTuple_FromArray', + 'HPyType_GenericNew', + 'HPyType_FromSpec', + '_HPy_AsStruct_Legacy', + '_HPy_AsStruct_Object', + '_HPy_AsStruct_Type', + '_HPy_AsStruct_Long', + '_HPy_AsStruct_Float', + '_HPy_AsStruct_Unicode', + '_HPy_AsStruct_Tuple', + '_HPy_AsStruct_List', + '_HPy_AsStruct_Dict', + 'HPyTracker_New', + 'HPyTracker_Add', + 'HPyTracker_ForgetAll', + 'HPyTracker_Close', + 'HPyBytes_AsString', + 'HPyBytes_AS_STRING', + 'HPyTupleBuilder_New', + 'HPyTupleBuilder_Set', + 'HPyTupleBuilder_Build', + 'HPyTupleBuilder_Cancel', + 'HPyListBuilder_New', + 'HPyListBuilder_Set', + 'HPyListBuilder_Build', + 'HPyListBuilder_Cancel', + 'HPy_TypeCheck', + 'HPyContextVar_Get', + 'HPyType_GetName', + 'HPyType_IsSubtype', + 'HPyUnicode_Substring', + 'HPy_Call', + 'HPy_CallMethod', + } + + def generate(self): + lines = [] + w = lines.append + w('#include "debug_internal.h"') + w('') + for func in self.api.functions: + debug_wrapper = self.gen_debug_wrapper(func) + if debug_wrapper: + w(debug_wrapper) + w('') + return '\n'.join(lines) + + def gen_debug_wrapper(self, func): + if func.name in self.NO_WRAPPER: + return + # + assert not func.is_varargs() + node = get_debug_wrapper_node(func) + const_return = get_return_constant(func) + if const_return: + make_void(node) + signature = toC(node) + rettype = get_context_return_type(node, const_return) + # + def get_params_and_decls(): + lst = [] + decls = [] + for p in node.type.args.params: + if p.name == 'ctx': + lst.append('get_info(dctx)->uctx') + elif toC(p.type) == 'DHPy': + decls.append(' HPy dh_%s = DHPy_unwrap(dctx, %s);' % (p.name, p.name)) + lst.append('dh_%s' % p.name) + elif toC(p.type) in ('DHPy *', 'DHPy []'): + assert False, ('C type %s not supported, please write the wrapper ' + 'for %s by hand' % (toC(p.type), func.name)) + else: + lst.append(p.name) + return (', '.join(lst), '\n'.join(decls)) + (params, param_decls) = get_params_and_decls() + # + lines = [] + w = lines.append + w(signature) + w('{') + w(' if (!get_ctx_info(dctx)->is_valid) {') + w(' report_invalid_debug_context();') + w(' }') + if param_decls: + w(param_decls) + w(' get_ctx_info(dctx)->is_valid = false;') + if rettype == 'void': + w(f' {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + elif rettype == 'DHPy': + w(f' HPy universal_result = {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + w(' return DHPy_open(dctx, universal_result);') + else: + w(f' {rettype} universal_result = {func.name}({params});') + w(' get_ctx_info(dctx)->is_valid = true;') + w(' return universal_result;') + w('}') + return '\n'.join(lines) + + +class autogen_debug_ctx_call_i(AutoGenFile): + PATH = 'hpy/debug/src/autogen_debug_ctx_call.i' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_CALL: + continue + # + c_ret_type = toC(hpyfunc.return_type()) + args = ['next_dctx'] + dhpys = [] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + if toC(param.type) == 'HPy': + dhpys.append(pname) + args.append(f'dh_{pname}') + else: + args.append(f'a->{pname}') + args = ', '.join(args) + # + w(f' case HPyFunc_{NAME}: {{') + w(f' HPyFunc_{name} f = (HPyFunc_{name})func;') + w(f' _HPyFunc_args_{NAME} *a = (_HPyFunc_args_{NAME}*)args;') + for pname in dhpys: + w(f' DHPy dh_{pname} = _py2dh(dctx, a->{pname});') + # + w(' HPyContext *next_dctx = _switch_to_next_dctx_from_cache(dctx);') + w(' if (next_dctx == NULL) {') + if c_ret_type == 'HPy': + w(' a->result = NULL;') + elif c_ret_type == 'int' or c_ret_type == 'HPy_ssize_t' or c_ret_type == 'HPy_hash_t': + w(' a->result = -1;') + else: + assert c_ret_type == 'void', c_ret_type + " not implemented" + w(' return;') + w(' }') + # + if c_ret_type == 'void': + w(f' f({args});') + elif c_ret_type == 'HPy': + w(f' DHPy dh_result = f({args});') + else: + w(f' a->result = f({args});') + # + w(' _switch_back_to_original_dctx(dctx, next_dctx);') + # + for pname in dhpys: + w(f' DHPy_close_and_check(dctx, dh_{pname});') + # + if c_ret_type == 'HPy': + w(f' a->result = _dh2py(dctx, dh_result);') + w(f' DHPy_close(dctx, dh_result);') + # + w(f' return;') + w(f' }}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/doc.py b/graalpython/hpy/hpy/tools/autogen/doc.py new file mode 100644 index 0000000000..9824768866 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/doc.py @@ -0,0 +1,173 @@ +import textwrap +import re + +from .autogenfile import AutoGenFile +from .parse import toC +from .ctx import autogen_ctx_h +from .conf import (DOC_C_API_PAGES_SPECIAL_CASES, + DOC_MANUAL_API_MAPPING, + DOC_PREFIX_TABLE) + + +CTX_NAME = '_HPyContext_s' + +RST_DISCLAIMER = """ +.. note: DO NOT EDIT THIS FILE! + This file is automatically generated by {clsname} + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen +""" + +class AutoGenRstFile(AutoGenFile): + LANGUAGE = 'rst' + PATH = None + DISCLAIMER = None + + def __init__(self, api): + if self.DISCLAIMER is None and self.LANGUAGE == 'rst': + self.DISCLAIMER = RST_DISCLAIMER + self.api = api + +class autogen_function_index(AutoGenRstFile): + PATH = 'docs/api-reference/function-index.rst' + LANGUAGE = 'rst' + + def generate(self): + lines = [] + w = lines.append + w('HPy Core API Function Index') + w('###########################') + w('') + functions = list(self.api.functions) + # sort the list of functions by 'func.name' + functions.sort(key=lambda x: x.name) + for func in functions: + if func.name[0] != '_': + w(f'* :c:func:`{func.name}`') + return '\n'.join(lines) + + +class AutoGenFilePart: + PATH = None + BEGIN_MARKER = None + END_MARKER = None + + def __init__(self, api): + self.api = api + + def generate(self, old): + raise NotImplementedError + + def write(self, root): + if not self.BEGIN_MARKER or not self.END_MARKER: + raise RuntimeError("missing BEGIN_MARKER or END_MARKER") + n_begin = len(self.BEGIN_MARKER) + with root.join(self.PATH).open('r', encoding='utf-8') as f: + content = f.read() + start = content.find(self.BEGIN_MARKER) + if start < 0: + raise RuntimeError(f'begin marker "{self.BEGIN_MARKER}" not found' + f'in file {self.PATH}') + end = content.find(self.END_MARKER, start + n_begin) + if end < 0: + raise RuntimeError(f'end marker "{self.END_MARKER}" not found in' + f'file {self.PATH}') + new_content = self.generate(content[(start+n_begin):end]) + with root.join(self.PATH).open('w', encoding='utf-8') as f: + f.write(content[:start + n_begin] + new_content + content[end:]) + + +GROUP_PATTERN = re.compile('Py(\\w+)_.*') + +class autogen_doc_api_mapping(AutoGenFilePart): + PATH = 'docs/porting-guide.rst' + BEGIN_MARKER = '.. mark: BEGIN API MAPPING\n' + END_MARKER = '.. mark: END API MAPPING\n' + + def _get_page(self, cpython_fun_name): + if cpython_fun_name in DOC_C_API_PAGES_SPECIAL_CASES: + return DOC_C_API_PAGES_SPECIAL_CASES[cpython_fun_name] + '.html' + + first_underscore = cpython_fun_name.find('_') + if cpython_fun_name.startswith('Py') and first_underscore != -1: + prefix = cpython_fun_name[2:first_underscore].lower() + return DOC_PREFIX_TABLE.get(prefix, prefix) + '.html' + else: + return 'abstract.html' + + def generate(self, old_content): + table_directive = '.. _table-mapping:\n.. table:: Safe API function mapping\n' + assert old_content.strip().startswith(table_directive) + + lines = [] + w = lines.append + w(':widths: auto') + w('') + mapping = {x.cpython_name: x.name for x in self.api.functions if x.cpython_name} + mapping.update(DOC_MANUAL_API_MAPPING) + max_width0 = 0 + max_width1 = 0 + rows = [] + cpy_functions = list(mapping.keys()) + # sort the list of functions by 'cpython_name' + cpy_functions.sort() + for cpy_func in cpy_functions: + assert cpy_func + page = self._get_page(cpy_func) + col0 = f'`{cpy_func} `_' + col1 = f':c:func:`{mapping[cpy_func]}`' + rows.append((col0, col1)) + max_width0 = max(max_width0, len(col0)) + max_width1 = max(max_width1, len(col1)) + + sep = '=' * max_width0 + ' ' + '=' * max_width1 + w(sep) + w('C API function'.ljust(max_width0) + ' HPY API function') + w(sep) + for row in rows: + w(f'{row[0].ljust(max_width0)} {row[1]}') + w(sep) + w('') + return table_directive + textwrap.indent('\n'.join(lines), ' ' * 4) + + +class autogen_hpy_ctx(AutoGenRstFile): + PATH = 'docs/api-reference/hpy-ctx.rst' + + def generate(self): + lines = [] + w = lines.append + w(textwrap.dedent( + ''' + HPy Context + =========== + + The ``HPyContext`` structure is also part of the API since it provides handles + for built-in objects. For a high-level description of the context, please also + read :ref:`api:hpycontext`. + + .. warning:: It is fine to use handles from the context (e.g. ``ctx->h_None``) + but it is **STRONGLY** discouraged to directly call any context function. + This is because, for example, when compiling for :term:`CPython ABI`, the + context functions won't be used. + ''')) + # Put all variable declarations into a list in order + # to be able to sort them by their given context index. + var_decls = list(self.api.variables) + + # sort the list of var declaration by 'decl.ctx_index' + var_decls.sort(key=lambda x: x.ctx_index) + + w(f'.. c:struct:: {CTX_NAME}') + w(f' :module: {autogen_ctx_h.PATH}') + w('') + w(f' .. c:member:: const char *{CTX_NAME}.name') + w('') + w(f' .. c:member:: int {CTX_NAME}.abi_version') + for var in var_decls: + w('') + w(f' .. c:member:: {toC(var.node)}') + return '\n'.join(lines) + diff --git a/graalpython/hpy/hpy/tools/autogen/hpyfunc.py b/graalpython/hpy/hpy/tools/autogen/hpyfunc.py new file mode 100644 index 0000000000..1e233277f2 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/hpyfunc.py @@ -0,0 +1,204 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl + +NO_CALL = ('VARARGS', 'KEYWORDS', 'INITPROC', 'DESTROYFUNC', + 'GETBUFFERPROC', 'RELEASEBUFFERPROC', 'TRAVERSEPROC', 'MOD_CREATE', + 'VECTORCALLFUNC', 'NEWFUNC') +NO_TRAMPOLINE = NO_CALL + ('RICHCMPFUNC',) + +# This is a list of type that can automatically be converted from Python to HPy +# and vice versa. +AUTO_CONVERSION_TYPES = ('HPy', 'HPy_ssize_t', 'void *', 'int', 'char *', + 'HPy_hash_t', 'HPy_RichCmpOp', 'void') + +class autogen_hpyfunc_declare_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/autogen_hpyfunc_declare.h' + + ## #define _HPyFunc_DECLARE_HPyFunc_NOARGS(SYM) \ + ## static HPy SYM(HPyContext *ctx, HPy self) + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + # declare a function named 'SYM' of the appropriate type + funcdecl = hpyfunc.node.type.type + symdecl = deepcopy(funcdecl) + if isinstance(symdecl.type, c_ast.PtrDecl): + symdecl.type = symdecl.type.type + symdecl.type.declname = 'SYM' + symdecl = toC(symdecl) + # + # generate a macro emitting 'symdecl' + name = hpyfunc.base_name().upper() + w(f'#define _HPyFunc_DECLARE_HPyFunc_{name}(SYM) static {symdecl}') + w('') + + for hpyfunc in self.api.hpyfunc_typedefs: + # generate the typedef for HPyFunc_{base_name} + w(f'{toC(hpyfunc.node)};') + + return '\n'.join(lines) + + +def hpy_to_cpy(declnode): + if toC(declnode.type) == 'HPy': + declnode = deepcopy(declnode) + declnode.type.type.names = ['cpy_PyObject'] + declnode.type = c_ast.PtrDecl(type=declnode.type, quals=[]) + return declnode + + +class autogen_hpyfunc_trampoline_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_hpyfunc_trampolines.h' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + NAME = hpyfunc.base_name().upper() + if NAME in NO_TRAMPOLINE: + continue + # + tramp_node = deepcopy(hpyfunc.node.type.type) + if isinstance(tramp_node.type, c_ast.PtrDecl): + tramp_node.type = tramp_node.type.type + tramp_node.type.declname = 'SYM' + tramp_node = hpy_to_cpy(tramp_node) + assert toC(tramp_node.args.params[0].type) in ['void', 'HPyContext *'] + tramp_node.args.params = [hpy_to_cpy(p) + for p in tramp_node.args.params[1:]] + for i, param in enumerate(tramp_node.args.params): + typedecl = find_typedecl(param.type) + if typedecl.declname is None: + param.name = 'arg%d' % i + typedecl.declname = 'arg%d' % i + arg_names = [param.name for param in tramp_node.args.params] + arg_names = ', '.join(arg_names) + # + # generate the struct that will contain all parameters + w(f'typedef struct {{') + for param in tramp_node.args.params: + w(f' {toC(param)};') + if toC(tramp_node.type) != 'void': + w(f' {toC(tramp_node.type)} result;') + w(f'}} _HPyFunc_args_{NAME};') + w('') + # + # generate the trampoline itself + w(f'#define _HPyFunc_TRAMPOLINE_HPyFunc_{NAME}(SYM, IMPL) \\') + w(f' static {toC(tramp_node)} \\') + w(f' {{ \\') + w(f' _HPyFunc_args_{NAME} a = {{ {arg_names} }}; \\') + w(f' _HPy_CallRealFunctionFromTrampoline( \\') + w(f' _ctx_for_trampolines, HPyFunc_{NAME}, (HPyCFunction)IMPL, &a); \\') + if toC(tramp_node.type) == 'void': + w(f' return; \\') + else: + w(f' return a.result; \\') + w(f' }}') + w('') + return '\n'.join(lines) + + +def _py2h(type): + if type == 'HPy': + return '_py2h' + elif type in AUTO_CONVERSION_TYPES: + return '' + raise TypeError(f'cannot generate automatic conversion for type \'{type}\'') + + +def _h2py(type): + if type == 'HPy': + return '_h2py' + elif type in AUTO_CONVERSION_TYPES: + return '' + raise TypeError(f'cannot generate automatic conversion for type \'{type}\'') + + +class autogen_ctx_call_i(AutoGenFile): + PATH = 'hpy/universal/src/autogen_ctx_call.i' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_CALL: + continue + # + c_ret_type = toC(hpyfunc.return_type()) + args = ['ctx'] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + args.append(f'{_py2h(toC(param.type))}(a->{pname})') + args = ', '.join(args) + # + w(f' case HPyFunc_{NAME}: {{') + w(f' HPyFunc_{name} f = (HPyFunc_{name})func;') + w(f' _HPyFunc_args_{NAME} *a = (_HPyFunc_args_{NAME}*)args;') + if c_ret_type == 'void': + w(f' f({args});') + else: + w(f' a->result = {_h2py(c_ret_type)}(f({args}));') + w(f' return;') + w(f' }}') + return '\n'.join(lines) + + +class autogen_cpython_hpyfunc_trampoline_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_hpyfunc_trampolines.h' + + def generate(self): + lines = [] + w = lines.append + for hpyfunc in self.api.hpyfunc_typedefs: + name = hpyfunc.base_name() + NAME = name.upper() + if NAME in NO_TRAMPOLINE: + continue + # + tramp_node = deepcopy(hpyfunc.node.type.type) + if isinstance(tramp_node.type, c_ast.PtrDecl): + tramp_node.type = tramp_node.type.type + tramp_node.type.declname = 'SYM' + tramp_node = hpy_to_cpy(tramp_node) + tramp_node.args.params = [hpy_to_cpy(p) + for p in tramp_node.args.params[1:]] + for i, param in enumerate(tramp_node.args.params): + typedecl = find_typedecl(param.type) + if typedecl.declname is None: + param.name = 'arg%d' % i + typedecl.declname = 'arg%d' % i + + result = _h2py(toC(hpyfunc.return_type())) + args = ['_HPyGetContext()'] + func_ptr_ret_type = toC(hpyfunc.return_type()) + func_ptr_sig = ['HPyContext *'] + for i, param in enumerate(hpyfunc.params()[1:]): + pname = param.name + if pname is None: + pname = 'arg%d' % i + func_ptr_sig.append(toC(param.type)) + args.append(f'{_py2h(toC(param.type))}({pname})') + args = ', '.join(args) + func_ptr_sig = ', '.join(func_ptr_sig) + # + w(f'typedef {func_ptr_ret_type} (*_HPyCFunction_{NAME})({func_ptr_sig});') + w(f'#define _HPyFunc_TRAMPOLINE_HPyFunc_{NAME}(SYM, IMPL) \\') + w(f' static {toC(tramp_node)} \\') + w(f' {{ \\') + w(f' _HPyCFunction_{NAME} func = (_HPyCFunction_{NAME})IMPL; \\') + if toC(tramp_node.type) == 'void': + w(f' func({args}); \\') + w(f' return; \\') + else: + w(f' return {result}(func({args})); \\') + w(f' }}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/hpyslot.py b/graalpython/hpy/hpy/tools/autogen/hpyslot.py new file mode 100644 index 0000000000..3eff4d81ca --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/hpyslot.py @@ -0,0 +1,18 @@ +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC + +class autogen_hpyslot_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/autogen_hpyslot.h' + + def generate(self): + lines = [] + w = lines.append + w('typedef enum {') + for slot in self.api.hpyslots: + w(f' {slot.name} = {slot.value},') + w('} HPySlot_Slot;') + w('') + for slot in self.api.hpyslots: + w(f'#define _HPySlot_SIG__{slot.name} {slot.hpyfunc}') + return '\n'.join(lines) diff --git a/graalpython/hpy/hpy/tools/autogen/parse.py b/graalpython/hpy/hpy/tools/autogen/parse.py new file mode 100644 index 0000000000..6615eb1d50 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/parse.py @@ -0,0 +1,278 @@ +from copy import deepcopy +import sys +import attr +import re +import py +import pycparser +import shutil +from pycparser import c_ast +from pycparser.c_generator import CGenerator +from sysconfig import get_config_var +from .conf import SPECIAL_CASES, RETURN_CONSTANT + +PUBLIC_API_H = py.path.local(__file__).dirpath('public_api.h') +CURRENT_DIR = py.path.local(__file__).dirpath() +AUTOGEN_H = py.path.local(__file__).dirpath('autogen.h') + + +def toC(node): + return toC.gen.visit(node) +toC.gen = CGenerator() + +def find_typedecl(node): + while not isinstance(node, c_ast.TypeDecl): + node = node.type + return node + +def get_context_return_type(func_node, const_return): + return 'void' if const_return else toC(func_node.type.type) + +def make_void(func_node): + voidid = c_ast.IdentifierType(names=['void']) + func_node.type.type.type = c_ast.TypeDecl(declname='void', quals=[], align=[], type=voidid) + +def get_return_constant(func): + return RETURN_CONSTANT.get(func.node.name) + +def maybe_make_void(func, node): + if RETURN_CONSTANT.get(func.node.name): + make_void(node) + +@attr.s +class Function: + _BASE_NAME = re.compile(r'^_?HPy_?') + + name = attr.ib() + cpython_name = attr.ib() + ctx_index = attr.ib() + node = attr.ib(repr=False) + + def base_name(self): + return self._BASE_NAME.sub('', self.name) + + def ctx_name(self): + # e.g. "ctx_Module_Create" + return self._BASE_NAME.sub(r'ctx_', self.name) + + def is_varargs(self): + return (len(self.node.type.args.params) > 0 and + isinstance(self.node.type.args.params[-1], c_ast.EllipsisParam)) + + +@attr.s +class GlobalVar: + name = attr.ib() + ctx_index = attr.ib() + node = attr.ib(repr=False) + + def ctx_name(self): + return self.name + + +@attr.s +class HPyFunc: + _BASE_NAME = re.compile(r'^HPyFunc_?') + + name = attr.ib() + node = attr.ib(repr=False) + + def base_name(self): + return self._BASE_NAME.sub('', self.name) + + def params(self): + return self.node.type.type.args.params + + def return_type(self): + return self.node.type.type.type + +@attr.s +class HPySlot: + # represent a declaration contained inside enum HPySlot_Slot, such as: + # HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC) + + name = attr.ib() # "HPy_nb_add" + value = attr.ib() # "7" + hpyfunc = attr.ib() # "HPyFunc_BINARYFUNC" + node = attr.ib(repr=False) + + +class HPyAPIVisitor(pycparser.c_ast.NodeVisitor): + def __init__(self, api, convert_name): + self.api = api + self.convert_name = convert_name + self.cur_index = -1 + self.all_indices = [] + + def _consume_ctx_index(self): + idx = self.cur_index + self.all_indices.append(idx) + self.cur_index = -1 + return idx + + def verify_context_indices(self): + """ + Verifies if context indices are without gaps. This function raises an + assertion error if not. + For example: + [0, 1, 2, 3] is valid + [0, 1, 3] is invalid + """ + self.all_indices.sort() + for i in range(1, len(self.all_indices)): + prev = self.all_indices[i-1] + cur = self.all_indices[i] + assert prev + 1 == cur, \ + "context indices have gaps: %s -> %s" % (prev, cur) + + def _is_function_ptr(self, node): + return (isinstance(node, c_ast.PtrDecl) and + isinstance(node.type, c_ast.FuncDecl)) + + def visit_Decl(self, node): + if isinstance(node.type, c_ast.FuncDecl): + self._visit_function(node) + elif isinstance(node.type, c_ast.TypeDecl): + self._visit_global_var(node) + + def visit_Typedef(self, node): + # find only typedefs to function pointers whose name starts by HPyFunc_ + if node.name.startswith('HPyFunc_') and self._is_function_ptr(node.type): + self._visit_hpyfunc_typedef(node) + elif node.name == 'HPySlot_Slot': + self._visit_hpyslot_slot(node) + + def visit_Pragma(self, node): + parts = node.string.split('=') + if len(parts) != 2: + raise ValueError('invalid pragma: %s' % node) + self.cur_index = int(parts[1]) + + def _visit_function(self, node): + name = node.name + if not name.startswith('HPy') and not name.startswith('_HPy'): + print('WARNING: Ignoring non-hpy declaration: %s' % name) + return + for p in node.type.args.params: + if hasattr(p, 'name') and p.name is None: + raise ValueError("non-named argument in declaration of %s" % + name) + cpy_name = self.convert_name(name) + idx = self._consume_ctx_index() + if idx == -1: + raise ValueError('missing context index for %s' % name) + func = Function(name, cpy_name, idx, node) + self.api.functions.append(func) + + def _visit_global_var(self, node): + name = node.name + if not name.startswith('h_'): + print('WARNING: Ignoring non-hpy variable declaration: %s' % name) + return + assert toC(node.type.type) == "HPy" + idx = self._consume_ctx_index() + if idx == -1: + raise ValueError('missing context index for %s' % name) + var = GlobalVar(name, idx, node) + self.api.variables.append(var) + + def _visit_hpyfunc_typedef(self, node): + hpyfunc = HPyFunc(node.name, node) + self.api.hpyfunc_typedefs.append(hpyfunc) + + def _visit_hpyslot_slot(self, node): + for e in node.type.type.values.enumerators: + call = e.value + assert isinstance(call, c_ast.FuncCall) and call.name.name == 'SLOT' + assert len(call.args.exprs) == 2 + const_value, id_hpyfunc = call.args.exprs + assert isinstance(const_value, c_ast.Constant) and const_value.type == 'int' + assert isinstance(id_hpyfunc, c_ast.ID) + value = const_value.value + hpyfunc = id_hpyfunc.name + self.api.hpyslots.append(HPySlot(e.name, value, hpyfunc, e)) + + +def convert_name(hpy_name): + if hpy_name in SPECIAL_CASES: + return SPECIAL_CASES[hpy_name] + return re.sub(r'^_?HPy_?', 'Py', hpy_name) + + +class HPyAPI: + _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) + + def __init__(self, filename): + cpp_cmd = get_config_var('CC') + if cpp_cmd: + cpp_cmd = cpp_cmd.split(' ') + elif sys.platform == 'win32': + cpp_cmd = [shutil.which("cl.exe")] + if sys.platform == 'win32': + cpp_cmd += ['/E', '/I%s' % CURRENT_DIR] + else: + cpp_cmd += ['-E', '-I%s' % CURRENT_DIR] + + msvc = "cl.exe" in cpp_cmd[0].casefold() + + csource = pycparser.preprocess_file(filename, + cpp_path=str(cpp_cmd[0]), + cpp_args=cpp_cmd[1:]) + + # MSVC preprocesses _Pragma(foo) to __pragma(foo), + # but cparser needs to see a #pragma, not __pragma. + if msvc: + csource = re.sub(r'__pragma\(([^)]+)\)', r'#pragma \1\n', csource) + + # Remove comments. NOTE: this assumes that comments are never inside + # string literals, but there shouldn't be any here. + def replace_keeping_newlines(m): + return ' ' + m.group().count('\n') * '\n' + csource = self._r_comment.sub(replace_keeping_newlines, csource) + self.ast = pycparser.CParser().parse(csource) + ## print(); self.ast.show() + self.collect_declarations() + + @classmethod + def parse(cls, filename): + return cls(filename) + + def get_func(self, name): + return self._lookup(name, self.functions) + + def get_var(self, name): + return self._lookup(name, self.variables) + + def get_hpyfunc_typedef(self, name): + return self._lookup(name, self.hpyfunc_typedefs) + + def get_slot(self, name): + return self._lookup(name, self.hpyslots) + + def _lookup(self, name, collection): + for x in collection: + if x.name == name: + return x + raise KeyError(name) + + def collect_declarations(self): + self.functions = [] + self.variables = [] + self.hpyfunc_typedefs = [] + self.hpyslots = [] + v = HPyAPIVisitor(self, convert_name) + v.visit(self.ast) + + v.verify_context_indices() + + # Sort lists such that the generated files are deterministic. + # List elements are either 'Function', 'GlobalVar', or 'HPyFunc'. All + # of them have a 'node' attribute and the nodes have a 'coord' attr + # that provides the line and column number. We use that to sort. + def node_key(e): + coord = e.node.coord + return coord.line, coord.column + self.functions.sort(key=node_key) + self.variables.sort(key=node_key) + self.hpyfunc_typedefs.sort(key=node_key) + self.hpyslots.sort(key=node_key) diff --git a/graalpython/hpy/hpy/tools/autogen/public_api.h b/graalpython/hpy/hpy/tools/autogen/public_api.h new file mode 100644 index 0000000000..b081d943a8 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/public_api.h @@ -0,0 +1,1557 @@ +/* HPy public API */ + +/* + * IMPORTANT: In order to ensure backwards compatibility of HPyContext, it is + * necessary to define the order of the context members. To do so, use macro + * 'HPy_ID(idx)' for context handles and functions. When adding members, it + * doesn't matter where they are located in this file. It's just important that + * the maximum context index is incremented by exactly one. + */ + +#ifdef AUTOGEN + +/* Constants */ +HPy_ID(0) HPy h_None; +HPy_ID(1) HPy h_True; +HPy_ID(2) HPy h_False; +HPy_ID(3) HPy h_NotImplemented; +HPy_ID(4) HPy h_Ellipsis; + +/* Exceptions */ +HPy_ID(5) HPy h_BaseException; +HPy_ID(6) HPy h_Exception; +HPy_ID(7) HPy h_StopAsyncIteration; +HPy_ID(8) HPy h_StopIteration; +HPy_ID(9) HPy h_GeneratorExit; +HPy_ID(10) HPy h_ArithmeticError; +HPy_ID(11) HPy h_LookupError; +HPy_ID(12) HPy h_AssertionError; +HPy_ID(13) HPy h_AttributeError; +HPy_ID(14) HPy h_BufferError; +HPy_ID(15) HPy h_EOFError; +HPy_ID(16) HPy h_FloatingPointError; +HPy_ID(17) HPy h_OSError; +HPy_ID(18) HPy h_ImportError; +HPy_ID(19) HPy h_ModuleNotFoundError; +HPy_ID(20) HPy h_IndexError; +HPy_ID(21) HPy h_KeyError; +HPy_ID(22) HPy h_KeyboardInterrupt; +HPy_ID(23) HPy h_MemoryError; +HPy_ID(24) HPy h_NameError; +HPy_ID(25) HPy h_OverflowError; +HPy_ID(26) HPy h_RuntimeError; +HPy_ID(27) HPy h_RecursionError; +HPy_ID(28) HPy h_NotImplementedError; +HPy_ID(29) HPy h_SyntaxError; +HPy_ID(30) HPy h_IndentationError; +HPy_ID(31) HPy h_TabError; +HPy_ID(32) HPy h_ReferenceError; +HPy_ID(33) HPy h_SystemError; +HPy_ID(34) HPy h_SystemExit; +HPy_ID(35) HPy h_TypeError; +HPy_ID(36) HPy h_UnboundLocalError; +HPy_ID(37) HPy h_UnicodeError; +HPy_ID(38) HPy h_UnicodeEncodeError; +HPy_ID(39) HPy h_UnicodeDecodeError; +HPy_ID(40) HPy h_UnicodeTranslateError; +HPy_ID(41) HPy h_ValueError; +HPy_ID(42) HPy h_ZeroDivisionError; +HPy_ID(43) HPy h_BlockingIOError; +HPy_ID(44) HPy h_BrokenPipeError; +HPy_ID(45) HPy h_ChildProcessError; +HPy_ID(46) HPy h_ConnectionError; +HPy_ID(47) HPy h_ConnectionAbortedError; +HPy_ID(48) HPy h_ConnectionRefusedError; +HPy_ID(49) HPy h_ConnectionResetError; +HPy_ID(50) HPy h_FileExistsError; +HPy_ID(51) HPy h_FileNotFoundError; +HPy_ID(52) HPy h_InterruptedError; +HPy_ID(53) HPy h_IsADirectoryError; +HPy_ID(54) HPy h_NotADirectoryError; +HPy_ID(55) HPy h_PermissionError; +HPy_ID(56) HPy h_ProcessLookupError; +HPy_ID(57) HPy h_TimeoutError; +// EnvironmentError, IOError and WindowsError are intentionally omitted (they +// are all aliases of OSError since Python 3.3). + +/* Warnings */ +HPy_ID(58) HPy h_Warning; +HPy_ID(59) HPy h_UserWarning; +HPy_ID(60) HPy h_DeprecationWarning; +HPy_ID(61) HPy h_PendingDeprecationWarning; +HPy_ID(62) HPy h_SyntaxWarning; +HPy_ID(63) HPy h_RuntimeWarning; +HPy_ID(64) HPy h_FutureWarning; +HPy_ID(65) HPy h_ImportWarning; +HPy_ID(66) HPy h_UnicodeWarning; +HPy_ID(67) HPy h_BytesWarning; +HPy_ID(68) HPy h_ResourceWarning; + +/* Types */ +HPy_ID(69) HPy h_BaseObjectType; /* built-in 'object' */ +HPy_ID(70) HPy h_TypeType; /* built-in 'type' */ +HPy_ID(71) HPy h_BoolType; /* built-in 'bool' */ +HPy_ID(72) HPy h_LongType; /* built-in 'int' */ +HPy_ID(73) HPy h_FloatType; /* built-in 'float' */ +HPy_ID(74) HPy h_UnicodeType; /* built-in 'str' */ +HPy_ID(75) HPy h_TupleType; /* built-in 'tuple' */ +HPy_ID(76) HPy h_ListType; /* built-in 'list' */ +HPy_ID(238) HPy h_ComplexType; /* built-in 'complex' */ +HPy_ID(239) HPy h_BytesType; /* built-in 'bytes' */ +HPy_ID(240) HPy h_MemoryViewType; /* built-in 'memoryview' */ +HPy_ID(241) HPy h_CapsuleType; /* built-in 'capsule' */ +HPy_ID(242) HPy h_SliceType; /* built-in 'slice' */ +HPy_ID(263) HPy h_DictType; /* built-in 'dict' */ + +/* Reflection */ +HPy_ID(243) HPy h_Builtins; /* dict of builtins */ + +#endif + +HPy_ID(77) +HPy HPy_Dup(HPyContext *ctx, HPy h); +HPy_ID(78) +void HPy_Close(HPyContext *ctx, HPy h); + +HPy_ID(79) +HPy HPyLong_FromInt32_t(HPyContext *ctx, int32_t value); +HPy_ID(80) +HPy HPyLong_FromUInt32_t(HPyContext *ctx, uint32_t value); +HPy_ID(81) +HPy HPyLong_FromInt64_t(HPyContext *ctx, int64_t v); +HPy_ID(82) +HPy HPyLong_FromUInt64_t(HPyContext *ctx, uint64_t v); +HPy_ID(83) +HPy HPyLong_FromSize_t(HPyContext *ctx, size_t value); +HPy_ID(84) +HPy HPyLong_FromSsize_t(HPyContext *ctx, HPy_ssize_t value); + +HPy_ID(85) +int32_t HPyLong_AsInt32_t(HPyContext *ctx, HPy h); +HPy_ID(86) +uint32_t HPyLong_AsUInt32_t(HPyContext *ctx, HPy h); +HPy_ID(87) +uint32_t HPyLong_AsUInt32_tMask(HPyContext *ctx, HPy h); +HPy_ID(88) +int64_t HPyLong_AsInt64_t(HPyContext *ctx, HPy h); +HPy_ID(89) +uint64_t HPyLong_AsUInt64_t(HPyContext *ctx, HPy h); +HPy_ID(90) +uint64_t HPyLong_AsUInt64_tMask(HPyContext *ctx, HPy h); +HPy_ID(91) +size_t HPyLong_AsSize_t(HPyContext *ctx, HPy h); +HPy_ID(92) +HPy_ssize_t HPyLong_AsSsize_t(HPyContext *ctx, HPy h); +HPy_ID(93) +void* HPyLong_AsVoidPtr(HPyContext *ctx, HPy h); +HPy_ID(94) +double HPyLong_AsDouble(HPyContext *ctx, HPy h); + +HPy_ID(95) +HPy HPyFloat_FromDouble(HPyContext *ctx, double v); +HPy_ID(96) +double HPyFloat_AsDouble(HPyContext *ctx, HPy h); + +HPy_ID(97) +HPy HPyBool_FromBool(HPyContext *ctx, bool v); + + +/* abstract.h */ +HPy_ID(98) +HPy_ssize_t HPy_Length(HPyContext *ctx, HPy h); + +HPy_ID(99) +int HPyNumber_Check(HPyContext *ctx, HPy h); +HPy_ID(100) +HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(101) +HPy HPy_Subtract(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(102) +HPy HPy_Multiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(103) +HPy HPy_MatrixMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(104) +HPy HPy_FloorDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(105) +HPy HPy_TrueDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(106) +HPy HPy_Remainder(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(107) +HPy HPy_Divmod(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(108) +HPy HPy_Power(HPyContext *ctx, HPy h1, HPy h2, HPy h3); +HPy_ID(109) +HPy HPy_Negative(HPyContext *ctx, HPy h1); +HPy_ID(110) +HPy HPy_Positive(HPyContext *ctx, HPy h1); +HPy_ID(111) +HPy HPy_Absolute(HPyContext *ctx, HPy h1); +HPy_ID(112) +HPy HPy_Invert(HPyContext *ctx, HPy h1); +HPy_ID(113) +HPy HPy_Lshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(114) +HPy HPy_Rshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(115) +HPy HPy_And(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(116) +HPy HPy_Xor(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(117) +HPy HPy_Or(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(118) +HPy HPy_Index(HPyContext *ctx, HPy h1); +HPy_ID(119) +HPy HPy_Long(HPyContext *ctx, HPy h1); +HPy_ID(120) +HPy HPy_Float(HPyContext *ctx, HPy h1); + +HPy_ID(121) +HPy HPy_InPlaceAdd(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(122) +HPy HPy_InPlaceSubtract(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(123) +HPy HPy_InPlaceMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(124) +HPy HPy_InPlaceMatrixMultiply(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(125) +HPy HPy_InPlaceFloorDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(126) +HPy HPy_InPlaceTrueDivide(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(127) +HPy HPy_InPlaceRemainder(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(128) +HPy HPy_InPlacePower(HPyContext *ctx, HPy h1, HPy h2, HPy h3); +HPy_ID(129) +HPy HPy_InPlaceLshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(130) +HPy HPy_InPlaceRshift(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(131) +HPy HPy_InPlaceAnd(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(132) +HPy HPy_InPlaceXor(HPyContext *ctx, HPy h1, HPy h2); +HPy_ID(133) +HPy HPy_InPlaceOr(HPyContext *ctx, HPy h1, HPy h2); + +HPy_ID(134) +int HPyCallable_Check(HPyContext *ctx, HPy h); + +/** + * Call a Python object. + * + * :param ctx: + * The execution context. + * :param callable: + * A handle to the Python object to call (must not be ``HPy_NULL``). + * :param args: + * A handle to a tuple containing the positional arguments (must not be + * ``HPy_NULL`` but can, of course, be empty). + * :param kw: + * A handle to a Python dictionary containing the keyword arguments (may be + * ``HPy_NULL``). + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(135) +HPy HPy_CallTupleDict(HPyContext *ctx, HPy callable, HPy args, HPy kw); + +/** + * Call a Python object. + * + * :param ctx: + * The execution context. + * :param callable: + * A handle to the Python object to call (must not be ``HPy_NULL``). + * :param args: + * A pointer to an array of positional and keyword arguments. This argument + * must not be ``NULL`` if ``nargs > 0`` or + * ``HPy_Length(ctx, kwnames) > 0``. + * :param nargs: + * The number of positional arguments in ``args``. + * :param kwnames: + * A handle to the tuple of keyword argument names (may be ``HPy_NULL``). + * The values of the keyword arguments are also passed in ``args`` appended + * to the positional arguments. Argument ``nargs`` does not include the + * keyword argument count. + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(261) +HPy HPy_Call(HPyContext *ctx, HPy callable, const HPy *args, size_t nargs, HPy kwnames); + +/** + * Call a method of a Python object. + * + * :param ctx: + * The execution context. + * :param name: + * A handle to the name (a Unicode object) of the method. Must not be + * ``HPy_NULL``. + * :param args: + * A pointer to an array of the arguments. The receiver is ``args[0]``, and + * the positional and keyword arguments are starting at ``args[1]``. This + * argument must not be ``NULL`` since a receiver is always required. + * :param nargs: + * The number of positional arguments in ``args`` including the receiver at + * ``args[0]`` (therefore, ``nargs`` must be at least ``1``). + * :param kwnames: + * A handle to the tuple of keyword argument names (may be ``HPy_NULL``). + * The values of the keyword arguments are also passed in ``args`` appended + * to the positional arguments. Argument ``nargs`` does not include the + * keyword argument count. + * + * :returns: + * The result of the call on success, or ``HPy_NULL`` in case of an error. + */ +HPy_ID(262) +HPy HPy_CallMethod(HPyContext *ctx, HPy name, const HPy *args, size_t nargs, HPy kwnames); + +/** + * Return a new iterator for iterable object ``obj``. This is the equivalent + * of the Python expression ``iter(obj)``. + * + * :param ctx: + * The execution context. + * :param obj: + * An iterable Python object (must not be ``HPy_NULL``). If the object is + * not iterable, a ``TypeError`` will be raised. + * + * :returns: + * The new iterator, ``obj`` itself if it is already an iterator, or + * ``HPy_NULL`` on failure. + */ +HPy_ID(269) +HPy HPy_GetIter(HPyContext *ctx, HPy obj); + +/** + * Return the next value from iterator ``obj``. + * + * :param ctx: + * The execution context. + * :param obj: + * An iterator Python object (must not be ``HPy_NULL``). This can be + * verified with ``HPy_IterCheck``. Otherwise, the behavior is undefined. + * + * :returns: + * The new value in iterator ``obj``, or ``HPy_NULL`` on failure. If the + * iterator was exhausted normally, an exception will not be set. In + * case of some other error, one will be set. + */ +HPy_ID(270) +HPy HPyIter_Next(HPyContext *ctx, HPy obj); + +/** + * Tests if an object is an instance of a Python iterator. + * + * :param ctx: + * The execution context. + * :param obj: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``obj`` provides the ``Iterator`` protocol, and ``0`` + * otherwise. + */ +HPy_ID(271) +int HPyIter_Check(HPyContext *ctx, HPy obj); + +/* pyerrors.h */ +HPy_ID(136) +void HPy_FatalError(HPyContext *ctx, const char *message); +HPy_ID(137) +HPy HPyErr_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message); +HPy_ID(138) +HPy HPyErr_SetObject(HPyContext *ctx, HPy h_type, HPy h_value); + +/** + * Similar to :c:func:`HPyErr_SetFromErrnoWithFilenameObjects` but takes one + * filename (a C string) that will be decoded using + * :c:func:`HPyUnicode_DecodeFSDefault`. + * + * :param ctx: + * The execution context. + * :param h_type: + * The exception type to raise. + * :param filename_fsencoded: + * a filename; may be ``NULL`` + * + * :return: + * always returns ``HPy_NULL`` + */ +HPy_ID(139) +HPy HPyErr_SetFromErrnoWithFilename(HPyContext *ctx, HPy h_type, const char *filename_fsencoded); + +/** + * A convenience function to raise an exception when a C library function has + * returned an error and set the C variable ``errno``. It constructs an + * instance of the provided exception type ``h_type`` by calling + * ``h_type(errno, strerror(errno), filename1, 0, filename2)``. The exception + * instance is then raised. + * + * :param ctx: + * The execution context. + * :param h_type: + * The exception type to raise. + * :param filename1: + * A filename; may be ``HPy_NULL``. In the case of ``h_type`` is the + * ``OSError`` exception, this is used to define the filename attribute of + * the exception instance. + * :param filename2: + * another filename argument; may be ``HPy_NULL`` + * + * :return: + * always returns ``HPy_NULL`` + */ +HPy_ID(140) +HPy HPyErr_SetFromErrnoWithFilenameObjects(HPyContext *ctx, HPy h_type, HPy filename1, HPy filename2); +/* note: HPyErr_Occurred() returns a flag 0-or-1, instead of a 'PyObject *' */ +HPy_ID(141) +int HPyErr_Occurred(HPyContext *ctx); +HPy_ID(142) +int HPyErr_ExceptionMatches(HPyContext *ctx, HPy exc); +HPy_ID(143) +HPy HPyErr_NoMemory(HPyContext *ctx); +HPy_ID(144) +void HPyErr_Clear(HPyContext *ctx); +HPy_ID(145) +HPy HPyErr_NewException(HPyContext *ctx, const char *utf8_name, HPy base, HPy dict); +HPy_ID(146) +HPy HPyErr_NewExceptionWithDoc(HPyContext *ctx, const char *utf8_name, const char *utf8_doc, HPy base, HPy dict); +HPy_ID(147) +int HPyErr_WarnEx(HPyContext *ctx, HPy category, const char *utf8_message, HPy_ssize_t stack_level); +HPy_ID(148) +void HPyErr_WriteUnraisable(HPyContext *ctx, HPy obj); + +/* object.h */ +HPy_ID(149) +int HPy_IsTrue(HPyContext *ctx, HPy h); + +/** + * Create a type from a :c:struct:`HPyType_Spec` and an additional list of + * specification parameters. + * + * :param ctx: + * The execution context. + * :param spec: + * The type spec to use to create the type. + * :param params: + * A 0-terminated list of type specification parameters or ``NULL``. + * + * :returns: a handle of the created type on success, ``HPy_NULL`` on failure. + */ +HPy_ID(150) +HPy HPyType_FromSpec(HPyContext *ctx, HPyType_Spec *spec, + HPyType_SpecParam *params); +HPy_ID(151) +HPy HPyType_GenericNew(HPyContext *ctx, HPy type, const HPy *args, HPy_ssize_t nargs, HPy kw); + +HPy_ID(152) +HPy HPy_GetAttr(HPyContext *ctx, HPy obj, HPy name); +HPy_ID(153) +HPy HPy_GetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name); + +HPy_ID(154) +int HPy_HasAttr(HPyContext *ctx, HPy obj, HPy name); +HPy_ID(155) +int HPy_HasAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name); + +HPy_ID(156) +int HPy_SetAttr(HPyContext *ctx, HPy obj, HPy name, HPy value); +HPy_ID(157) +int HPy_SetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name, HPy value); + +HPy_ID(158) +HPy HPy_GetItem(HPyContext *ctx, HPy obj, HPy key); +HPy_ID(159) +HPy HPy_GetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); +HPy_ID(160) +HPy HPy_GetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key); + +/** + * Return the slice of sequence object ``obj`` between ``start`` and ``end``. + * This is the equivalent of the Python expression ``obj[start:end]``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * + * :returns: + * The requested slice or ``HPy_NULL`` on failure. + */ +HPy_ID(266) +HPy HPy_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + +HPy_ID(161) +int HPy_Contains(HPyContext *ctx, HPy container, HPy key); + +HPy_ID(162) +int HPy_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value); +HPy_ID(163) +int HPy_SetItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx, HPy value); +HPy_ID(164) +int HPy_SetItem_s(HPyContext *ctx, HPy obj, const char *utf8_key, HPy value); + +/** + * Assign the sequence object ``value`` to the slice in sequence object ``obj`` + * from ``start`` to ``end``. This is the equivalent of the Python statement + * ``obj[start:end] = value``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * :param value: + * The sequence object to assign (must not be ``HPy_NULL``). + * + * :returns: + * ``0`` on success; ``-1`` on failure + */ +HPy_ID(267) +int HPy_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value); + +HPy_ID(235) +int HPy_DelItem(HPyContext *ctx, HPy obj, HPy key); +HPy_ID(236) +int HPy_DelItem_i(HPyContext *ctx, HPy obj, HPy_ssize_t idx); +HPy_ID(237) +int HPy_DelItem_s(HPyContext *ctx, HPy obj, const char *utf8_key); + +/** + * Delete the slice of sequence object ``obj`` between ``start`` and ``end``. + * This is the equivalent of the Python statement ``del obj[start:end]``. + * + * :param ctx: + * The execution context. + * :param obj: + * A sliceable Python object (must not be ``HPy_NULL`` otherwise a + * ``SystemError`` will be raised). If the object is not sliceable, a + * ``TypeError`` will be raised. + * :param start: + * The start index (inclusive). + * :param end: + * The end index (exclusive). + * + * :returns: + * ``0`` on success; ``-1`` on failure + */ +HPy_ID(268) +int HPy_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end); + +/** + * Returns the type of the given object ``obj``. + * + * On failure, raises ``SystemError`` and returns ``HPy_NULL``. This is + * equivalent to the Python expression``type(obj)``. + * + * :param ctx: + * The execution context. + * :param obj: + * a Python object (must not be ``HPy_NULL``) + * + * :returns: + * The type of ``obj`` or ``HPy_NULL`` in case of errors. + */ +HPy_ID(165) +HPy HPy_Type(HPyContext *ctx, HPy obj); + +/** + * Checks if ``ob`` is an instance of ``type`` or any subtype of ``type``. + * + * :param ctx: + * The execution context. + * :param obj: + * a Python object (must not be ``HPy_NULL``) + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * Non-zero if object ``obj`` is an instance of type ``type`` or an instance + * of a subtype of ``type``, and ``0`` otherwise. + */ +HPy_ID(166) +int HPy_TypeCheck(HPyContext *ctx, HPy obj, HPy type); + +/** + * Return the type's name. + * + * Equivalent to getting the type's ``__name__`` attribute. If you want to + * retrieve the type's name as a handle that refers to a ``str``, then just use + * ``HPy_GetAttr_s(ctx, type, "__name__")``. + * + * :param ctx: + * The execution context. + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * The name of the type as C string (UTF-8 encoded) or ``NULL`` in case of + * an error. The returned pointer is read-only and guaranteed to be valid as + * long as the handle ``type`` is valid. + */ +HPy_ID(253) +const char *HPyType_GetName(HPyContext *ctx, HPy type); + +/** + * Checks if ``sub`` is a subtype of ``type``. + * + * This function only checks for actual subtypes, which means that + * ``__subclasscheck__()`` is not called on ``type``. + * + * :param ctx: + * The execution context. + * :param sub: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * :param type: + * A Python type object. This argument must not be ``HPy_NULL`` and must be + * a type (i.e. it must inherit from Python ``type``). If this is not the + * case, the behavior is undefined (verification of the argument is only + * done in debug mode). + * + * :returns: + * Non-zero if ``sub`` is a subtype of ``type``. + */ +HPy_ID(254) +int HPyType_IsSubtype(HPyContext *ctx, HPy sub, HPy type); + +HPy_ID(167) +int HPy_Is(HPyContext *ctx, HPy obj, HPy other); + +HPy_ID(168) +void* _HPy_AsStruct_Object(HPyContext *ctx, HPy h); +HPy_ID(169) +void* _HPy_AsStruct_Legacy(HPyContext *ctx, HPy h); +HPy_ID(228) +void* _HPy_AsStruct_Type(HPyContext *ctx, HPy h); +HPy_ID(229) +void* _HPy_AsStruct_Long(HPyContext *ctx, HPy h); +HPy_ID(230) +void* _HPy_AsStruct_Float(HPyContext *ctx, HPy h); +HPy_ID(231) +void* _HPy_AsStruct_Unicode(HPyContext *ctx, HPy h); +HPy_ID(232) +void* _HPy_AsStruct_Tuple(HPyContext *ctx, HPy h); +HPy_ID(233) +void* _HPy_AsStruct_List(HPyContext *ctx, HPy h); +HPy_ID(264) +void* _HPy_AsStruct_Dict(HPyContext *ctx, HPy h); +HPy_ID(234) +HPyType_BuiltinShape _HPyType_GetBuiltinShape(HPyContext *ctx, HPy h_type); + +HPy_ID(170) +HPy _HPy_New(HPyContext *ctx, HPy h_type, void **data); + +HPy_ID(171) +HPy HPy_Repr(HPyContext *ctx, HPy obj); +HPy_ID(172) +HPy HPy_Str(HPyContext *ctx, HPy obj); +HPy_ID(173) +HPy HPy_ASCII(HPyContext *ctx, HPy obj); +HPy_ID(174) +HPy HPy_Bytes(HPyContext *ctx, HPy obj); + +HPy_ID(175) +HPy HPy_RichCompare(HPyContext *ctx, HPy v, HPy w, int op); +HPy_ID(176) +int HPy_RichCompareBool(HPyContext *ctx, HPy v, HPy w, int op); + +HPy_ID(177) +HPy_hash_t HPy_Hash(HPyContext *ctx, HPy obj); + +/* bytesobject.h */ +HPy_ID(178) +int HPyBytes_Check(HPyContext *ctx, HPy h); +HPy_ID(179) +HPy_ssize_t HPyBytes_Size(HPyContext *ctx, HPy h); +HPy_ID(180) +HPy_ssize_t HPyBytes_GET_SIZE(HPyContext *ctx, HPy h); +HPy_ID(181) +const char* HPyBytes_AsString(HPyContext *ctx, HPy h); +HPy_ID(182) +const char* HPyBytes_AS_STRING(HPyContext *ctx, HPy h); +HPy_ID(183) +HPy HPyBytes_FromString(HPyContext *ctx, const char *bytes); +HPy_ID(184) +HPy HPyBytes_FromStringAndSize(HPyContext *ctx, const char *bytes, HPy_ssize_t len); + +/* unicodeobject.h */ +HPy_ID(185) +HPy HPyUnicode_FromString(HPyContext *ctx, const char *utf8); +HPy_ID(186) +int HPyUnicode_Check(HPyContext *ctx, HPy h); +HPy_ID(187) +HPy HPyUnicode_AsASCIIString(HPyContext *ctx, HPy h); +HPy_ID(188) +HPy HPyUnicode_AsLatin1String(HPyContext *ctx, HPy h); +HPy_ID(189) +HPy HPyUnicode_AsUTF8String(HPyContext *ctx, HPy h); +HPy_ID(190) +const char* HPyUnicode_AsUTF8AndSize(HPyContext *ctx, HPy h, HPy_ssize_t *size); +HPy_ID(191) +HPy HPyUnicode_FromWideChar(HPyContext *ctx, const wchar_t *w, HPy_ssize_t size); +HPy_ID(192) +HPy HPyUnicode_DecodeFSDefault(HPyContext *ctx, const char *v); +HPy_ID(193) +HPy HPyUnicode_DecodeFSDefaultAndSize(HPyContext *ctx, const char *v, HPy_ssize_t size); +HPy_ID(194) +HPy HPyUnicode_EncodeFSDefault(HPyContext *ctx, HPy h); +HPy_ID(195) +HPy_UCS4 HPyUnicode_ReadChar(HPyContext *ctx, HPy h, HPy_ssize_t index); +HPy_ID(196) +HPy HPyUnicode_DecodeASCII(HPyContext *ctx, const char *ascii, HPy_ssize_t size, const char *errors); +HPy_ID(197) +HPy HPyUnicode_DecodeLatin1(HPyContext *ctx, const char *latin1, HPy_ssize_t size, const char *errors); + +/** + * Decode a bytes-like object to a Unicode object. + * + * The bytes of the bytes-like object are decoded according to the given + * encoding and using the error handling defined by ``errors``. + * + * :param ctx: + * The execution context. + * :param obj: + * A bytes-like object. This can be, for example, Python *bytes*, + * *bytearray*, *memoryview*, *array.array* and objects that support the + * Buffer protocol. If this argument is `HPy_NULL``, a ``SystemError`` will + * be raised. If the argument is not a bytes-like object, a ``TypeError`` + * will be raised. + * :param encoding: + * The name (UTF-8 encoded C string) of the encoding to use. If the encoding + * does not exist, a ``LookupError`` will be raised. If this argument is + * ``NULL``, the default encoding ``UTF-8`` will be used. + * :param errors: + * The error handling (UTF-8 encoded C string) to use when decoding. The + * possible values depend on the used encoding. This argument may be + * ``NULL`` in which case it will default to ``"strict"``. + * + * :returns: + * A handle to a ``str`` object created from the decoded bytes or + * ``HPy_NULL`` in case of errors. + */ +HPy_ID(255) +HPy HPyUnicode_FromEncodedObject(HPyContext *ctx, HPy obj, const char *encoding, const char *errors); + +/** + * Return a substring of ``str``, from character index ``start`` (included) to + * character index ``end`` (excluded). + * + * Indices ``start`` and ``end`` must not be negative, otherwise an + * ``IndexError`` will be raised. If ``start >= len(str)`` or if + * ``end < start``, an empty string will be returned. If ``end > len(str)`` then + * ``end == len(str)`` will be assumed. + * + * :param ctx: + * The execution context. + * :param str: + * A Python Unicode object (must not be ``HPy_NULL``). Otherwise, the + * behavior is undefined (verification of the argument is only done in + * debug mode). + * :param start: + * The non-negative start index (inclusive). + * :param end: + * The non-negative end index (exclusive). + * + * :returns: + * The requested substring or ``HPy_NULL`` in case of an error. + */ +HPy_ID(256) +HPy HPyUnicode_Substring(HPyContext *ctx, HPy str, HPy_ssize_t start, HPy_ssize_t end); + +/* listobject.h */ + +/** + * Tests if an object is an instance of a Python list. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``list`` or an instance + * of a subtype of ``list``, and ``0`` otherwise. + */ +HPy_ID(198) +int HPyList_Check(HPyContext *ctx, HPy h); + +/** + * Creates a new list instance with length ``len``. + * + * :param ctx: + * The execution context. + * :param len: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * + * :returns: + * The new list instance on success, or ``HPy_NULL`` on failure. + */ +HPy_ID(199) +HPy HPyList_New(HPyContext *ctx, HPy_ssize_t len); + +/** + * Append item ``h_item`` to list ``h_list``. + * + * :param ctx: + * The execution context. + * :param h_list: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * :param h_item: + * The item to append (must not be ``HPy_NULL``). + * + * :returns: + * Return ``0`` if successful; return ``-1`` and set an exception if + * unsuccessful. + */ +HPy_ID(200) +int HPyList_Append(HPyContext *ctx, HPy h_list, HPy h_item); + +/** + * Insert the item ``h_item`` into list ``h_list`` in front of index ``index``. + * + * :param ctx: + * The execution context. + * :param h_list: + * A Python list object (must not be ``HPy_NULL``). Otherwise, a + * ``SystemError`` will be raised. + * :param index: + * The index where the element should be inserted before. A negative index + * is allowed and is then interpreted to be relative to the end of sequence. + * E.g. ``index == -1`` is the last element. + * If ``index < -n`` (where ``n`` is the length of the list), it will be + * replaced by ``0``. If ``index > n``, it will be replaced by ``n``. + * :param h_item: + * The item to insert (must not be ``HPy_NULL``). + * + * :returns: + * Return ``0`` if successful; return ``-1`` and set an exception if + * unsuccessful. + */ +HPy_ID(265) +int HPyList_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item); + +/* dictobject.h */ + +/** + * Tests if an object is an instance of a Python dict. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``dict`` or an instance + * of a subtype of ``dict``, and ``0`` otherwise. + */ +HPy_ID(201) +int HPyDict_Check(HPyContext *ctx, HPy h); + +/** + * Creates a new empty Python dictionary. + * + * :param ctx: + * The execution context. + * + * :returns: + * A handle to the new and empty Python dictionary or ``HPy_NULL`` in case + * of an error. + */ +HPy_ID(202) +HPy HPyDict_New(HPyContext *ctx); + +/** + * Returns a list of all keys from the dictionary. + * + * Note: This function will directly access the storage of the dict object and + * therefore ignores if method ``keys`` was overwritten. + * + * :param ctx: + * The execution context. + * :param h: + * A Python dict object. If this argument is ``HPy_NULL`` or not an + * instance of a Python dict, a ``SystemError`` will be raised. + * + * :returns: + * A Python list object containing all keys of the given dictionary or + * ``HPy_NULL`` in case of an error. + */ +HPy_ID(257) +HPy HPyDict_Keys(HPyContext *ctx, HPy h); + +/** + * Creates a copy of the provided Python dict object. + * + * :param ctx: + * The execution context. + * :param h: + * A Python dict object. If this argument is ``HPy_NULL`` or not an + * instance of a Python dict, a ``SystemError`` will be raised. + * + * :returns: + * Return a new dictionary that contains the same key-value pairs as ``h`` + * or ``HPy_NULL`` in case of an error. + */ +HPy_ID(258) +HPy HPyDict_Copy(HPyContext *ctx, HPy h); + +/* tupleobject.h */ + +/** + * Tests if an object is an instance of a Python tuple. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an arbitrary object (must not be ``HPy_NULL``). + * + * :returns: + * Non-zero if object ``h`` is an instance of type ``tuple`` or an instance + * of a subtype of ``tuple``, and ``0`` otherwise. + */ +HPy_ID(203) +int HPyTuple_Check(HPyContext *ctx, HPy h); + +/** + * Create a tuple from an array. + * + * Note: Consider to use the convenience function :c:func:`HPyTuple_Pack` to + * create a tuple. + * + * :param ctx: + * The execution context. + * :param items: + * An array of items to use for initialization of the tuple. + * :param n: + * The number of elements in array ``items``. + * + * :return: + * A new tuple with ``n`` elements or ``HPy_NULL`` in case of an error + * occurred. + */ +HPy_ID(204) +HPy HPyTuple_FromArray(HPyContext *ctx, const HPy items[], HPy_ssize_t n); + +/* sliceobject.h */ + +/** + * Creates a new empty Python slice object. + * + * :param ctx: + * The execution context. + * + * :param start: + * A handle to an object to be used as the slice start value. + * :param end: + * A handle to an object to be used as the slice end value. + * :param step: + * A handle to an object to be used as the slice step value. + * + * :returns: + * A handle to the new and empty Python slice object or ``HPy_NULL`` in case + * of an error. + */ +HPy_ID(272) +HPy HPySlice_New(HPyContext *ctx, HPy start, HPy stop, HPy step); + +/** + * Extract the start, stop and step data members from a slice object as C + * integers. + * + * The slice members may be arbitrary int-like objects. If they are not Python + * int objects, they will be coerced to int objects by calling their + * ``__index__`` method. + * + * If a slice member value is out of bounds, it will be set to the maximum value + * of ``HPy_ssize_t`` if the member was a positive number, or to the minimum + * value of ``HPy_ssize_t`` if it was a negative number. + * + * :param ctx: + * The execution context. + * :param slice: + * A handle to a Python slice object. This argument must be a slice object + * and must not be ``HPy_NULL``. Otherwise, behavior is undefined. + * :param start: + * A pointer to a variable where to write the unpacked slice start. Must not + * be ``NULL``. + * :param end: + * A pointer to a variable where to write the unpacked slice end. Must not + * :param step: + * A pointer to a variable where to write the unpacked slice step. Must not + * be ``NULL``. + * + * :returns: + * ``-1`` on error, ``0`` on success + */ + +HPy_ID(259) +int HPySlice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step); + +/* import.h */ +HPy_ID(205) +HPy HPyImport_ImportModule(HPyContext *ctx, const char *utf8_name); + +/* pycapsule.h */ +HPy_ID(244) +HPy HPyCapsule_New(HPyContext *ctx, void *pointer, const char *utf8_name, HPyCapsule_Destructor *destructor); +HPy_ID(245) +void* HPyCapsule_Get(HPyContext *ctx, HPy capsule, _HPyCapsule_key key, const char *utf8_name); +HPy_ID(246) +int HPyCapsule_IsValid(HPyContext *ctx, HPy capsule, const char *utf8_name); +HPy_ID(247) +int HPyCapsule_Set(HPyContext *ctx, HPy capsule, _HPyCapsule_key key, void *value); + +/* integration with the old CPython API */ +HPy_ID(206) +HPy HPy_FromPyObject(HPyContext *ctx, cpy_PyObject *obj); +HPy_ID(207) +cpy_PyObject *HPy_AsPyObject(HPyContext *ctx, HPy h); + +/* internal helpers which need to be exposed to modules for practical reasons :( */ +HPy_ID(208) +void _HPy_CallRealFunctionFromTrampoline(HPyContext *ctx, + HPyFunc_Signature sig, + HPyCFunction func, + void *args); + +/* Builders */ + +/** + * Create a new list builder for ``size`` elements. The builder is then able to + * take at most ``size`` elements. This function does not raise any + * exception (even if running out of memory). + * + * :param ctx: + * The execution context. + * :param size: + * The number of elements to hold. + */ +HPy_ID(209) +HPyListBuilder HPyListBuilder_New(HPyContext *ctx, HPy_ssize_t size); + +/** + * Assign an element to a certain index of the builder. Valid indices are in + * range ``0 <= index < size`` where ``size`` is the value passed to + * :c:func:`HPyListBuilder_New`. This function does not raise any exception. + * + * :param ctx: + * The execution context. + * :param builder: + * A list builder handle. + * :param index: + * The index to assign the object to. + * :param h_item: + * An HPy handle of the object to store or ``HPy_NULL``. Please note that + * HPy **never** steals handles and so, ``h_item`` needs to be closed by + * the caller. + */ +HPy_ID(210) +void HPyListBuilder_Set(HPyContext *ctx, HPyListBuilder builder, + HPy_ssize_t index, HPy h_item); + +/** + * Build a list from a list builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A list builder handle. + * + * :returns: + * An HPy handle to a list containing the values inserted with + * :c:func:`HPyListBuilder_Set` or ``HPy_NULL`` in case an error occurred + * during building or earlier when creating the builder or setting the + * items. + */ +HPy_ID(211) +HPy HPyListBuilder_Build(HPyContext *ctx, HPyListBuilder builder); + +/** + * Cancel building of a tuple and free any acquired resources. + * This function ignores if any error occurred previously when using the tuple + * builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + */ +HPy_ID(212) +void HPyListBuilder_Cancel(HPyContext *ctx, HPyListBuilder builder); + +/** + * Create a new tuple builder for ``size`` elements. The builder is then able + * to take at most ``size`` elements. This function does not raise any + * exception (even if running out of memory). + * + * :param ctx: + * The execution context. + * :param size: + * The number of elements to hold. + */ +HPy_ID(213) +HPyTupleBuilder HPyTupleBuilder_New(HPyContext *ctx, HPy_ssize_t size); + +/** + * Assign an element to a certain index of the builder. Valid indices are in + * range ``0 <= index < size`` where ``size`` is the value passed to + * :c:func:`HPyTupleBuilder_New`. This function does not raise * any exception. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + * :param index: + * The index to assign the object to. + * :param h_item: + * An HPy handle of the object to store or ``HPy_NULL``. Please note that + * HPy **never** steals handles and so, ``h_item`` needs to be closed by + * the caller. + */ +HPy_ID(214) +void HPyTupleBuilder_Set(HPyContext *ctx, HPyTupleBuilder builder, + HPy_ssize_t index, HPy h_item); + +/** + * Build a tuple from a tuple builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + * + * :returns: + * An HPy handle to a tuple containing the values inserted with + * :c:func:`HPyTupleBuilder_Set` or ``HPy_NULL`` in case an error occurred + * during building or earlier when creating the builder or setting the + * items. + */ +HPy_ID(215) +HPy HPyTupleBuilder_Build(HPyContext *ctx, HPyTupleBuilder builder); + +/** + * Cancel building of a tuple and free any acquired resources. + * This function ignores if any error occurred previously when using the tuple + * builder. + * + * :param ctx: + * The execution context. + * :param builder: + * A tuple builder handle. + */ +HPy_ID(216) +void HPyTupleBuilder_Cancel(HPyContext *ctx, HPyTupleBuilder builder); + +/* Helper for correctly closing handles */ + +HPy_ID(217) +HPyTracker HPyTracker_New(HPyContext *ctx, HPy_ssize_t size); +HPy_ID(218) +int HPyTracker_Add(HPyContext *ctx, HPyTracker ht, HPy h); +HPy_ID(219) +void HPyTracker_ForgetAll(HPyContext *ctx, HPyTracker ht); +HPy_ID(220) +void HPyTracker_Close(HPyContext *ctx, HPyTracker ht); + +/** + * HPyFields should be used ONLY in parts of memory which is known to the GC, + * e.g. memory allocated by HPy_New: + * + * - NEVER declare a local variable of type HPyField + * - NEVER use HPyField on a struct allocated by e.g. malloc() + * + * **CPython's note**: contrary to PyObject*, you don't need to manually + * manage refcounting when using HPyField: if you use HPyField_Store to + * overwrite an existing value, the old object will be automatically decrefed. + * This means that you CANNOT use HPyField_Store to write memory which + * contains uninitialized values, because it would try to decref a dangling + * pointer. + * + * Note that HPy_New automatically zeroes the memory it allocates, so + * everything works well out of the box. In case you are using manually + * allocated memory, you should initialize the HPyField to HPyField_NULL. + * + * Note the difference: + * + * - ``obj->f = HPyField_NULL``: this should be used only to initialize + * uninitialized memory. If you use it to overwrite a valid HPyField, you + * will cause a memory leak (at least on CPython) + * + * - HPyField_Store(ctx, &obj->f, HPy_NULL): this does the right thing and + * decref the old value. However, you CANNOT use it if the memory is not + * initialized. + * + * Note: target_object and source_object are there in case an implementation + * needs to add write and/or read barriers on the objects. They are ignored by + * CPython but e.g. PyPy needs a write barrier. +*/ +HPy_ID(221) +void HPyField_Store(HPyContext *ctx, HPy target_object, HPyField *target_field, HPy h); +HPy_ID(222) +HPy HPyField_Load(HPyContext *ctx, HPy source_object, HPyField source_field); + +/** + * Leaving Python execution: for releasing GIL and other use-cases. + * + * In most situations, users should prefer using convenience macros: + * HPy_BEGIN_LEAVE_PYTHON(context)/HPy_END_LEAVE_PYTHON(context) + * + * HPy extensions may leave Python execution when running Python independent + * code: long-running computations or blocking operations. When an extension + * has left the Python execution it must not call any HPy API other than + * HPy_ReenterPythonExecution. It can access pointers returned by HPy API, + * e.g., HPyUnicode_AsUTF8String, provided that they are valid at the point + * of calling HPy_LeavePythonExecution. + * + * Python execution must be reentered on the same thread as where it was left. + * The leave/enter calls must not be nested. Debug mode will, in the future, + * enforce these constraints. + * + * Python implementations may use this knowledge however they wish. The most + * obvious use case is to release the GIL, in which case the + * HPy_BEGIN_LEAVE_PYTHON/HPy_END_LEAVE_PYTHON becomes equivalent to + * Py_BEGIN_ALLOW_THREADS/Py_END_ALLOW_THREADS. +*/ +HPy_ID(223) +void HPy_ReenterPythonExecution(HPyContext *ctx, HPyThreadState state); +HPy_ID(224) +HPyThreadState HPy_LeavePythonExecution(HPyContext *ctx); + +/** + * HPyGlobal is an alternative to module state. HPyGlobal must be a statically + * allocated C global variable registered in HPyModuleDef.globals array. + * A HPyGlobal can be used only after the HPy module where it is registered was + * created using HPyModule_Create. + * + * HPyGlobal serves as an identifier of a Python object that should be globally + * available per one Python interpreter. Python objects referenced by HPyGlobals + * are destroyed automatically on the interpreter exit (not necessarily the + * process exit). + * + * HPyGlobal instance does not allow anything else but loading and storing + * a HPy handle using a HPyContext. Even if the HPyGlobal C variable may + * be shared between threads or different interpreter instances within one + * process, the API to load and store a handle from HPyGlobal is thread-safe (but + * like any other HPy API must not be called in HPy_LeavePythonExecution blocks). + * + * Given that a handle to object X1 is stored to HPyGlobal using HPyContext of + * Python interpreter I1, then loading a handle from the same HPyGlobal using + * HPyContext of Python interpreter I1 should give a handle to the same object + * X1. Another Python interpreter I2 running within the same process and using + * the same HPyGlobal variable will not be able to load X1 from it, it will have + * its own view on what is stored in the given HPyGlobal. + * + * Python interpreters may use indirection to isolate different interpreter + * instances, but alternative techniques such as copy-on-write or immortal + * objects can be used to avoid that indirection (even selectively on per + * object basis using tagged pointers). + * + * CPython HPy implementation may even provide configuration option that + * switches between a faster version that directly stores PyObject* to + * HPyGlobal but does not support subinterpreters, or a version that supports + * subinterpreters. For now, CPython HPy always stores PyObject* directly + * to HPyGlobal. + * + * While the standard implementation does not fully enforce the documented + * contract, the HPy debug mode will enforce it (not implemented yet). + * + * **Implementation notes:** + * All Python interpreters running in one process must be compatible, because + * they will share all HPyGlobal C level variables. The internal data stored + * in HPyGlobal are specific for each HPy implementation, each implementation + * is also responsible for handling thread-safety when initializing the + * internal data in HPyModule_Create. Note that HPyModule_Create may be called + * concurrently depending on the semantics of the Python implementation (GIL vs + * no GIL) and also depending on the whether there may be multiple instances of + * given Python interpreter running within the same process. In the future, HPy + * ABI may include a contract that internal data of each HPyGlobal must be + * initialized to its address using atomic write and HPy implementations will + * not be free to choose what to store in HPyGlobal, however, this will allow + * multiple different HPy implementations within one process. This contract may + * also be activated only by some runtime option, letting the HPy implementation + * use more optimized HPyGlobal implementation otherwise. +*/ +HPy_ID(225) +void HPyGlobal_Store(HPyContext *ctx, HPyGlobal *global, HPy h); +HPy_ID(226) +HPy HPyGlobal_Load(HPyContext *ctx, HPyGlobal global); + +/* Debugging helpers */ +HPy_ID(227) +void _HPy_Dump(HPyContext *ctx, HPy h); + +/* Evaluating Python statements/expressions */ + +/** + * Parse and compile the Python source code. + * + * :param ctx: + * The execution context. + * :param utf8_source: + * Python source code given as UTF-8 encoded C string (must not be ``NULL``). + * :param utf8_filename: + * The filename (UTF-8 encoded C string) to use for construction of the code + * object. It may appear in tracebacks or in ``SyntaxError`` exception + * messages. + * :param kind: + * The source kind which tells the parser if a single expression, statement, + * or a whole file should be parsed (see enum :c:enum:`HPy_SourceKind`). + * + * :returns: + * A Python code object resulting from the parsed and compiled Python source + * code or ``HPy_NULL`` in case of errors. + */ +HPy_ID(248) +HPy HPy_Compile_s(HPyContext *ctx, const char *utf8_source, const char *utf8_filename, HPy_SourceKind kind); + +/** + * Evaluate a precompiled code object. + * + * Code objects can be compiled from a string using :c:func:`HPy_Compile_s`. + * + * :param ctx: + * The execution context. + * :param code: + * The code object to evaluate. + * :param globals: + * A Python dictionary defining the global variables for the evaluation. + * :param locals: + * A mapping object defining the local variables for the evaluation. + * + * :returns: + * The result produced by the executed code. May be ``HPy_NULL`` in case of + * errors. + */ +HPy_ID(249) +HPy HPy_EvalCode(HPyContext *ctx, HPy code, HPy globals, HPy locals); +HPy_ID(250) +HPy HPyContextVar_New(HPyContext *ctx, const char *name, HPy default_value); +HPy_ID(251) +int32_t HPyContextVar_Get(HPyContext *ctx, HPy context_var, HPy default_value, HPy *result); +HPy_ID(252) +HPy HPyContextVar_Set(HPyContext *ctx, HPy context_var, HPy value); + +/** + * Set the call function for the given object. + * + * By defining slot ``HPy_tp_call`` for some type, instances of this type will + * be callable objects. The specified call function will be used by default for + * every instance. This should account for the most common case (every instance + * of an object uses the same call function) but to still provide the necessary + * flexibility, function ``HPy_SetCallFunction`` allows to set different (maybe + * specialized) call functions for each instance. This must be done in the + * constructor of an object. + * + * A more detailed description on how to use that function can be found in + * section :ref:`porting-guide:calling protocol`. + * + * :param ctx: + * The execution context. + * :param h: + * A handle to an object implementing the call protocol, i.e., the object's + * type must have slot ``HPy_tp_call``. Otherwise, a ``TypeError`` will be + * raised. This argument must not be ``HPy_NULL``. + * :param def: + * A pointer to the call function definition to set (must not be + * ``NULL``). The definition is usually created using + * :c:macro:`HPyDef_CALL_FUNCTION` + * + * :returns: + * ``0`` in case of success and ``-1`` in case of an error. + */ +HPy_ID(260) +int HPy_SetCallFunction(HPyContext *ctx, HPy h, HPyCallFunction *func); + +/* ******* + hpyfunc + ******* + + These typedefs are used to generate the various macros used by + include/common/hpyfunc.h +*/ +typedef HPy (*HPyFunc_noargs)(HPyContext *ctx, HPy self); +typedef HPy (*HPyFunc_o)(HPyContext *ctx, HPy self, HPy arg); +typedef HPy (*HPyFunc_varargs)(HPyContext *ctx, HPy self, const HPy *args, size_t nargs); +typedef HPy (*HPyFunc_keywords)(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames); + +typedef HPy (*HPyFunc_unaryfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_binaryfunc)(HPyContext *ctx, HPy, HPy); +typedef HPy (*HPyFunc_ternaryfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_inquiry)(HPyContext *ctx, HPy); +typedef HPy_ssize_t (*HPyFunc_lenfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_ssizeargfunc)(HPyContext *ctx, HPy, HPy_ssize_t); +typedef HPy (*HPyFunc_ssizessizeargfunc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy_ssize_t); +typedef int (*HPyFunc_ssizeobjargproc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy); +typedef int (*HPyFunc_ssizessizeobjargproc)(HPyContext *ctx, HPy, HPy_ssize_t, HPy_ssize_t, HPy); +typedef int (*HPyFunc_objobjargproc)(HPyContext *ctx, HPy, HPy, HPy); +typedef void (*HPyFunc_freefunc)(HPyContext *ctx, void *); +typedef HPy (*HPyFunc_getattrfunc)(HPyContext *ctx, HPy, char *); +typedef HPy (*HPyFunc_getattrofunc)(HPyContext *ctx, HPy, HPy); +typedef int (*HPyFunc_setattrfunc)(HPyContext *ctx, HPy, char *, HPy); +typedef int (*HPyFunc_setattrofunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef HPy (*HPyFunc_reprfunc)(HPyContext *ctx, HPy); +typedef HPy_hash_t (*HPyFunc_hashfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_richcmpfunc)(HPyContext *ctx, HPy, HPy, HPy_RichCmpOp); +typedef HPy (*HPyFunc_getiterfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_iternextfunc)(HPyContext *ctx, HPy); +typedef HPy (*HPyFunc_descrgetfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_descrsetfunc)(HPyContext *ctx, HPy, HPy, HPy); +typedef int (*HPyFunc_initproc)(HPyContext *ctx, HPy self, + const HPy *args, HPy_ssize_t nargs, HPy kw); +typedef HPy (*HPyFunc_newfunc)(HPyContext *ctx, HPy type, const HPy *args, + HPy_ssize_t nargs, HPy kw); +typedef HPy (*HPyFunc_getter)(HPyContext *ctx, HPy, void *); +typedef int (*HPyFunc_setter)(HPyContext *ctx, HPy, HPy, void *); +typedef int (*HPyFunc_objobjproc)(HPyContext *ctx, HPy, HPy); +typedef int (*HPyFunc_getbufferproc)(HPyContext *ctx, HPy, HPy_buffer *, int); +typedef void (*HPyFunc_releasebufferproc)(HPyContext *ctx, HPy, HPy_buffer *); +typedef int (*HPyFunc_traverseproc)(void *object, HPyFunc_visitproc visit, void *arg); +typedef void (*HPyFunc_destructor)(HPyContext *ctx, HPy); + +typedef void (*HPyFunc_destroyfunc)(void *); + +// Note: separate type, because we need a different trampoline +typedef HPy (*HPyFunc_mod_create)(HPyContext *ctx, HPy); + + +/* ~~~ HPySlot_Slot ~~~ + + The following enum is used to generate autogen_hpyslot.h, which contains: + + - The real definition of the enum HPySlot_Slot + + - the macros #define _HPySlot_SIGNATURE_* + +*/ + +// NOTE: if you uncomment/enable a slot below, make sure to write a corresponding +// test in test_slots.py + +/* Note that the magic numbers are the same as CPython */ +typedef enum { + HPy_bf_getbuffer = SLOT(1, HPyFunc_GETBUFFERPROC), + HPy_bf_releasebuffer = SLOT(2, HPyFunc_RELEASEBUFFERPROC), + HPy_mp_ass_subscript = SLOT(3, HPyFunc_OBJOBJARGPROC), + HPy_mp_length = SLOT(4, HPyFunc_LENFUNC), + HPy_mp_subscript = SLOT(5, HPyFunc_BINARYFUNC), + HPy_nb_absolute = SLOT(6, HPyFunc_UNARYFUNC), + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_nb_and = SLOT(8, HPyFunc_BINARYFUNC), + HPy_nb_bool = SLOT(9, HPyFunc_INQUIRY), + HPy_nb_divmod = SLOT(10, HPyFunc_BINARYFUNC), + HPy_nb_float = SLOT(11, HPyFunc_UNARYFUNC), + HPy_nb_floor_divide = SLOT(12, HPyFunc_BINARYFUNC), + HPy_nb_index = SLOT(13, HPyFunc_UNARYFUNC), + HPy_nb_inplace_add = SLOT(14, HPyFunc_BINARYFUNC), + HPy_nb_inplace_and = SLOT(15, HPyFunc_BINARYFUNC), + HPy_nb_inplace_floor_divide = SLOT(16, HPyFunc_BINARYFUNC), + HPy_nb_inplace_lshift = SLOT(17, HPyFunc_BINARYFUNC), + HPy_nb_inplace_multiply = SLOT(18, HPyFunc_BINARYFUNC), + HPy_nb_inplace_or = SLOT(19, HPyFunc_BINARYFUNC), + HPy_nb_inplace_power = SLOT(20, HPyFunc_TERNARYFUNC), + HPy_nb_inplace_remainder = SLOT(21, HPyFunc_BINARYFUNC), + HPy_nb_inplace_rshift = SLOT(22, HPyFunc_BINARYFUNC), + HPy_nb_inplace_subtract = SLOT(23, HPyFunc_BINARYFUNC), + HPy_nb_inplace_true_divide = SLOT(24, HPyFunc_BINARYFUNC), + HPy_nb_inplace_xor = SLOT(25, HPyFunc_BINARYFUNC), + HPy_nb_int = SLOT(26, HPyFunc_UNARYFUNC), + HPy_nb_invert = SLOT(27, HPyFunc_UNARYFUNC), + HPy_nb_lshift = SLOT(28, HPyFunc_BINARYFUNC), + HPy_nb_multiply = SLOT(29, HPyFunc_BINARYFUNC), + HPy_nb_negative = SLOT(30, HPyFunc_UNARYFUNC), + HPy_nb_or = SLOT(31, HPyFunc_BINARYFUNC), + HPy_nb_positive = SLOT(32, HPyFunc_UNARYFUNC), + HPy_nb_power = SLOT(33, HPyFunc_TERNARYFUNC), + HPy_nb_remainder = SLOT(34, HPyFunc_BINARYFUNC), + HPy_nb_rshift = SLOT(35, HPyFunc_BINARYFUNC), + HPy_nb_subtract = SLOT(36, HPyFunc_BINARYFUNC), + HPy_nb_true_divide = SLOT(37, HPyFunc_BINARYFUNC), + HPy_nb_xor = SLOT(38, HPyFunc_BINARYFUNC), + HPy_sq_ass_item = SLOT(39, HPyFunc_SSIZEOBJARGPROC), + HPy_sq_concat = SLOT(40, HPyFunc_BINARYFUNC), + HPy_sq_contains = SLOT(41, HPyFunc_OBJOBJPROC), + HPy_sq_inplace_concat = SLOT(42, HPyFunc_BINARYFUNC), + HPy_sq_inplace_repeat = SLOT(43, HPyFunc_SSIZEARGFUNC), + HPy_sq_item = SLOT(44, HPyFunc_SSIZEARGFUNC), + HPy_sq_length = SLOT(45, HPyFunc_LENFUNC), + HPy_sq_repeat = SLOT(46, HPyFunc_SSIZEARGFUNC), + //HPy_tp_alloc = SLOT(47, HPyFunc_X), NOT SUPPORTED + //HPy_tp_base = SLOT(48, HPyFunc_X), + //HPy_tp_bases = SLOT(49, HPyFunc_X), + HPy_tp_call = SLOT(50, HPyFunc_KEYWORDS), + //HPy_tp_clear = SLOT(51, HPyFunc_X), NOT SUPPORTED, use tp_traverse + //HPy_tp_dealloc = SLOT(52, HPyFunc_X), NOT SUPPORTED + //HPy_tp_del = SLOT(53, HPyFunc_X), + HPy_tp_descr_get = SLOT(54, HPyFunc_TERNARYFUNC), + //HPy_tp_descr_set = SLOT(55, HPyFunc_X), + //HPy_tp_doc = SLOT(56, HPyFunc_X), + //HPy_tp_getattr = SLOT(57, HPyFunc_X), + //HPy_tp_getattro = SLOT(58, HPyFunc_X), + HPy_tp_hash = SLOT(59, HPyFunc_HASHFUNC), + HPy_tp_init = SLOT(60, HPyFunc_INITPROC), + //HPy_tp_is_gc = SLOT(61, HPyFunc_X), + //HPy_tp_iter = SLOT(62, HPyFunc_X), + //HPy_tp_iternext = SLOT(63, HPyFunc_X), + //HPy_tp_methods = SLOT(64, HPyFunc_X), NOT SUPPORTED + HPy_tp_new = SLOT(65, HPyFunc_NEWFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + HPy_tp_richcompare = SLOT(67, HPyFunc_RICHCMPFUNC), + //HPy_tp_setattr = SLOT(68, HPyFunc_X), + //HPy_tp_setattro = SLOT(69, HPyFunc_X), + HPy_tp_str = SLOT(70, HPyFunc_REPRFUNC), + HPy_tp_traverse = SLOT(71, HPyFunc_TRAVERSEPROC), + //HPy_tp_members = SLOT(72, HPyFunc_X), NOT SUPPORTED + //HPy_tp_getset = SLOT(73, HPyFunc_X), NOT SUPPORTED + //HPy_tp_free = SLOT(74, HPyFunc_X), NOT SUPPORTED + HPy_nb_matrix_multiply = SLOT(75, HPyFunc_BINARYFUNC), + HPy_nb_inplace_matrix_multiply = SLOT(76, HPyFunc_BINARYFUNC), + //HPy_am_await = SLOT(77, HPyFunc_X), + //HPy_am_aiter = SLOT(78, HPyFunc_X), + //HPy_am_anext = SLOT(79, HPyFunc_X), + HPy_tp_finalize = SLOT(80, HPyFunc_DESTRUCTOR), + + /* extra HPy slots */ + HPy_tp_destroy = SLOT(1000, HPyFunc_DESTROYFUNC), + + /** + * Module create slot: the function receives loader spec and should + * return an HPy handle representing the module. Currently, creating + * real module objects cannot be done by user code, so the only other + * useful thing that this slot can do is to create another object that + * can work as a module, such as SimpleNamespace. + */ + HPy_mod_create = SLOT(2000, HPyFunc_MOD_CREATE), + /** + * Module exec slot: the function receives module object that was created + * by the runtime from HPyModuleDef. This slot can do any initialization + * of the module, such as adding types. There can be multiple exec slots + * and they will be executed in the declaration order. + */ + HPy_mod_exec = SLOT(2001, HPyFunc_INQUIRY), + +} HPySlot_Slot; diff --git a/graalpython/hpy/hpy/tools/autogen/pypy.py b/graalpython/hpy/hpy/tools/autogen/pypy.py new file mode 100644 index 0000000000..702a519e84 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/pypy.py @@ -0,0 +1,40 @@ +from .autogenfile import AutoGenFile +from .parse import toC + +# this class should probably be moved somewhere in the PyPy repo +class autogen_pypy_txt(AutoGenFile): + PATH = 'hpy/tools/autogen/autogen_pypy.txt' + LANGUAGE = 'txt' # to avoid inserting the disclaimer + + def generate(self): + lines = [] + w = lines.append + w("typedef struct _HPyContext_s {") + w(" int abi_version;") + for var in self.api.variables: + w(" struct _HPy_s %s;" % var.ctx_name()) + for func in self.api.functions: + w(" void * %s;" % func.ctx_name()) + w("} _struct_HPyContext_s;") + w("") + w("") + # generate stubs for all the API functions + for func in self.api.functions: + w(self.stub(func)) + return '\n'.join(lines) + + def stub(self, func): + signature = toC(func.node) + if func.is_varargs(): + return '# %s' % signature + # + argnames = [p.name for p in func.node.type.args.params] + lines = [] + w = lines.append + w('@API.func("%s")' % signature) + w('def %s(space, %s):' % (func.name, ', '.join(argnames))) + w(' from rpython.rlib.nonconst import NonConstant # for the annotator') + w(' if NonConstant(False): return 0') + w(' raise NotImplementedError') + w('') + return '\n'.join(lines) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/__init__.py b/graalpython/hpy/hpy/tools/autogen/testing/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/__init__.py rename to graalpython/hpy/hpy/tools/autogen/testing/__init__.py diff --git a/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py b/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py new file mode 100644 index 0000000000..2e1a971d63 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/testing/test_autogen.py @@ -0,0 +1,224 @@ +import textwrap +import difflib +import py +import pytest +from hpy.tools.autogen.parse import HPyAPI +from hpy.tools.autogen.ctx import autogen_ctx_h, autogen_ctx_def_h +from hpy.tools.autogen.trampolines import (autogen_trampolines_h, + cpython_autogen_api_impl_h) +from hpy.tools.autogen.hpyslot import autogen_hpyslot_h + +def src_equal(exp, got): + # try to compare two C sources, ignoring whitespace + exp = textwrap.dedent(exp).strip() + got = textwrap.dedent(got).strip() + if exp.split() != got.split(): + diff = difflib.unified_diff(exp.splitlines(), got.splitlines(), + fromfile='expected', + tofile='got') + print() + for line in diff: + print(line) + return False + return True + +@pytest.mark.usefixtures('initargs') +class BaseTestAutogen: + + @pytest.fixture + def initargs(self, tmpdir): + self.tmpdir = tmpdir + + def parse(self, src): + fname = self.tmpdir.join('test_api.h') + # automatically add useful typedefs + src = """ + #define STRINGIFY(X) #X + #define HPy_ID(X) _Pragma(STRINGIFY(id=X)) \\ + + typedef int HPy; + typedef int HPyContext; + """ + src + fname.write(src) + return HPyAPI.parse(fname) + + +class TestHPyAPI(BaseTestAutogen): + + def test_ctx_name(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(2) void* _HPy_AsStruct(HPyContext *ctx, HPy h); + """) + assert api.get_var('h_None').ctx_name() == 'h_None' + assert api.get_func('HPy_Dup').ctx_name() == 'ctx_Dup' + assert api.get_func('_HPy_AsStruct').ctx_name() == 'ctx_AsStruct' + + def test_cpython_name(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(1) long HPyLong_AsLong(HPyContext *ctx, HPy h); + HPy_ID(2) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + assert api.get_func('HPy_Dup').cpython_name is None + assert api.get_func('HPyLong_AsLong').cpython_name == 'PyLong_AsLong' + assert api.get_func('HPy_Add').cpython_name == 'PyNumber_Add' + + def test_hpyslot(self): + api = self.parse(""" + typedef enum { + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + } HPySlot_Slot; + """) + nb_add = api.get_slot('HPy_nb_add') + assert nb_add.value == '7' + assert nb_add.hpyfunc == 'HPyFunc_BINARYFUNC' + # + tp_repr = api.get_slot('HPy_tp_repr') + assert tp_repr.value == '66' + assert tp_repr.hpyfunc == 'HPyFunc_REPRFUNC' + + def test_parse_id(self): + api = self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(1) + long HPyFoo_Bar(HPyContext *ctx, HPy h); + """) + assert len(api.variables) == 1 + assert len(api.functions) == 1 + assert api.variables[0].ctx_index == 0 + assert api.functions[0].ctx_index == 1 + + # don't allow gaps in the sequence of IDs + with pytest.raises(AssertionError): + self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(3) long HPyFoo_Bar(HPyContext *ctx, HPy h); + """) + + # don't allow re-using of IDs + with pytest.raises(AssertionError): + self.parse(""" + HPy_ID(0) HPy h_Foo; + HPy_ID(0) HPy h_Foo; + """) + + # all context members must have an ID + with pytest.raises(ValueError): + self.parse("HPy h_Foo;") + + # pragmas must be of form '#pramga key=value' + with pytest.raises(ValueError): + self.parse("#pragma hello\nHPy h_Foo;") + + # pragmas value must be an integer + with pytest.raises(ValueError): + self.parse("#pragma hello=world\nHPy h_Foo;") + +class TestAutoGen(BaseTestAutogen): + + def test_autogen_ctx_h(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + got = autogen_ctx_h(api).generate() + exp = """ + struct _HPyContext_s { + const char *name; // used just to make debugging and testing easier + void *_private; // used by implementations to store custom data + int abi_version; + HPy h_None; + HPy (*ctx_Add)(HPyContext *ctx, HPy h1, HPy h2); + }; + """ + assert src_equal(exp, got) + + def test_autogen_ctx_def_h(self): + api = self.parse(""" + HPy_ID(0) HPy h_None; + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + """) + got = autogen_ctx_def_h(api).generate() + exp = """ + struct _HPyContext_s g_universal_ctx = { + .name = "HPy Universal ABI (CPython backend)", + ._private = NULL, + .abi_version = HPY_ABI_VERSION, + /* h_None & co. are initialized by init_universal_ctx() */ + .ctx_Add = &ctx_Add, + }; + """ + assert src_equal(exp, got) + + def test_autogen_trampolines_h(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + HPy_ID(1) void HPy_Close(HPyContext *ctx, HPy h); + HPy_ID(2) void* _HPy_AsStruct(HPyContext *ctx, HPy h); + """) + got = autogen_trampolines_h(api).generate() + exp = """ + HPyAPI_FUNC HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2) { + return ctx->ctx_Add ( ctx, h1, h2 ); + } + + HPyAPI_FUNC void HPy_Close(HPyContext *ctx, HPy h) { + ctx->ctx_Close ( ctx, h ); + } + + HPyAPI_FUNC void *_HPy_AsStruct(HPyContext *ctx, HPy h) { + return ctx->ctx_AsStruct ( ctx, h ); + } + """ + assert src_equal(got, exp) + + def test_cpython_api_impl_h(self): + api = self.parse(""" + HPy_ID(0) HPy HPy_Dup(HPyContext *ctx, HPy h); + HPy_ID(1) HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2); + HPy_ID(2) HPy HPyLong_FromLong(HPyContext *ctx, long value); + HPy_ID(3) char* HPyBytes_AsString(HPyContext *ctx, HPy h); + """) + got = cpython_autogen_api_impl_h(api).generate() + exp = """ + HPyAPI_FUNC + HPy HPy_Add(HPyContext *ctx, HPy h1, HPy h2) + { + return _py2h(PyNumber_Add(_h2py(h1), _h2py(h2))); + } + + HPyAPI_FUNC + HPy HPyLong_FromLong(HPyContext *ctx, long value) + { + return _py2h(PyLong_FromLong(value)); + } + + HPyAPI_FUNC + char *HPyBytes_AsString(HPyContext *ctx, HPy h) + { + return PyBytes_AsString(_h2py(h)); + } + """ + assert src_equal(got, exp) + + def test_autogen_hpyslot_h(self): + api = self.parse(""" + typedef enum { + HPy_nb_add = SLOT(7, HPyFunc_BINARYFUNC), + HPy_tp_repr = SLOT(66, HPyFunc_REPRFUNC), + } HPySlot_Slot; + """) + got = autogen_hpyslot_h(api).generate() + exp = """ + typedef enum { + HPy_nb_add = 7, + HPy_tp_repr = 66, + } HPySlot_Slot; + + #define _HPySlot_SIG__HPy_nb_add HPyFunc_BINARYFUNC + #define _HPySlot_SIG__HPy_tp_repr HPyFunc_REPRFUNC + """ + assert src_equal(got, exp) diff --git a/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py b/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py new file mode 100644 index 0000000000..7e0032979d --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/testing/test_hpyfunc.py @@ -0,0 +1,145 @@ +from hpy.tools.autogen.hpyfunc import autogen_hpyfunc_declare_h +from hpy.tools.autogen.hpyfunc import autogen_hpyfunc_trampoline_h +from hpy.tools.autogen.hpyfunc import autogen_ctx_call_i +from hpy.tools.autogen.hpyfunc import autogen_cpython_hpyfunc_trampoline_h +from hpy.tools.autogen.testing.test_autogen import BaseTestAutogen, src_equal + +class TestHPyFunc(BaseTestAutogen): + + def test_parse(self): + api = self.parse(""" + typedef int HPyFunc_Signature; + typedef HPy (*HPyFunc_noargs)(HPyContext *ctx, HPy self); + """) + assert len(api.hpyfunc_typedefs) == 1 + hpyfunc = api.get_hpyfunc_typedef('HPyFunc_noargs') + assert hpyfunc.name == 'HPyFunc_noargs' + assert hpyfunc.base_name() == 'noargs' + + def test_autogen_hpyfunc_declare_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy self); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """) + got = autogen_hpyfunc_declare_h(api).generate() + exp = """ + #define _HPyFunc_DECLARE_HPyFunc_FOO(SYM) static HPy SYM(HPyContext *ctx, HPy self) + #define _HPyFunc_DECLARE_HPyFunc_BAR(SYM) static HPy SYM(HPyContext *ctx, HPy, int) + + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy self); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """ + assert src_equal(got, exp) + + def test_autogen_hpyfunc_trampoline_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + typedef void (*HPyFunc_proc)(HPyContext *ctx, int x); + """) + got = autogen_hpyfunc_trampoline_h(api).generate() + exp = r""" + typedef struct { + cpy_PyObject *arg; + int xy; + cpy_PyObject * result; + } _HPyFunc_args_FOO; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_FOO(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg, int xy) \ + { \ + _HPyFunc_args_FOO a = { arg, xy }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_FOO, (HPyCFunction)IMPL, &a); \ + return a.result; \ + } + + typedef struct { + cpy_PyObject *arg0; + int arg1; + cpy_PyObject * result; + } _HPyFunc_args_BAR; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_BAR(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg0, int arg1) \ + { \ + _HPyFunc_args_BAR a = { arg0, arg1 }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_BAR, (HPyCFunction)IMPL, &a); \ + return a.result; \ + } + + typedef struct { + int x; + } _HPyFunc_args_PROC; + + #define _HPyFunc_TRAMPOLINE_HPyFunc_PROC(SYM, IMPL) \ + static void SYM(int x) \ + { \ + _HPyFunc_args_PROC a = { x }; \ + _HPy_CallRealFunctionFromTrampoline( \ + _ctx_for_trampolines, HPyFunc_PROC, (HPyCFunction)IMPL, &a); \ + return; \ + } + """ + assert src_equal(got, exp) + + def test_autogen_ctx_call_i(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef int (*HPyFunc_bar)(HPyContext *ctx); + typedef int (*HPyFunc_baz)(HPyContext *ctx, HPy, int); + typedef void (*HPyFunc_proc)(HPyContext *ctx, int x); + """) + got = autogen_ctx_call_i(api).generate() + exp = r""" + case HPyFunc_FOO: { + HPyFunc_foo f = (HPyFunc_foo)func; + _HPyFunc_args_FOO *a = (_HPyFunc_args_FOO*)args; + a->result = _h2py(f(ctx, _py2h(a->arg), a->xy)); + return; + } + case HPyFunc_BAR: { + HPyFunc_bar f = (HPyFunc_bar)func; + _HPyFunc_args_BAR *a = (_HPyFunc_args_BAR*)args; + a->result = f(ctx); + return; + } + case HPyFunc_BAZ: { + HPyFunc_baz f = (HPyFunc_baz)func; + _HPyFunc_args_BAZ *a = (_HPyFunc_args_BAZ*)args; + a->result = f(ctx, _py2h(a->arg0), a->arg1); + return; + } + case HPyFunc_PROC: { + HPyFunc_proc f = (HPyFunc_proc)func; + _HPyFunc_args_PROC *a = (_HPyFunc_args_PROC*)args; + f(ctx, a->x); + return; + } + """ + assert src_equal(got, exp) + + def test_autogen_cpython_hpyfunc_trampoline_h(self): + api = self.parse(""" + typedef HPy (*HPyFunc_foo)(HPyContext *ctx, HPy arg, int xy); + typedef HPy (*HPyFunc_bar)(HPyContext *ctx, HPy, int); + """) + got = autogen_cpython_hpyfunc_trampoline_h(api).generate() + exp = r""" + typedef HPy (*_HPyCFunction_FOO)(HPyContext *, HPy, int); + #define _HPyFunc_TRAMPOLINE_HPyFunc_FOO(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg, int xy) \ + { \ + _HPyCFunction_FOO func = (_HPyCFunction_FOO)IMPL; \ + return _h2py(func(_HPyGetContext(), _py2h(arg), xy)); \ + } + typedef HPy (*_HPyCFunction_BAR)(HPyContext *, HPy, int); + #define _HPyFunc_TRAMPOLINE_HPyFunc_BAR(SYM, IMPL) \ + static cpy_PyObject *SYM(cpy_PyObject *arg0, int arg1) \ + { \ + _HPyCFunction_BAR func = (_HPyCFunction_BAR)IMPL; \ + return _h2py(func(_HPyGetContext(), _py2h(arg0), arg1)); \ + } + """ + assert src_equal(got, exp) diff --git a/graalpython/hpy/hpy/tools/autogen/trace.py b/graalpython/hpy/hpy/tools/autogen/trace.py new file mode 100644 index 0000000000..5cdc212081 --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/trace.py @@ -0,0 +1,182 @@ +from copy import deepcopy +from pycparser import c_ast +from .autogenfile import AutoGenFile +from .parse import toC, find_typedecl, get_context_return_type, \ + maybe_make_void, make_void, get_return_constant + + +# We will call the delegate context's function but we still need to unwrap +# the context. This is in contrast to, e.g., the debug mode, where you would +# manually write a wrapper function. Here we can generate that as well. +NO_WRAPPER = { + '_HPy_CallRealFunctionFromTrampoline', + 'HPy_FatalError', +} + +class Ctx2TctxVisitor(c_ast.NodeVisitor): + """Visitor which renames all ctx to tctx""" + + def visit_TypeDecl(self, node): + if node.declname == 'ctx': + node.declname = 'tctx' + self.generic_visit(node) + +def funcnode_with_new_name(node, name): + newnode = deepcopy(node) + typedecl = find_typedecl(newnode) + typedecl.declname = name + return newnode + +def get_trace_wrapper_node(func): + newnode = funcnode_with_new_name(func.node, 'trace_%s' % func.ctx_name()) + maybe_make_void(func, newnode) + # rename ctx to tctx + visitor = Ctx2TctxVisitor() + visitor.visit(newnode) + return newnode + +class autogen_tracer_ctx_init_h(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_ctx_init.h' + + def generate(self): + lines = [] + w = lines.append + # emit the declarations for all the trace_ctx_* functions + for func in self.api.functions: + if func.name not in NO_WRAPPER: + w(toC(get_trace_wrapper_node(func)) + ';') + n_decls = len(self.api.functions) + len(self.api.variables) + w('') + w(f'static inline void trace_ctx_init_info(HPyTraceInfo *info, HPyContext *uctx)') + w('{') + w(f' info->magic_number = HPY_TRACE_MAGIC;') + w(f' info->uctx = uctx;') + w(f' info->call_counts = (uint64_t *)calloc({n_decls}, sizeof(uint64_t));') + w(f' info->durations = (_HPyTime_t *)calloc({n_decls}, sizeof(_HPyTime_t));') + w(f' info->on_enter_func = HPy_NULL;') + w(f' info->on_exit_func = HPy_NULL;') + w('}') + w('') + w(f'static inline void trace_ctx_free_info(HPyTraceInfo *info)') + w('{') + w(f' assert(info->magic_number == HPY_TRACE_MAGIC);') + w(f' free(info->call_counts);') + w(f' free(info->durations);') + w(f' HPy_Close(info->uctx, info->on_enter_func);') + w(f' HPy_Close(info->uctx, info->on_exit_func);') + w('}') + w('') + w(f'static inline void trace_ctx_init_fields(HPyContext *tctx, HPyContext *uctx)') + w('{') + for var in self.api.variables: + name = var.name + w(f' tctx->{name} = uctx->{name};') + for func in self.api.functions: + if func.name in NO_WRAPPER: + name = func.ctx_name() + w(f' tctx->{name} = uctx->{name};') + else: + name = func.ctx_name() + w(f' tctx->{name} = &trace_{name};') + w('}') + return '\n'.join(lines) + + +class autogen_tracer_wrappers(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_wrappers.c' + + def generate(self): + lines = [] + w = lines.append + w('#include "trace_internal.h"') + w('') + for func in self.api.functions: + debug_wrapper = self.gen_trace_wrapper(func) + if debug_wrapper: + w(debug_wrapper) + w('') + return '\n'.join(lines) + + def gen_trace_wrapper(self, func): + if func.name in NO_WRAPPER: + return + + assert not func.is_varargs() + node = get_trace_wrapper_node(func) + const_return = get_return_constant(func) + if const_return: + make_void(node) + signature = toC(node) + rettype = get_context_return_type(node, const_return) + + def get_params(): + lst = [] + for p in node.type.args.params: + if p.name == 'ctx': + lst.append('uctx') + else: + lst.append(p.name) + return ', '.join(lst) + params = get_params() + + lines = [] + w = lines.append + w(signature) + w('{') + w(f' HPyTraceInfo *info = hpy_trace_on_enter(tctx, {func.ctx_index});') + w(f' HPyContext *uctx = info->uctx;') + w(f' _HPyTime_t _ts_start, _ts_end;') + w(f' _HPyClockStatus_t r0, r1;') + w(f' r0 = get_monotonic_clock(&_ts_start);') + if rettype == 'void': + w(f' {func.name}({params});') + else: + w(f' {rettype} res = {func.name}({params});') + w(f' r1 = get_monotonic_clock(&_ts_end);') + w(f' hpy_trace_on_exit(info, {func.ctx_index}, r0, r1, &_ts_start, &_ts_end);') + if rettype != 'void': + w(f' return res;') + w('}') + return '\n'.join(lines) + + +class autogen_trace_func_table_c(AutoGenFile): + PATH = 'hpy/trace/src/autogen_trace_func_table.c' + + def generate(self): + lines = [] + w = lines.append + + n_funcs = len(self.api.functions) + n_decls = n_funcs + len(self.api.variables) + func_table = ['NO_FUNC'] * n_decls + for func in self.api.functions: + name = func.ctx_name() + func_table[func.ctx_index] = f'"{name}"' + + w('#include "trace_internal.h"') + w('') + w(f'#define TRACE_NFUNC {n_funcs}') + w('') + w('#define NO_FUNC ""') + w('static const char *trace_func_table[] = {') + for func in func_table: + w(f' {func},') + w(f' NULL /* sentinel */') + w('};') + w('') + w('int hpy_trace_get_nfunc(void)') + w('{') + w(' return TRACE_NFUNC;') + w('}') + w('') + w('const char * hpy_trace_get_func_name(int idx)') + w('{') + w(f' if (idx >= 0 && idx < {n_decls})') + w(' return trace_func_table[idx];') + w(' return NULL;') + w('}') + w('') + return '\n'.join(lines) + + diff --git a/graalpython/hpy/hpy/tools/autogen/trampolines.py b/graalpython/hpy/hpy/tools/autogen/trampolines.py new file mode 100644 index 0000000000..366f27cd4a --- /dev/null +++ b/graalpython/hpy/hpy/tools/autogen/trampolines.py @@ -0,0 +1,140 @@ +from copy import deepcopy +from .autogenfile import AutoGenFile +from .parse import toC,find_typedecl, get_context_return_type, \ + make_void, get_return_constant +from . import conf + + +class autogen_trampolines_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/universal/autogen_trampolines.h' + + def generate(self): + lines = [] + for func in self.api.functions: + trampoline = self.gen_trampoline(func) + if trampoline: + lines.append(trampoline) + lines.append('') + return '\n'.join(lines) + + def gen_trampoline(self, func): + # HPyAPI_FUNC HPy HPyModule_Create(HPyContext *ctx, HPyModuleDef *def) { + # return ctx->ctx_Module_Create ( ctx, def ); + # } + if func.name in conf.NO_TRAMPOLINES: + return None + const_return = get_return_constant(func) + rettype = get_context_return_type(func.node, const_return) + parts = [] + w = parts.append + w('HPyAPI_FUNC') + w(toC(func.node)) + w('{\n ') + + # trampolines cannot deal with varargs easily + assert not func.is_varargs() + + if rettype == 'void': + w('ctx->%s' % func.ctx_name()) + else: + w('return ctx->%s' % func.ctx_name()) + w('(') + params = [p.name for p in func.node.type.args.params] + w(', '.join(params)) + w(');') + + if const_return: + w('return %s;' % const_return) + + w('\n}') + return ' '.join(parts) + + +class cpython_autogen_api_impl_h(AutoGenFile): + PATH = 'hpy/devel/include/hpy/cpython/autogen_api_impl.h' + GENERATE_CONST_RETURN = True + + def signature(self, func, const_return): + """ + Return the C signature of the impl function. + + In CPython mode, the name it's the same as in public_api: + HPy_Add ==> HPyAPI_FUNC HPy_Add + HPyLong_FromLong ==> HPyAPI_FUNC HPyLong_FromLong + + See also universal_autogen_ctx_impl_h. + """ + sig = toC(func.node) + return 'HPyAPI_FUNC %s' % sig + + def generate(self): + lines = [] + for func in self.api.functions: + if not func.cpython_name: + continue + lines.append(self.gen_implementation(func)) + lines.append('') + return '\n'.join(lines) + + def gen_implementation(self, func): + def call(pyfunc, return_type): + # return _py2h(PyNumber_Add(_h2py(x), _h2py(y))) + args = [] + for p in func.node.type.args.params: + if toC(p.type) == 'HPyContext *': + continue + elif toC(p.type) == 'HPy': + arg = '_h2py(%s)' % p.name + elif toC(p.type) == 'HPyThreadState': + arg = '_h2threads(%s)' % p.name + else: + arg = p.name + args.append(arg) + result = '%s(%s)' % (pyfunc, ', '.join(args)) + if return_type == 'HPy': + result = '_py2h(%s)' % result + elif return_type == 'HPyThreadState': + result = '_threads2h(%s)' % result + return result + # + lines = [] + w = lines.append + pyfunc = func.cpython_name + if not pyfunc: + raise ValueError(f"Cannot generate implementation for {self}") + const_return = get_return_constant(func) + return_type = get_context_return_type(func.node, const_return) + return_stmt = '' if return_type == 'void' else 'return ' + w(self.signature(func, const_return)) + w('{') + w(' %s%s;' % (return_stmt, call(pyfunc, return_type))) + + if self.GENERATE_CONST_RETURN and const_return: + w(' return %s;' % const_return) + + w('}') + return '\n'.join(lines) + + +class universal_autogen_ctx_impl_h(cpython_autogen_api_impl_h): + PATH = 'hpy/universal/src/autogen_ctx_impl.h' + GENERATE_CONST_RETURN = False + + def signature(self, func, const_return): + """ + Return the C signature of the impl function. + + In Universal mode, the name is prefixed by ctx_: + HPy_Add ==> HPyAPI_IMPL ctx_Add + HPyLong_FromLong ==> HPyAPI_IMPL ctx_Long_FromLong + + See also cpython_autogen_api_impl_h. + """ + newnode = deepcopy(func.node) + if const_return: + make_void(newnode) + typedecl = find_typedecl(newnode) + # rename the function + typedecl.declname = func.ctx_name() + sig = toC(newnode) + return 'HPyAPI_IMPL %s' % sig diff --git a/graalpython/hpy/hpy/tools/include_path.py b/graalpython/hpy/hpy/tools/include_path.py new file mode 100644 index 0000000000..61dd06a28d --- /dev/null +++ b/graalpython/hpy/hpy/tools/include_path.py @@ -0,0 +1,4 @@ +"""Prints the include path for the current Python interpreter.""" + +from sysconfig import get_paths as gp +print(gp()['include']) diff --git a/graalpython/hpy/hpy/tools/valgrind/hpy.supp b/graalpython/hpy/hpy/tools/valgrind/hpy.supp new file mode 100644 index 0000000000..c1139552a8 --- /dev/null +++ b/graalpython/hpy/hpy/tools/valgrind/hpy.supp @@ -0,0 +1,44 @@ +{ + <_HPyModuleDef_CreatePyModuleDef_leak> + Memcheck:Leak + match-leak-kinds: definite,indirect + ... + fun:_HPyModuleDef_CreatePyModuleDef + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:ctx_Type_FromSpec + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyUnicode_New.part.42 + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyUnicode_New + ... +} + +{ + + Memcheck:Leak + match-leak-kinds: definite + ... + fun:PyLong_FromLong + ... +} diff --git a/graalpython/hpy/hpy/tools/valgrind/python.supp b/graalpython/hpy/hpy/tools/valgrind/python.supp new file mode 100644 index 0000000000..26a6f22c89 --- /dev/null +++ b/graalpython/hpy/hpy/tools/valgrind/python.supp @@ -0,0 +1,389 @@ +# +# This is a valgrind suppression file that should be used when using valgrind. +# +# Here's an example of running valgrind: +# +# cd python/dist/src +# valgrind --tool=memcheck --suppressions=Misc/valgrind-python.supp \ +# ./python -E -tt ./Lib/test/regrtest.py -u bsddb,network +# +# You must edit Objects/obmalloc.c and uncomment Py_USING_MEMORY_DEBUGGER +# to use the preferred suppressions with Py_ADDRESS_IN_RANGE. +# +# If you do not want to recompile Python, you can uncomment +# suppressions for PyObject_Free and PyObject_Realloc. +# +# See Misc/README.valgrind for more information. + +# all tool names: Addrcheck,Memcheck,cachegrind,helgrind,massif +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Addr4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 4 + Memcheck:Value4 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Invalid read of size 8 (x86_64 aka amd64) + Memcheck:Value8 + fun:Py_ADDRESS_IN_RANGE +} + +{ + ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value + Memcheck:Cond + fun:Py_ADDRESS_IN_RANGE +} + +# +# Leaks (including possible leaks) +# Hmmm, I wonder if this masks some real leaks. I think it does. +# Will need to fix that. +# + +{ + Suppress leaking the GIL. Happens once per process, see comment in ceval.c. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_InitThreads +} + +{ + Suppress leaking the GIL after a fork. + Memcheck:Leak + fun:malloc + fun:PyThread_allocate_lock + fun:PyEval_ReInitThreads +} + +{ + Suppress leaking the autoTLSkey. This looks like it shouldn't leak though. + Memcheck:Leak + fun:malloc + fun:PyThread_create_key + fun:_PyGILState_Init + fun:Py_InitializeEx + fun:Py_Main +} + +{ + Hmmm, is this a real leak or like the GIL? + Memcheck:Leak + fun:malloc + fun:PyThread_ReInitTLS +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:realloc + fun:_PyObject_GC_Resize + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_New + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +{ + Handle PyMalloc confusing valgrind (possibly leaked) + Memcheck:Leak + fun:malloc + fun:_PyObject_GC_NewVar + fun:COMMENT_THIS_LINE_TO_DISABLE_LEAK_WARNING +} + +# +# Non-python specific leaks +# + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:calloc + fun:allocate_dtv + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +{ + Handle pthread issue (possibly leaked) + Memcheck:Leak + fun:memalign + fun:_dl_allocate_tls_storage + fun:_dl_allocate_tls +} + +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Addr4 +### fun:PyObject_Free +###} +### +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Value4 +### fun:PyObject_Free +###} +### +###{ +### ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value +### Memcheck:Cond +### fun:PyObject_Free +###} + +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Addr4 +### fun:PyObject_Realloc +###} +### +###{ +### ADDRESS_IN_RANGE/Invalid read of size 4 +### Memcheck:Value4 +### fun:PyObject_Realloc +###} +### +###{ +### ADDRESS_IN_RANGE/Conditional jump or move depends on uninitialised value +### Memcheck:Cond +### fun:PyObject_Realloc +###} + +### +### All the suppressions below are for errors that occur within libraries +### that Python uses. The problems to not appear to be related to Python's +### use of the libraries. +### + +{ + Generic ubuntu ld problems + Memcheck:Addr8 + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so + obj:/lib/ld-2.4.so +} + +{ + Generic gentoo ld problems + Memcheck:Cond + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so + obj:/lib/ld-2.3.4.so +} + +{ + DBM problems, see test_dbm + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_close +} + +{ + DBM problems, see test_dbm + Memcheck:Value8 + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + DBM problems, see test_dbm + Memcheck:Cond + fun:memmove + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + obj:/usr/lib/libdb1.so.2 + fun:dbm_store + fun:dbm_ass_sub +} + +{ + GDBM problems, see test_gdbm + Memcheck:Param + write(buf) + fun:write + fun:gdbm_open + +} + +{ + ZLIB problems, see test_gzip + Memcheck:Cond + obj:/lib/libz.so.1.2.3 + obj:/lib/libz.so.1.2.3 + fun:deflate +} + +{ + Avoid problems w/readline doing a putenv and leaking on exit + Memcheck:Leak + fun:malloc + fun:xmalloc + fun:sh_set_lines_and_columns + fun:_rl_get_screen_size + fun:_rl_init_terminal_io + obj:/lib/libreadline.so.4.3 + fun:rl_initialize +} + +### +### These occur from somewhere within the SSL, when running +### test_socket_sll. They are too general to leave on by default. +### +###{ +### somewhere in SSL stuff +### Memcheck:Cond +### fun:memset +###} +###{ +### somewhere in SSL stuff +### Memcheck:Value4 +### fun:memset +###} +### +###{ +### somewhere in SSL stuff +### Memcheck:Cond +### fun:MD5_Update +###} +### +###{ +### somewhere in SSL stuff +### Memcheck:Value4 +### fun:MD5_Update +###} + +# +# All of these problems come from using test_socket_ssl +# +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_bin2bn +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:BN_num_bits_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont_word +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BN_mod_exp_mont +} + +{ + from test_socket_ssl + Memcheck:Param + write(buf) + fun:write + obj:/usr/lib/libcrypto.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:RSA_verify +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_set_key_unchecked +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:DES_encrypt2 +} + +{ + from test_socket_ssl + Memcheck:Cond + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Value4 + obj:/usr/lib/libssl.so.0.9.7 +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:BUF_MEM_grow_clean +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:memcpy + fun:ssl3_read_bytes +} + +{ + from test_socket_ssl + Memcheck:Cond + fun:SHA1_Update +} + +{ + from test_socket_ssl + Memcheck:Value4 + fun:SHA1_Update +} diff --git a/graalpython/hpy/hpy/trace/__init__.py b/graalpython/hpy/hpy/trace/__init__.py new file mode 100644 index 0000000000..83d5308790 --- /dev/null +++ b/graalpython/hpy/hpy/trace/__init__.py @@ -0,0 +1,6 @@ +import hpy.universal + +get_call_counts = hpy.universal._trace.get_call_counts +get_durations = hpy.universal._trace.get_durations +set_trace_functions = hpy.universal._trace.set_trace_functions +get_frequency = hpy.universal._trace.get_frequency diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/_tracemod.c b/graalpython/hpy/hpy/trace/src/_tracemod.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/_tracemod.c rename to graalpython/hpy/hpy/trace/src/_tracemod.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h b/graalpython/hpy/hpy/trace/src/autogen_trace_ctx_init.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_ctx_init.h rename to graalpython/hpy/hpy/trace/src/autogen_trace_ctx_init.h diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c b/graalpython/hpy/hpy/trace/src/autogen_trace_func_table.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_func_table.c rename to graalpython/hpy/hpy/trace/src/autogen_trace_func_table.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c b/graalpython/hpy/hpy/trace/src/autogen_trace_wrappers.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/autogen_trace_wrappers.c rename to graalpython/hpy/hpy/trace/src/autogen_trace_wrappers.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/hpy_trace.h b/graalpython/hpy/hpy/trace/src/include/hpy_trace.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/hpy_trace.h rename to graalpython/hpy/hpy/trace/src/include/hpy_trace.h diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/trace_ctx.c b/graalpython/hpy/hpy/trace/src/trace_ctx.c similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/trace_ctx.c rename to graalpython/hpy/hpy/trace/src/trace_ctx.c diff --git a/graalpython/com.oracle.graal.python.jni/src/trace/trace_internal.h b/graalpython/hpy/hpy/trace/src/trace_internal.h similarity index 100% rename from graalpython/com.oracle.graal.python.jni/src/trace/trace_internal.h rename to graalpython/hpy/hpy/trace/src/trace_internal.h diff --git a/graalpython/hpy/hpy/universal/src/api.h b/graalpython/hpy/hpy/universal/src/api.h new file mode 100644 index 0000000000..ea4b8108eb --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/api.h @@ -0,0 +1,18 @@ +#ifndef HPY_API_H +#define HPY_API_H + +#include "hpy.h" + +extern struct _HPyContext_s g_universal_ctx; + +/* declare alloca() */ +#if defined(_MSC_VER) +# include /* for alloca() */ +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) +# include +# endif +#endif + +#endif /* HPY_API_H */ diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i b/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i new file mode 100644 index 0000000000..7c6a759f11 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_call.i @@ -0,0 +1,180 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.hpyfunc.autogen_ctx_call_i + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + + case HPyFunc_NOARGS: { + HPyFunc_noargs f = (HPyFunc_noargs)func; + _HPyFunc_args_NOARGS *a = (_HPyFunc_args_NOARGS*)args; + a->result = _h2py(f(ctx, _py2h(a->self))); + return; + } + case HPyFunc_O: { + HPyFunc_o f = (HPyFunc_o)func; + _HPyFunc_args_O *a = (_HPyFunc_args_O*)args; + a->result = _h2py(f(ctx, _py2h(a->self), _py2h(a->arg))); + return; + } + case HPyFunc_UNARYFUNC: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_BINARYFUNC: { + HPyFunc_binaryfunc f = (HPyFunc_binaryfunc)func; + _HPyFunc_args_BINARYFUNC *a = (_HPyFunc_args_BINARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_TERNARYFUNC: { + HPyFunc_ternaryfunc f = (HPyFunc_ternaryfunc)func; + _HPyFunc_args_TERNARYFUNC *a = (_HPyFunc_args_TERNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_INQUIRY: { + HPyFunc_inquiry f = (HPyFunc_inquiry)func; + _HPyFunc_args_INQUIRY *a = (_HPyFunc_args_INQUIRY*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_LENFUNC: { + HPyFunc_lenfunc f = (HPyFunc_lenfunc)func; + _HPyFunc_args_LENFUNC *a = (_HPyFunc_args_LENFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_SSIZEARGFUNC: { + HPyFunc_ssizeargfunc f = (HPyFunc_ssizeargfunc)func; + _HPyFunc_args_SSIZEARGFUNC *a = (_HPyFunc_args_SSIZEARGFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_SSIZESSIZEARGFUNC: { + HPyFunc_ssizessizeargfunc f = (HPyFunc_ssizessizeargfunc)func; + _HPyFunc_args_SSIZESSIZEARGFUNC *a = (_HPyFunc_args_SSIZESSIZEARGFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1), (a->arg2))); + return; + } + case HPyFunc_SSIZEOBJARGPROC: { + HPyFunc_ssizeobjargproc f = (HPyFunc_ssizeobjargproc)func; + _HPyFunc_args_SSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZEOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_SSIZESSIZEOBJARGPROC: { + HPyFunc_ssizessizeobjargproc f = (HPyFunc_ssizessizeobjargproc)func; + _HPyFunc_args_SSIZESSIZEOBJARGPROC *a = (_HPyFunc_args_SSIZESSIZEOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), (a->arg2), _py2h(a->arg3))); + return; + } + case HPyFunc_OBJOBJARGPROC: { + HPyFunc_objobjargproc f = (HPyFunc_objobjargproc)func; + _HPyFunc_args_OBJOBJARGPROC *a = (_HPyFunc_args_OBJOBJARGPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_FREEFUNC: { + HPyFunc_freefunc f = (HPyFunc_freefunc)func; + _HPyFunc_args_FREEFUNC *a = (_HPyFunc_args_FREEFUNC*)args; + f(ctx, (a->arg0)); + return; + } + case HPyFunc_GETATTRFUNC: { + HPyFunc_getattrfunc f = (HPyFunc_getattrfunc)func; + _HPyFunc_args_GETATTRFUNC *a = (_HPyFunc_args_GETATTRFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_GETATTROFUNC: { + HPyFunc_getattrofunc f = (HPyFunc_getattrofunc)func; + _HPyFunc_args_GETATTROFUNC *a = (_HPyFunc_args_GETATTROFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_SETATTRFUNC: { + HPyFunc_setattrfunc f = (HPyFunc_setattrfunc)func; + _HPyFunc_args_SETATTRFUNC *a = (_HPyFunc_args_SETATTRFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), (a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_SETATTROFUNC: { + HPyFunc_setattrofunc f = (HPyFunc_setattrofunc)func; + _HPyFunc_args_SETATTROFUNC *a = (_HPyFunc_args_SETATTROFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_REPRFUNC: { + HPyFunc_reprfunc f = (HPyFunc_reprfunc)func; + _HPyFunc_args_REPRFUNC *a = (_HPyFunc_args_REPRFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_HASHFUNC: { + HPyFunc_hashfunc f = (HPyFunc_hashfunc)func; + _HPyFunc_args_HASHFUNC *a = (_HPyFunc_args_HASHFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_RICHCMPFUNC: { + HPyFunc_richcmpfunc f = (HPyFunc_richcmpfunc)func; + _HPyFunc_args_RICHCMPFUNC *a = (_HPyFunc_args_RICHCMPFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), (a->arg2))); + return; + } + case HPyFunc_GETITERFUNC: { + HPyFunc_getiterfunc f = (HPyFunc_getiterfunc)func; + _HPyFunc_args_GETITERFUNC *a = (_HPyFunc_args_GETITERFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_ITERNEXTFUNC: { + HPyFunc_iternextfunc f = (HPyFunc_iternextfunc)func; + _HPyFunc_args_ITERNEXTFUNC *a = (_HPyFunc_args_ITERNEXTFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + return; + } + case HPyFunc_DESCRGETFUNC: { + HPyFunc_descrgetfunc f = (HPyFunc_descrgetfunc)func; + _HPyFunc_args_DESCRGETFUNC *a = (_HPyFunc_args_DESCRGETFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_DESCRSETFUNC: { + HPyFunc_descrsetfunc f = (HPyFunc_descrsetfunc)func; + _HPyFunc_args_DESCRSETFUNC *a = (_HPyFunc_args_DESCRSETFUNC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), _py2h(a->arg2))); + return; + } + case HPyFunc_GETTER: { + HPyFunc_getter f = (HPyFunc_getter)func; + _HPyFunc_args_GETTER *a = (_HPyFunc_args_GETTER*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0), (a->arg1))); + return; + } + case HPyFunc_SETTER: { + HPyFunc_setter f = (HPyFunc_setter)func; + _HPyFunc_args_SETTER *a = (_HPyFunc_args_SETTER*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1), (a->arg2))); + return; + } + case HPyFunc_OBJOBJPROC: { + HPyFunc_objobjproc f = (HPyFunc_objobjproc)func; + _HPyFunc_args_OBJOBJPROC *a = (_HPyFunc_args_OBJOBJPROC*)args; + a->result = (f(ctx, _py2h(a->arg0), _py2h(a->arg1))); + return; + } + case HPyFunc_DESTRUCTOR: { + HPyFunc_destructor f = (HPyFunc_destructor)func; + _HPyFunc_args_DESTRUCTOR *a = (_HPyFunc_args_DESTRUCTOR*)args; + f(ctx, _py2h(a->arg0)); + return; + } diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h b/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h new file mode 100644 index 0000000000..4bdeec338e --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_def.h @@ -0,0 +1,207 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.ctx.autogen_ctx_def_h + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + +struct _HPyContext_s g_universal_ctx = { + .name = "HPy Universal ABI (CPython backend)", + ._private = NULL, + .abi_version = HPY_ABI_VERSION, + /* h_None & co. are initialized by init_universal_ctx() */ + .ctx_Dup = &ctx_Dup, + .ctx_Close = &ctx_Close, + .ctx_Long_FromInt32_t = &ctx_Long_FromInt32_t, + .ctx_Long_FromUInt32_t = &ctx_Long_FromUInt32_t, + .ctx_Long_FromInt64_t = &ctx_Long_FromInt64_t, + .ctx_Long_FromUInt64_t = &ctx_Long_FromUInt64_t, + .ctx_Long_FromSize_t = &ctx_Long_FromSize_t, + .ctx_Long_FromSsize_t = &ctx_Long_FromSsize_t, + .ctx_Long_AsInt32_t = &ctx_Long_AsInt32_t, + .ctx_Long_AsUInt32_t = &ctx_Long_AsUInt32_t, + .ctx_Long_AsUInt32_tMask = &ctx_Long_AsUInt32_tMask, + .ctx_Long_AsInt64_t = &ctx_Long_AsInt64_t, + .ctx_Long_AsUInt64_t = &ctx_Long_AsUInt64_t, + .ctx_Long_AsUInt64_tMask = &ctx_Long_AsUInt64_tMask, + .ctx_Long_AsSize_t = &ctx_Long_AsSize_t, + .ctx_Long_AsSsize_t = &ctx_Long_AsSsize_t, + .ctx_Long_AsVoidPtr = &ctx_Long_AsVoidPtr, + .ctx_Long_AsDouble = &ctx_Long_AsDouble, + .ctx_Float_FromDouble = &ctx_Float_FromDouble, + .ctx_Float_AsDouble = &ctx_Float_AsDouble, + .ctx_Bool_FromBool = &ctx_Bool_FromBool, + .ctx_Length = &ctx_Length, + .ctx_Number_Check = &ctx_Number_Check, + .ctx_Add = &ctx_Add, + .ctx_Subtract = &ctx_Subtract, + .ctx_Multiply = &ctx_Multiply, + .ctx_MatrixMultiply = &ctx_MatrixMultiply, + .ctx_FloorDivide = &ctx_FloorDivide, + .ctx_TrueDivide = &ctx_TrueDivide, + .ctx_Remainder = &ctx_Remainder, + .ctx_Divmod = &ctx_Divmod, + .ctx_Power = &ctx_Power, + .ctx_Negative = &ctx_Negative, + .ctx_Positive = &ctx_Positive, + .ctx_Absolute = &ctx_Absolute, + .ctx_Invert = &ctx_Invert, + .ctx_Lshift = &ctx_Lshift, + .ctx_Rshift = &ctx_Rshift, + .ctx_And = &ctx_And, + .ctx_Xor = &ctx_Xor, + .ctx_Or = &ctx_Or, + .ctx_Index = &ctx_Index, + .ctx_Long = &ctx_Long, + .ctx_Float = &ctx_Float, + .ctx_InPlaceAdd = &ctx_InPlaceAdd, + .ctx_InPlaceSubtract = &ctx_InPlaceSubtract, + .ctx_InPlaceMultiply = &ctx_InPlaceMultiply, + .ctx_InPlaceMatrixMultiply = &ctx_InPlaceMatrixMultiply, + .ctx_InPlaceFloorDivide = &ctx_InPlaceFloorDivide, + .ctx_InPlaceTrueDivide = &ctx_InPlaceTrueDivide, + .ctx_InPlaceRemainder = &ctx_InPlaceRemainder, + .ctx_InPlacePower = &ctx_InPlacePower, + .ctx_InPlaceLshift = &ctx_InPlaceLshift, + .ctx_InPlaceRshift = &ctx_InPlaceRshift, + .ctx_InPlaceAnd = &ctx_InPlaceAnd, + .ctx_InPlaceXor = &ctx_InPlaceXor, + .ctx_InPlaceOr = &ctx_InPlaceOr, + .ctx_Callable_Check = &ctx_Callable_Check, + .ctx_CallTupleDict = &ctx_CallTupleDict, + .ctx_Call = &ctx_Call, + .ctx_CallMethod = &ctx_CallMethod, + .ctx_GetIter = &ctx_GetIter, + .ctx_Iter_Next = &ctx_Iter_Next, + .ctx_Iter_Check = &ctx_Iter_Check, + .ctx_FatalError = &ctx_FatalError, + .ctx_Err_SetString = &ctx_Err_SetString, + .ctx_Err_SetObject = &ctx_Err_SetObject, + .ctx_Err_SetFromErrnoWithFilename = &ctx_Err_SetFromErrnoWithFilename, + .ctx_Err_SetFromErrnoWithFilenameObjects = &ctx_Err_SetFromErrnoWithFilenameObjects, + .ctx_Err_Occurred = &ctx_Err_Occurred, + .ctx_Err_ExceptionMatches = &ctx_Err_ExceptionMatches, + .ctx_Err_NoMemory = &ctx_Err_NoMemory, + .ctx_Err_Clear = &ctx_Err_Clear, + .ctx_Err_NewException = &ctx_Err_NewException, + .ctx_Err_NewExceptionWithDoc = &ctx_Err_NewExceptionWithDoc, + .ctx_Err_WarnEx = &ctx_Err_WarnEx, + .ctx_Err_WriteUnraisable = &ctx_Err_WriteUnraisable, + .ctx_IsTrue = &ctx_IsTrue, + .ctx_Type_FromSpec = &ctx_Type_FromSpec, + .ctx_Type_GenericNew = &ctx_Type_GenericNew, + .ctx_GetAttr = &ctx_GetAttr, + .ctx_GetAttr_s = &ctx_GetAttr_s, + .ctx_HasAttr = &ctx_HasAttr, + .ctx_HasAttr_s = &ctx_HasAttr_s, + .ctx_SetAttr = &ctx_SetAttr, + .ctx_SetAttr_s = &ctx_SetAttr_s, + .ctx_GetItem = &ctx_GetItem, + .ctx_GetItem_i = &ctx_GetItem_i, + .ctx_GetItem_s = &ctx_GetItem_s, + .ctx_GetSlice = &ctx_GetSlice, + .ctx_Contains = &ctx_Contains, + .ctx_SetItem = &ctx_SetItem, + .ctx_SetItem_i = &ctx_SetItem_i, + .ctx_SetItem_s = &ctx_SetItem_s, + .ctx_SetSlice = &ctx_SetSlice, + .ctx_DelItem = &ctx_DelItem, + .ctx_DelItem_i = &ctx_DelItem_i, + .ctx_DelItem_s = &ctx_DelItem_s, + .ctx_DelSlice = &ctx_DelSlice, + .ctx_Type = &ctx_Type, + .ctx_TypeCheck = &ctx_TypeCheck, + .ctx_Type_GetName = &ctx_Type_GetName, + .ctx_Type_IsSubtype = &ctx_Type_IsSubtype, + .ctx_Is = &ctx_Is, + .ctx_AsStruct_Object = &ctx_AsStruct_Object, + .ctx_AsStruct_Legacy = &ctx_AsStruct_Legacy, + .ctx_AsStruct_Type = &ctx_AsStruct_Type, + .ctx_AsStruct_Long = &ctx_AsStruct_Long, + .ctx_AsStruct_Float = &ctx_AsStruct_Float, + .ctx_AsStruct_Unicode = &ctx_AsStruct_Unicode, + .ctx_AsStruct_Tuple = &ctx_AsStruct_Tuple, + .ctx_AsStruct_List = &ctx_AsStruct_List, + .ctx_AsStruct_Dict = &ctx_AsStruct_Dict, + .ctx_Type_GetBuiltinShape = &ctx_Type_GetBuiltinShape, + .ctx_New = &ctx_New, + .ctx_Repr = &ctx_Repr, + .ctx_Str = &ctx_Str, + .ctx_ASCII = &ctx_ASCII, + .ctx_Bytes = &ctx_Bytes, + .ctx_RichCompare = &ctx_RichCompare, + .ctx_RichCompareBool = &ctx_RichCompareBool, + .ctx_Hash = &ctx_Hash, + .ctx_Bytes_Check = &ctx_Bytes_Check, + .ctx_Bytes_Size = &ctx_Bytes_Size, + .ctx_Bytes_GET_SIZE = &ctx_Bytes_GET_SIZE, + .ctx_Bytes_AsString = &ctx_Bytes_AsString, + .ctx_Bytes_AS_STRING = &ctx_Bytes_AS_STRING, + .ctx_Bytes_FromString = &ctx_Bytes_FromString, + .ctx_Bytes_FromStringAndSize = &ctx_Bytes_FromStringAndSize, + .ctx_Unicode_FromString = &ctx_Unicode_FromString, + .ctx_Unicode_Check = &ctx_Unicode_Check, + .ctx_Unicode_AsASCIIString = &ctx_Unicode_AsASCIIString, + .ctx_Unicode_AsLatin1String = &ctx_Unicode_AsLatin1String, + .ctx_Unicode_AsUTF8String = &ctx_Unicode_AsUTF8String, + .ctx_Unicode_AsUTF8AndSize = &ctx_Unicode_AsUTF8AndSize, + .ctx_Unicode_FromWideChar = &ctx_Unicode_FromWideChar, + .ctx_Unicode_DecodeFSDefault = &ctx_Unicode_DecodeFSDefault, + .ctx_Unicode_DecodeFSDefaultAndSize = &ctx_Unicode_DecodeFSDefaultAndSize, + .ctx_Unicode_EncodeFSDefault = &ctx_Unicode_EncodeFSDefault, + .ctx_Unicode_ReadChar = &ctx_Unicode_ReadChar, + .ctx_Unicode_DecodeASCII = &ctx_Unicode_DecodeASCII, + .ctx_Unicode_DecodeLatin1 = &ctx_Unicode_DecodeLatin1, + .ctx_Unicode_FromEncodedObject = &ctx_Unicode_FromEncodedObject, + .ctx_Unicode_Substring = &ctx_Unicode_Substring, + .ctx_List_Check = &ctx_List_Check, + .ctx_List_New = &ctx_List_New, + .ctx_List_Append = &ctx_List_Append, + .ctx_List_Insert = &ctx_List_Insert, + .ctx_Dict_Check = &ctx_Dict_Check, + .ctx_Dict_New = &ctx_Dict_New, + .ctx_Dict_Keys = &ctx_Dict_Keys, + .ctx_Dict_Copy = &ctx_Dict_Copy, + .ctx_Tuple_Check = &ctx_Tuple_Check, + .ctx_Tuple_FromArray = &ctx_Tuple_FromArray, + .ctx_Slice_New = &ctx_Slice_New, + .ctx_Slice_Unpack = &ctx_Slice_Unpack, + .ctx_Import_ImportModule = &ctx_Import_ImportModule, + .ctx_Capsule_New = &ctx_Capsule_New, + .ctx_Capsule_Get = &ctx_Capsule_Get, + .ctx_Capsule_IsValid = &ctx_Capsule_IsValid, + .ctx_Capsule_Set = &ctx_Capsule_Set, + .ctx_FromPyObject = &ctx_FromPyObject, + .ctx_AsPyObject = &ctx_AsPyObject, + .ctx_CallRealFunctionFromTrampoline = &ctx_CallRealFunctionFromTrampoline, + .ctx_ListBuilder_New = &ctx_ListBuilder_New, + .ctx_ListBuilder_Set = &ctx_ListBuilder_Set, + .ctx_ListBuilder_Build = &ctx_ListBuilder_Build, + .ctx_ListBuilder_Cancel = &ctx_ListBuilder_Cancel, + .ctx_TupleBuilder_New = &ctx_TupleBuilder_New, + .ctx_TupleBuilder_Set = &ctx_TupleBuilder_Set, + .ctx_TupleBuilder_Build = &ctx_TupleBuilder_Build, + .ctx_TupleBuilder_Cancel = &ctx_TupleBuilder_Cancel, + .ctx_Tracker_New = &ctx_Tracker_New, + .ctx_Tracker_Add = &ctx_Tracker_Add, + .ctx_Tracker_ForgetAll = &ctx_Tracker_ForgetAll, + .ctx_Tracker_Close = &ctx_Tracker_Close, + .ctx_Field_Store = &ctx_Field_Store, + .ctx_Field_Load = &ctx_Field_Load, + .ctx_ReenterPythonExecution = &ctx_ReenterPythonExecution, + .ctx_LeavePythonExecution = &ctx_LeavePythonExecution, + .ctx_Global_Store = &ctx_Global_Store, + .ctx_Global_Load = &ctx_Global_Load, + .ctx_Dump = &ctx_Dump, + .ctx_Compile_s = &ctx_Compile_s, + .ctx_EvalCode = &ctx_EvalCode, + .ctx_ContextVar_New = &ctx_ContextVar_New, + .ctx_ContextVar_Get = &ctx_ContextVar_Get, + .ctx_ContextVar_Set = &ctx_ContextVar_Set, + .ctx_SetCallFunction = &ctx_SetCallFunction, +}; diff --git a/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h b/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h new file mode 100644 index 0000000000..15c6e35b39 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/autogen_ctx_impl.h @@ -0,0 +1,612 @@ + +/* + DO NOT EDIT THIS FILE! + + This file is automatically generated by hpy.tools.autogen.trampolines.universal_autogen_ctx_impl_h + See also hpy.tools.autogen and hpy/tools/public_api.h + + Run this to regenerate: + make autogen + +*/ + +HPyAPI_IMPL HPy ctx_Long_FromSize_t(HPyContext *ctx, size_t value) +{ + return _py2h(PyLong_FromSize_t(value)); +} + +HPyAPI_IMPL HPy ctx_Long_FromSsize_t(HPyContext *ctx, HPy_ssize_t value) +{ + return _py2h(PyLong_FromSsize_t(value)); +} + +HPyAPI_IMPL size_t ctx_Long_AsSize_t(HPyContext *ctx, HPy h) +{ + return PyLong_AsSize_t(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Long_AsSsize_t(HPyContext *ctx, HPy h) +{ + return PyLong_AsSsize_t(_h2py(h)); +} + +HPyAPI_IMPL void *ctx_Long_AsVoidPtr(HPyContext *ctx, HPy h) +{ + return PyLong_AsVoidPtr(_h2py(h)); +} + +HPyAPI_IMPL double ctx_Long_AsDouble(HPyContext *ctx, HPy h) +{ + return PyLong_AsDouble(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Float_FromDouble(HPyContext *ctx, double v) +{ + return _py2h(PyFloat_FromDouble(v)); +} + +HPyAPI_IMPL double ctx_Float_AsDouble(HPyContext *ctx, HPy h) +{ + return PyFloat_AsDouble(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Bool_FromBool(HPyContext *ctx, bool v) +{ + return _py2h(PyBool_FromLong(v)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Length(HPyContext *ctx, HPy h) +{ + return PyObject_Length(_h2py(h)); +} + +HPyAPI_IMPL int ctx_Number_Check(HPyContext *ctx, HPy h) +{ + return PyNumber_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Add(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Add(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Subtract(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Subtract(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Multiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Multiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_MatrixMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_MatrixMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_FloorDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_FloorDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_TrueDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_TrueDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Remainder(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Remainder(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Divmod(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Divmod(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Power(HPyContext *ctx, HPy h1, HPy h2, HPy h3) +{ + return _py2h(PyNumber_Power(_h2py(h1), _h2py(h2), _h2py(h3))); +} + +HPyAPI_IMPL HPy ctx_Negative(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Negative(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Positive(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Positive(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Absolute(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Absolute(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Invert(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Invert(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Lshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Lshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Rshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Rshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_And(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_And(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Xor(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Xor(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Or(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_Or(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_Index(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Index(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Long(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Long(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_Float(HPyContext *ctx, HPy h1) +{ + return _py2h(PyNumber_Float(_h2py(h1))); +} + +HPyAPI_IMPL HPy ctx_InPlaceAdd(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceAdd(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceSubtract(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceSubtract(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceMatrixMultiply(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceMatrixMultiply(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceFloorDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceFloorDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceTrueDivide(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceTrueDivide(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceRemainder(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceRemainder(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlacePower(HPyContext *ctx, HPy h1, HPy h2, HPy h3) +{ + return _py2h(PyNumber_InPlacePower(_h2py(h1), _h2py(h2), _h2py(h3))); +} + +HPyAPI_IMPL HPy ctx_InPlaceLshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceLshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceRshift(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceRshift(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceAnd(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceAnd(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceXor(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceXor(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL HPy ctx_InPlaceOr(HPyContext *ctx, HPy h1, HPy h2) +{ + return _py2h(PyNumber_InPlaceOr(_h2py(h1), _h2py(h2))); +} + +HPyAPI_IMPL int ctx_Callable_Check(HPyContext *ctx, HPy h) +{ + return PyCallable_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_GetIter(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_GetIter(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Iter_Next(HPyContext *ctx, HPy obj) +{ + return _py2h(PyIter_Next(_h2py(obj))); +} + +HPyAPI_IMPL int ctx_Iter_Check(HPyContext *ctx, HPy obj) +{ + return PyIter_Check(_h2py(obj)); +} + +HPyAPI_IMPL void ctx_Err_SetString(HPyContext *ctx, HPy h_type, const char *utf8_message) +{ + PyErr_SetString(_h2py(h_type), utf8_message); +} + +HPyAPI_IMPL void ctx_Err_SetObject(HPyContext *ctx, HPy h_type, HPy h_value) +{ + PyErr_SetObject(_h2py(h_type), _h2py(h_value)); +} + +HPyAPI_IMPL HPy ctx_Err_SetFromErrnoWithFilename(HPyContext *ctx, HPy h_type, const char *filename_fsencoded) +{ + return _py2h(PyErr_SetFromErrnoWithFilename(_h2py(h_type), filename_fsencoded)); +} + +HPyAPI_IMPL void ctx_Err_SetFromErrnoWithFilenameObjects(HPyContext *ctx, HPy h_type, HPy filename1, HPy filename2) +{ + PyErr_SetFromErrnoWithFilenameObjects(_h2py(h_type), _h2py(filename1), _h2py(filename2)); +} + +HPyAPI_IMPL int ctx_Err_ExceptionMatches(HPyContext *ctx, HPy exc) +{ + return PyErr_ExceptionMatches(_h2py(exc)); +} + +HPyAPI_IMPL void ctx_Err_NoMemory(HPyContext *ctx) +{ + PyErr_NoMemory(); +} + +HPyAPI_IMPL void ctx_Err_Clear(HPyContext *ctx) +{ + PyErr_Clear(); +} + +HPyAPI_IMPL HPy ctx_Err_NewException(HPyContext *ctx, const char *utf8_name, HPy base, HPy dict) +{ + return _py2h(PyErr_NewException(utf8_name, _h2py(base), _h2py(dict))); +} + +HPyAPI_IMPL HPy ctx_Err_NewExceptionWithDoc(HPyContext *ctx, const char *utf8_name, const char *utf8_doc, HPy base, HPy dict) +{ + return _py2h(PyErr_NewExceptionWithDoc(utf8_name, utf8_doc, _h2py(base), _h2py(dict))); +} + +HPyAPI_IMPL int ctx_Err_WarnEx(HPyContext *ctx, HPy category, const char *utf8_message, HPy_ssize_t stack_level) +{ + return PyErr_WarnEx(_h2py(category), utf8_message, stack_level); +} + +HPyAPI_IMPL void ctx_Err_WriteUnraisable(HPyContext *ctx, HPy obj) +{ + PyErr_WriteUnraisable(_h2py(obj)); +} + +HPyAPI_IMPL int ctx_IsTrue(HPyContext *ctx, HPy h) +{ + return PyObject_IsTrue(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_GetAttr(HPyContext *ctx, HPy obj, HPy name) +{ + return _py2h(PyObject_GetAttr(_h2py(obj), _h2py(name))); +} + +HPyAPI_IMPL HPy ctx_GetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name) +{ + return _py2h(PyObject_GetAttrString(_h2py(obj), utf8_name)); +} + +HPyAPI_IMPL int ctx_HasAttr(HPyContext *ctx, HPy obj, HPy name) +{ + return PyObject_HasAttr(_h2py(obj), _h2py(name)); +} + +HPyAPI_IMPL int ctx_HasAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name) +{ + return PyObject_HasAttrString(_h2py(obj), utf8_name); +} + +HPyAPI_IMPL int ctx_SetAttr(HPyContext *ctx, HPy obj, HPy name, HPy value) +{ + return PyObject_SetAttr(_h2py(obj), _h2py(name), _h2py(value)); +} + +HPyAPI_IMPL int ctx_SetAttr_s(HPyContext *ctx, HPy obj, const char *utf8_name, HPy value) +{ + return PyObject_SetAttrString(_h2py(obj), utf8_name, _h2py(value)); +} + +HPyAPI_IMPL HPy ctx_GetItem(HPyContext *ctx, HPy obj, HPy key) +{ + return _py2h(PyObject_GetItem(_h2py(obj), _h2py(key))); +} + +HPyAPI_IMPL HPy ctx_GetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PySequence_GetSlice(_h2py(obj), start, end)); +} + +HPyAPI_IMPL int ctx_Contains(HPyContext *ctx, HPy container, HPy key) +{ + return PySequence_Contains(_h2py(container), _h2py(key)); +} + +HPyAPI_IMPL int ctx_SetItem(HPyContext *ctx, HPy obj, HPy key, HPy value) +{ + return PyObject_SetItem(_h2py(obj), _h2py(key), _h2py(value)); +} + +HPyAPI_IMPL int ctx_SetSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end, HPy value) +{ + return PySequence_SetSlice(_h2py(obj), start, end, _h2py(value)); +} + +HPyAPI_IMPL int ctx_DelItem(HPyContext *ctx, HPy obj, HPy key) +{ + return PyObject_DelItem(_h2py(obj), _h2py(key)); +} + +HPyAPI_IMPL int ctx_DelSlice(HPyContext *ctx, HPy obj, HPy_ssize_t start, HPy_ssize_t end) +{ + return PySequence_DelSlice(_h2py(obj), start, end); +} + +HPyAPI_IMPL HPy ctx_Repr(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Repr(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Str(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Str(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_ASCII(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_ASCII(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_Bytes(HPyContext *ctx, HPy obj) +{ + return _py2h(PyObject_Bytes(_h2py(obj))); +} + +HPyAPI_IMPL HPy ctx_RichCompare(HPyContext *ctx, HPy v, HPy w, int op) +{ + return _py2h(PyObject_RichCompare(_h2py(v), _h2py(w), op)); +} + +HPyAPI_IMPL int ctx_RichCompareBool(HPyContext *ctx, HPy v, HPy w, int op) +{ + return PyObject_RichCompareBool(_h2py(v), _h2py(w), op); +} + +HPyAPI_IMPL HPy_hash_t ctx_Hash(HPyContext *ctx, HPy obj) +{ + return PyObject_Hash(_h2py(obj)); +} + +HPyAPI_IMPL int ctx_Bytes_Check(HPyContext *ctx, HPy h) +{ + return PyBytes_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Bytes_Size(HPyContext *ctx, HPy h) +{ + return PyBytes_Size(_h2py(h)); +} + +HPyAPI_IMPL HPy_ssize_t ctx_Bytes_GET_SIZE(HPyContext *ctx, HPy h) +{ + return PyBytes_GET_SIZE(_h2py(h)); +} + +HPyAPI_IMPL const char *ctx_Bytes_AsString(HPyContext *ctx, HPy h) +{ + return PyBytes_AsString(_h2py(h)); +} + +HPyAPI_IMPL const char *ctx_Bytes_AS_STRING(HPyContext *ctx, HPy h) +{ + return PyBytes_AS_STRING(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Bytes_FromString(HPyContext *ctx, const char *bytes) +{ + return _py2h(PyBytes_FromString(bytes)); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromString(HPyContext *ctx, const char *utf8) +{ + return _py2h(PyUnicode_FromString(utf8)); +} + +HPyAPI_IMPL int ctx_Unicode_Check(HPyContext *ctx, HPy h) +{ + return PyUnicode_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsASCIIString(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsASCIIString(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsLatin1String(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsLatin1String(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Unicode_AsUTF8String(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_AsUTF8String(_h2py(h))); +} + +HPyAPI_IMPL const char *ctx_Unicode_AsUTF8AndSize(HPyContext *ctx, HPy h, HPy_ssize_t *size) +{ + return PyUnicode_AsUTF8AndSize(_h2py(h), size); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromWideChar(HPyContext *ctx, const wchar_t *w, HPy_ssize_t size) +{ + return _py2h(PyUnicode_FromWideChar(w, size)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeFSDefault(HPyContext *ctx, const char *v) +{ + return _py2h(PyUnicode_DecodeFSDefault(v)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeFSDefaultAndSize(HPyContext *ctx, const char *v, HPy_ssize_t size) +{ + return _py2h(PyUnicode_DecodeFSDefaultAndSize(v, size)); +} + +HPyAPI_IMPL HPy ctx_Unicode_EncodeFSDefault(HPyContext *ctx, HPy h) +{ + return _py2h(PyUnicode_EncodeFSDefault(_h2py(h))); +} + +HPyAPI_IMPL HPy_UCS4 ctx_Unicode_ReadChar(HPyContext *ctx, HPy h, HPy_ssize_t index) +{ + return PyUnicode_ReadChar(_h2py(h), index); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeASCII(HPyContext *ctx, const char *ascii, HPy_ssize_t size, const char *errors) +{ + return _py2h(PyUnicode_DecodeASCII(ascii, size, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_DecodeLatin1(HPyContext *ctx, const char *latin1, HPy_ssize_t size, const char *errors) +{ + return _py2h(PyUnicode_DecodeLatin1(latin1, size, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_FromEncodedObject(HPyContext *ctx, HPy obj, const char *encoding, const char *errors) +{ + return _py2h(PyUnicode_FromEncodedObject(_h2py(obj), encoding, errors)); +} + +HPyAPI_IMPL HPy ctx_Unicode_Substring(HPyContext *ctx, HPy str, HPy_ssize_t start, HPy_ssize_t end) +{ + return _py2h(PyUnicode_Substring(_h2py(str), start, end)); +} + +HPyAPI_IMPL int ctx_List_Check(HPyContext *ctx, HPy h) +{ + return PyList_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_List_New(HPyContext *ctx, HPy_ssize_t len) +{ + return _py2h(PyList_New(len)); +} + +HPyAPI_IMPL int ctx_List_Append(HPyContext *ctx, HPy h_list, HPy h_item) +{ + return PyList_Append(_h2py(h_list), _h2py(h_item)); +} + +HPyAPI_IMPL int ctx_List_Insert(HPyContext *ctx, HPy h_list, HPy_ssize_t index, HPy h_item) +{ + return PyList_Insert(_h2py(h_list), index, _h2py(h_item)); +} + +HPyAPI_IMPL int ctx_Dict_Check(HPyContext *ctx, HPy h) +{ + return PyDict_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Dict_New(HPyContext *ctx) +{ + return _py2h(PyDict_New()); +} + +HPyAPI_IMPL HPy ctx_Dict_Keys(HPyContext *ctx, HPy h) +{ + return _py2h(PyDict_Keys(_h2py(h))); +} + +HPyAPI_IMPL HPy ctx_Dict_Copy(HPyContext *ctx, HPy h) +{ + return _py2h(PyDict_Copy(_h2py(h))); +} + +HPyAPI_IMPL int ctx_Tuple_Check(HPyContext *ctx, HPy h) +{ + return PyTuple_Check(_h2py(h)); +} + +HPyAPI_IMPL HPy ctx_Slice_New(HPyContext *ctx, HPy start, HPy stop, HPy step) +{ + return _py2h(PySlice_New(_h2py(start), _h2py(stop), _h2py(step))); +} + +HPyAPI_IMPL int ctx_Slice_Unpack(HPyContext *ctx, HPy slice, HPy_ssize_t *start, HPy_ssize_t *stop, HPy_ssize_t *step) +{ + return PySlice_Unpack(_h2py(slice), start, stop, step); +} + +HPyAPI_IMPL HPy ctx_Import_ImportModule(HPyContext *ctx, const char *utf8_name) +{ + return _py2h(PyImport_ImportModule(utf8_name)); +} + +HPyAPI_IMPL int ctx_Capsule_IsValid(HPyContext *ctx, HPy capsule, const char *utf8_name) +{ + return PyCapsule_IsValid(_h2py(capsule), utf8_name); +} + +HPyAPI_IMPL void ctx_ReenterPythonExecution(HPyContext *ctx, HPyThreadState state) +{ + PyEval_RestoreThread(_h2threads(state)); +} + +HPyAPI_IMPL HPyThreadState ctx_LeavePythonExecution(HPyContext *ctx) +{ + return _threads2h(PyEval_SaveThread()); +} + +HPyAPI_IMPL HPy ctx_EvalCode(HPyContext *ctx, HPy code, HPy globals, HPy locals) +{ + return _py2h(PyEval_EvalCode(_h2py(code), _h2py(globals), _h2py(locals))); +} + +HPyAPI_IMPL HPy ctx_ContextVar_New(HPyContext *ctx, const char *name, HPy default_value) +{ + return _py2h(PyContextVar_New(name, _h2py(default_value))); +} + +HPyAPI_IMPL HPy ctx_ContextVar_Set(HPyContext *ctx, HPy context_var, HPy value) +{ + return _py2h(PyContextVar_Set(_h2py(context_var), _h2py(value))); +} + diff --git a/graalpython/hpy/hpy/universal/src/ctx.c b/graalpython/hpy/hpy/universal/src/ctx.c new file mode 100644 index 0000000000..4e9860233b --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx.c @@ -0,0 +1,10 @@ +#include "hpy.h" +#include "handles.h" + +#include "hpy/runtime/ctx_funcs.h" +#include "hpy/runtime/ctx_type.h" +#include "ctx_meth.h" +#include "ctx_misc.h" + +#include "autogen_ctx_impl.h" +#include "autogen_ctx_def.h" diff --git a/graalpython/hpy/hpy/universal/src/ctx_meth.c b/graalpython/hpy/hpy/universal/src/ctx_meth.c new file mode 100644 index 0000000000..923f81e23f --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_meth.c @@ -0,0 +1,136 @@ +#include +#include "ctx_meth.h" +#include "hpy/runtime/ctx_type.h" +#include "hpy/runtime/ctx_module.h" +#include "handles.h" + +static void _buffer_h2py(HPyContext *ctx, const HPy_buffer *src, Py_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_AsPyObject(ctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +static void _buffer_py2h(HPyContext *ctx, const Py_buffer *src, HPy_buffer *dest) +{ + dest->buf = src->buf; + dest->obj = HPy_FromPyObject(ctx, src->obj); + dest->len = src->len; + dest->itemsize = src->itemsize; + dest->readonly = src->readonly; + dest->ndim = src->ndim; + dest->format = src->format; + dest->shape = src->shape; + dest->strides = src->strides; + dest->suboffsets = src->suboffsets; + dest->internal = src->internal; +} + +HPyAPI_IMPL void +ctx_CallRealFunctionFromTrampoline(HPyContext *ctx, HPyFunc_Signature sig, + HPyCFunction func, void *args) +{ + switch (sig) { + case HPyFunc_VARARGS: { + HPyFunc_varargs f = (HPyFunc_varargs)func; + _HPyFunc_args_VARARGS *a = (_HPyFunc_args_VARARGS*)args; + HPy *h_args = (HPy *)alloca(a->nargs * sizeof(HPy)); + for (HPy_ssize_t i = 0; i < a->nargs; i++) { + h_args[i] = _py2h(a->args[i]); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, a->nargs)); + return; + } + case HPyFunc_KEYWORDS: { + HPyFunc_keywords f = (HPyFunc_keywords)func; + _HPyFunc_args_KEYWORDS *a = (_HPyFunc_args_KEYWORDS*)args; + size_t n_kwnames = a->kwnames != NULL ? PyTuple_GET_SIZE(a->kwnames) : 0; + size_t nargs = PyVectorcall_NARGS(a->nargsf); + size_t nargs_with_kw = nargs + n_kwnames; + HPy *h_args = (HPy *)alloca(nargs_with_kw * sizeof(HPy)); + for (size_t i = 0; i < nargs_with_kw; i++) { + h_args[i] = _py2h(a->args[i]); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kwnames))); + return; + } + case HPyFunc_INITPROC: { + HPyFunc_initproc f = (HPyFunc_initproc)func; + _HPyFunc_args_INITPROC *a = (_HPyFunc_args_INITPROC*)args; + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + HPy *h_args = (HPy *)alloca(nargs * sizeof(HPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i)); + } + a->result = f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kw)); + return; + } + case HPyFunc_NEWFUNC: { + HPyFunc_newfunc f = (HPyFunc_newfunc)func; + _HPyFunc_args_NEWFUNC *a = (_HPyFunc_args_NEWFUNC*)args; + Py_ssize_t nargs = PyTuple_GET_SIZE(a->args); + HPy *h_args = (HPy *)alloca(nargs * sizeof(HPy)); + for (Py_ssize_t i = 0; i < nargs; i++) { + h_args[i] = _py2h(PyTuple_GET_ITEM(a->args, i)); + } + a->result = _h2py(f(ctx, _py2h(a->self), h_args, nargs, _py2h(a->kw))); + return; + } + case HPyFunc_GETBUFFERPROC: { + HPyFunc_getbufferproc f = (HPyFunc_getbufferproc)func; + _HPyFunc_args_GETBUFFERPROC *a = (_HPyFunc_args_GETBUFFERPROC*)args; + HPy_buffer hbuf; + a->result = f(ctx, _py2h(a->self), &hbuf, a->flags); + if (a->result < 0) { + a->view->obj = NULL; + return; + } + _buffer_h2py(ctx, &hbuf, a->view); + HPy_Close(ctx, hbuf.obj); + return; + } + case HPyFunc_RELEASEBUFFERPROC: { + HPyFunc_releasebufferproc f = (HPyFunc_releasebufferproc)func; + _HPyFunc_args_RELEASEBUFFERPROC *a = (_HPyFunc_args_RELEASEBUFFERPROC*)args; + HPy_buffer hbuf; + _buffer_py2h(ctx, a->view, &hbuf); + f(ctx, _py2h(a->self), &hbuf); + // XXX: copy back from hbuf? + HPy_Close(ctx, hbuf.obj); + return; + } + case HPyFunc_TRAVERSEPROC: { + HPyFunc_traverseproc f = (HPyFunc_traverseproc)func; + _HPyFunc_args_TRAVERSEPROC *a = (_HPyFunc_args_TRAVERSEPROC*)args; + a->result = call_traverseproc_from_trampoline(f, a->self, + a->visit, a->arg); + return; + } + case HPyFunc_CAPSULE_DESTRUCTOR: { + HPyFunc_Capsule_Destructor f = (HPyFunc_Capsule_Destructor)func; + PyObject *capsule = (PyObject *)args; + const char *name = PyCapsule_GetName(capsule); + f(name, PyCapsule_GetPointer(capsule, name), + PyCapsule_GetContext(capsule)); + return; + } + case HPyFunc_MOD_CREATE: { + HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func; + _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args; + a->result = _h2py(f(ctx, _py2h(a->arg0))); + _HPyModule_CheckCreateSlotResult(&a->result); + return; + } +#include "autogen_ctx_call.i" + default: + Py_FatalError("Unsupported HPyFunc_Signature in ctx_meth.c"); + } +} diff --git a/graalpython/hpy/hpy/universal/src/ctx_meth.h b/graalpython/hpy/hpy/universal/src/ctx_meth.h new file mode 100644 index 0000000000..16e8a4b47c --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_meth.h @@ -0,0 +1,6 @@ +#include "hpy.h" +#include "api.h" + +HPyAPI_IMPL void +ctx_CallRealFunctionFromTrampoline(HPyContext *ctx, HPyFunc_Signature sig, + HPyCFunction func, void *args); diff --git a/graalpython/hpy/hpy/universal/src/ctx_misc.c b/graalpython/hpy/hpy/universal/src/ctx_misc.c new file mode 100644 index 0000000000..753354d43c --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_misc.c @@ -0,0 +1,83 @@ +#include +#include "hpy.h" +#include "handles.h" +#include "ctx_misc.h" + +HPyAPI_IMPL HPy +ctx_FromPyObject(HPyContext *ctx, cpy_PyObject *obj) +{ + Py_XINCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL cpy_PyObject * +ctx_AsPyObject(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XINCREF(obj); + return obj; +} + +HPyAPI_IMPL void +ctx_Close(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XDECREF(obj); +} + +HPyAPI_IMPL HPy +ctx_Dup(HPyContext *ctx, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XINCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL void +ctx_Field_Store(HPyContext *ctx, HPy target_object, HPyField *target_field, HPy h) +{ + PyObject *obj = _h2py(h); + PyObject *target_py_obj = _hf2py(*target_field); + Py_XINCREF(obj); + *target_field = _py2hf(obj); + Py_XDECREF(target_py_obj); +} + +HPyAPI_IMPL HPy +ctx_Field_Load(HPyContext *ctx, HPy source_object, HPyField source_field) +{ + PyObject *obj = _hf2py(source_field); + Py_INCREF(obj); + return _py2h(obj); +} + + +HPyAPI_IMPL void +ctx_Global_Store(HPyContext *ctx, HPyGlobal *global, HPy h) +{ + PyObject *obj = _h2py(h); + Py_XDECREF(_hg2py(*global)); + Py_XINCREF(obj); + *global = _py2hg(obj); +} + +HPyAPI_IMPL HPy +ctx_Global_Load(HPyContext *ctx, HPyGlobal global) +{ + PyObject *obj = _hg2py(global); + Py_INCREF(obj); + return _py2h(obj); +} + +HPyAPI_IMPL void +ctx_FatalError(HPyContext *ctx, const char *message) +{ + Py_FatalError(message); +} + +HPyAPI_IMPL int +ctx_Type_IsSubtype(HPyContext *ctx, HPy sub, HPy type) +{ + return PyType_IsSubtype((PyTypeObject *)_h2py(sub), + (PyTypeObject *)_h2py(type)); +} diff --git a/graalpython/hpy/hpy/universal/src/ctx_misc.h b/graalpython/hpy/hpy/universal/src/ctx_misc.h new file mode 100644 index 0000000000..deb27ec5e5 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/ctx_misc.h @@ -0,0 +1,20 @@ +#ifndef HPY_CTX_MISC_H +#define HPY_CTX_MISC_H + +#include "hpy.h" +#include "api.h" + +HPyAPI_IMPL HPy ctx_FromPyObject(HPyContext *ctx, cpy_PyObject *obj); +HPyAPI_IMPL cpy_PyObject *ctx_AsPyObject(HPyContext *ctx, HPy h); +HPyAPI_IMPL void ctx_Close(HPyContext *ctx, HPy h); +HPyAPI_IMPL HPy ctx_Dup(HPyContext *ctx, HPy h); +HPyAPI_IMPL void ctx_Field_Store(HPyContext *ctx, HPy target_object, + HPyField *target_field, HPy h); +HPyAPI_IMPL HPy ctx_Field_Load(HPyContext *ctx, HPy source_object, + HPyField source_field); +HPyAPI_IMPL void ctx_Global_Store(HPyContext *ctx, HPyGlobal *global, HPy h); +HPyAPI_IMPL HPy ctx_Global_Load(HPyContext *ctx, HPyGlobal global); +HPyAPI_IMPL void ctx_FatalError(HPyContext *ctx, const char *message); +HPyAPI_IMPL int ctx_Type_IsSubtype(HPyContext *ctx, HPy sub, HPy type); + +#endif /* HPY_CTX_MISC_H */ diff --git a/graalpython/hpy/hpy/universal/src/handles.h b/graalpython/hpy/hpy/universal/src/handles.h new file mode 100644 index 0000000000..3d0282d786 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/handles.h @@ -0,0 +1,58 @@ +#ifndef HPY_HANDLES_H +#define HPY_HANDLES_H + +#include +#include "hpy.h" + +// The main reason for +1/-1 is to make sure that if people casts HPy to +// PyObject* directly, things explode. Moreover, with this we can easily +// distinguish normal and debug handles in gdb, by only looking at the last +// bit. + +static inline HPy _py2h(PyObject *obj) { + if (obj == NULL) + return HPy_NULL; + return (HPy){(HPy_ssize_t)obj + 1}; +} + +static inline PyObject *_h2py(HPy h) { + if HPy_IsNull(h) + return NULL; + return (PyObject *)(h._i - 1); +} + +static inline HPyField _py2hf(PyObject *obj) +{ + HPy h = _py2h(obj); + return (HPyField){ ._i = h._i }; +} + +static inline PyObject * _hf2py(HPyField hf) +{ + HPy h = { ._i = hf._i }; + return _h2py(h); +} + +static inline HPyThreadState _threads2h(PyThreadState* s) +{ + return (HPyThreadState) { (intptr_t) s }; +} + +static inline PyThreadState* _h2threads(HPyThreadState h) +{ + return (PyThreadState*) h._i; +} + +static inline HPyGlobal _py2hg(PyObject *obj) +{ + HPy h = _py2h(obj); + return (HPyGlobal){ ._i = h._i }; +} + +static inline PyObject * _hg2py(HPyGlobal hf) +{ + HPy h = { ._i = hf._i }; + return _h2py(h); +} + +#endif /* HPY_HANDLES_H */ diff --git a/graalpython/hpy/hpy/universal/src/hpymodule.c b/graalpython/hpy/hpy/universal/src/hpymodule.c new file mode 100644 index 0000000000..b86e418e61 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/hpymodule.c @@ -0,0 +1,691 @@ +#define PY_SSIZE_T_CLEAN +#include +#ifdef MS_WIN32 +# include +# include "misc_win32.h" +#else +# include +// required for strncasecmp +#include +#endif +#include +#include + + +#include "api.h" +#include "handles.h" +#include "hpy/version.h" +#include "hpy_debug.h" +#include "hpy_trace.h" +#include "hpy/runtime/ctx_module.h" + +#ifdef PYPY_VERSION +# error "Cannot build hpy.universal on top of PyPy. PyPy comes with its own version of it" +#endif +#ifdef GRAALVM_PYTHON +# error "Cannot build hpy.universal on top of GraalPy. GraalPy comes with its own version of it" +#endif + +static const char *hpy_mode_names[] = { + "MODE_UNIVERSAL", + "MODE_DEBUG", + "MODE_TRACE", + // "MODE_DEBUG_TRACE", + // "MODE_TRACE_DEBUG", + NULL +}; + +typedef enum { + MODE_INVALID = -1, + MODE_UNIVERSAL = 0, + MODE_DEBUG = 1, + MODE_TRACE = 2, + /* We do currently not test the combinations of debug and trace mode, so we + do not offer them right now. This may change in future. */ + // MODE_DEBUG_TRACE = 3, + // MODE_TRACE_DEBUG = 4 +} HPyMode; + +typedef uint32_t (*VersionGetterFuncPtr)(void); +typedef HPyModuleDef* (*InitFuncPtr)(void); +typedef void (*InitContextFuncPtr)(HPyContext*); + +static const char *init_prefix = "HPyInit"; +static const char *init_ctx_prefix = "HPyInitGlobalContext_"; + +static inline int +_hpy_strncmp_ignore_case(const char *s0, const char *s1, size_t n) +{ +#ifdef MS_WIN32 + return _strnicmp(s0, s1, n); +#else + return strncasecmp(s0, s1, n); +#endif +} + +static HPyContext * get_context(HPyMode mode) +{ + switch (mode) + { + case MODE_INVALID: + return NULL; + case MODE_DEBUG: + return hpy_debug_get_ctx(&g_universal_ctx); + case MODE_TRACE: + return hpy_trace_get_ctx(&g_universal_ctx); + // case MODE_DEBUG_TRACE: + // return hpy_debug_get_ctx(hpy_trace_get_ctx(&g_universal_ctx)); + // case MODE_TRACE_DEBUG: + // return hpy_trace_get_ctx(hpy_debug_get_ctx(&g_universal_ctx)); + default: + return &g_universal_ctx; + } +} + +static PyObject * +get_encoded_name(PyObject *name) { + PyObject *tmp; + PyObject *encoded = NULL; + PyObject *modname = NULL; + Py_ssize_t name_len, lastdot; + + /* Get the short name (substring after last dot) */ + name_len = PyUnicode_GetLength(name); + lastdot = PyUnicode_FindChar(name, '.', 0, name_len, -1); + if (lastdot < -1) { + return NULL; + } else if (lastdot >= 0) { + tmp = PyUnicode_Substring(name, lastdot + 1, name_len); + if (tmp == NULL) + return NULL; + name = tmp; + /* "name" now holds a new reference to the substring */ + } else { + Py_INCREF(name); + } + + /* Encode to ASCII */ + encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); + if (encoded != NULL) { + } else { + goto error; + } + + /* Replace '-' by '_' */ + modname = PyObject_CallMethod(encoded, "replace", "cc", '-', '_'); + if (modname == NULL) + goto error; + + Py_DECREF(name); + Py_DECREF(encoded); + return modname; +error: + Py_DECREF(name); + Py_XDECREF(encoded); + return NULL; +} + +static bool validate_abi_tag(const char *shortname, const char *soname, + uint32_t required_major_version, uint32_t required_minor_version) { + char *substr = strstr(soname, ".hpy"); + if (substr != NULL) { + substr += strlen(".hpy"); + if (*substr >= '0' && *substr <= '9') { + // It is a number w/o sign and whitespace, we can now parse it with atoi + uint32_t abi_tag = (uint32_t) atoi(substr); + if (abi_tag == required_major_version) { + return true; + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': mismatch between the " + "HPy ABI tag encoded in the filename and the major version requested " + "by the HPy extension itself. Major version tag parsed from " + "filename: %" PRIu32 ". Requested version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, abi_tag, required_major_version, required_minor_version); + return false; + } + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': could not find " + "HPy ABI tag encoded in the filename. The extension claims to be compiled with " + "HPy ABI version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, required_major_version, required_minor_version); + return false; +} + +static void dlsym_error(const char *soname, const char *symbol_name) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at " + "path '%s' while trying to find symbol '%s'. Did you use" + "the HPy_MODINIT macro to register your module? Error " + "message from dlsym/WinAPI: %s", soname, symbol_name, error); +} + +static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode, PyObject *spec) +{ + PyObject *name = NULL; + PyObject *pathbytes = NULL; + PyModuleDef *pydef = NULL; + PyObject *py_mod = NULL; + + name = get_encoded_name(name_unicode); + if (name == NULL) { + goto error; + } + + pathbytes = PyUnicode_EncodeFSDefault(path); + if (pathbytes == NULL) + goto error; + const char *soname = PyBytes_AS_STRING(pathbytes); + + void *mylib = dlopen(soname, RTLD_NOW); // who closes this? + if (mylib == NULL) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at path " + "'%s'. Error message from dlopen/WinAPI: %s", soname, error); + goto error; + } + + const char *shortname = PyBytes_AS_STRING(name); + char minor_version_symbol_name[258]; + char major_version_symbol_name[258]; + PyOS_snprintf(minor_version_symbol_name, sizeof(minor_version_symbol_name), + "get_required_hpy_minor_version_%.200s", shortname); + PyOS_snprintf(major_version_symbol_name, sizeof(major_version_symbol_name), + "get_required_hpy_major_version_%.200s", shortname); + void *minor_version_ptr = dlsym(mylib, minor_version_symbol_name); + void *major_version_ptr = dlsym(mylib, major_version_symbol_name); + if (minor_version_ptr == NULL || major_version_ptr == NULL) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at path " + "'%s'. Cannot locate the required minimal HPy versions as symbols '%s' and `%s`. " + "Error message from dlopen/WinAPI: %s", + soname, minor_version_symbol_name, major_version_symbol_name, error); + goto error; + } + uint32_t required_minor_version = ((VersionGetterFuncPtr) minor_version_ptr)(); + uint32_t required_major_version = ((VersionGetterFuncPtr) major_version_ptr)(); + if (required_major_version != HPY_ABI_VERSION || required_minor_version > HPY_ABI_VERSION_MINOR) { + // For now, we have only one major version, but in the future at this + // point we would decide which HPyContext to create + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' requires unsupported version of the HPy runtime. " + "Requested version: %" PRIu32 ".%" PRIu32 ". Current HPy version: %" PRIu32 ".%" PRIu32 ".", + shortname, required_major_version, required_minor_version, + HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR); + goto error; + } + + // Sanity check of the tag in the shared object filename + if (!validate_abi_tag(shortname, soname, required_major_version, required_minor_version)) { + goto error; + } + + HPyContext *ctx = get_context(mode); + if (ctx == NULL) + goto error; + + char init_ctx_name[258]; + PyOS_snprintf(init_ctx_name, sizeof(init_ctx_name), "%.20s_%.200s", + init_ctx_prefix, shortname); + void *initctxfn = dlsym(mylib, init_ctx_name); + if (initctxfn == NULL) { + dlsym_error(soname, init_ctx_name); + goto error; + } + ((InitContextFuncPtr)initctxfn)(ctx); + + char init_name[258]; + PyOS_snprintf(init_name, sizeof(init_name), "%.20s_%.200s", + init_prefix, shortname); + void *initfn = dlsym(mylib, init_name); + if (initfn == NULL) { + dlsym_error(soname, init_name); + goto error; + } + + HPyModuleDef* hpydef = ((InitFuncPtr)initfn)(); + if (hpydef == NULL) { + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at " + "path '%s'. Function '%s' returned NULL.", soname, init_name); + goto error; + } + + pydef = _HPyModuleDef_CreatePyModuleDef(hpydef); + if (pydef == NULL) { + goto error; + } + + py_mod = PyModule_FromDefAndSpec(pydef, spec); + if (py_mod == NULL) + goto error; + + if (PyModule_Check(py_mod)) { + if (PyModule_ExecDef(py_mod, pydef) != 0) + goto error; + } + + Py_XDECREF(name); + Py_XDECREF(pathbytes); + return py_mod; +error: + Py_XDECREF(py_mod); + if (pydef != NULL) + PyMem_Free(pydef); + Py_XDECREF(name); + Py_XDECREF(pathbytes); + return NULL; +} + +static PyObject *load(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"name", "path", "spec", "debug", "mode", NULL}; + PyObject *name_unicode; + PyObject *path; + PyObject *spec; + int debug = 0; + int mode = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOO|pi", kwlist, + &name_unicode, &path, &spec, &debug, &mode)) { + return NULL; + } + HPyMode hmode = debug ? MODE_DEBUG : MODE_UNIVERSAL; + // 'mode' just overwrites 'debug' + if (mode > 0) + { + hmode = (HPyMode) mode; + } + return do_load(name_unicode, path, hmode, spec); +} + +static HPyMode get_mode_from_string(const char *s, Py_ssize_t n) +{ + if (_hpy_strncmp_ignore_case("debug", s, n) == 0) + return MODE_DEBUG; + if (_hpy_strncmp_ignore_case("trace", s, n) == 0) + return MODE_TRACE; + // if (_hpy_strncmp_ignore_case("debug+trace", s, n) == 0) + // return MODE_DEBUG_TRACE; + // if (_hpy_strncmp_ignore_case("trace+debug", s, n) == 0) + // return MODE_TRACE_DEBUG; + if (_hpy_strncmp_ignore_case("universal", s, n) == 0) + return MODE_UNIVERSAL; + return MODE_INVALID; +} + +/* + * A little helper that does a fast-path if 'mapping' is a dict. + */ +static int mapping_get_item(PyObject *mapping, const char *skey, PyObject **value) +{ + PyObject *key = PyUnicode_FromString(skey); + if (key == NULL) + return 1; + + // fast-path if 'mapping' is a dict + if (PyDict_Check(mapping)) { + *value = PyDict_GetItem(mapping, key); + Py_DECREF(key); + /* 'NULL' means, the key is not present in the dict, so just return + 'NULL'. Since PyDict_GetItem does not set an error, we don't need to + clear any error. */ + } else { + *value = PyObject_GetItem(mapping, key); + Py_DECREF(key); + if (*value == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + } else { + return 1; + } + } + } + return 0; +} + +/* + * HPY_MODE := MODE | (MODULE_NAME ':' MODE { ',' MODULE_NAME ':' MODE }) + * MODULE_NAME := IDENTIFIER + * MODE := 'debug' | 'trace' | 'universal' + */ +static HPyMode get_hpy_mode_from_environ(const char *s_name, PyObject *env) +{ + PyObject *value; + Py_ssize_t size; + HPyMode res; + const char *s_value; + + if (mapping_get_item(env, "HPY", &value)) { + return MODE_INVALID; + } + + /* 'value == NULL' is not an error; this just means that the key was not + present in 'env'. */ + if (value == NULL) { + return MODE_UNIVERSAL; + } + + s_value = PyUnicode_AsUTF8AndSize(value, &size); + if (s_value == NULL) { + Py_DECREF(value); + return MODE_INVALID; + } + res = MODE_INVALID; + char *colon = strchr(s_value, ':'); + if (colon) { + // case 2: modes are specified per module + char *name_start = (char *)s_value; + char *comma; + size_t mode_len; + while (colon) { + comma = strchr(colon + 1, ','); + if (comma) { + mode_len = comma - colon - 1; + } else { + mode_len = (s_value + size) - colon - 1; + } + if (strncmp(s_name, name_start, colon - name_start) == 0) { + res = get_mode_from_string(colon + 1, mode_len); + break; + } + if (comma) { + name_start = comma + 1; + colon = strchr(name_start, ':'); + } else { + colon = NULL; + } + } + } else { + // case 1: mode was globally specified + res = get_mode_from_string(s_value, size); + } + + if (res == MODE_INVALID) + PyErr_Format(PyExc_ValueError, "invalid HPy mode: %.50s", s_value); + Py_DECREF(value); + return res; +} + +static PyObject * +load_bootstrap(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"name", "ext_name", "package", "file", "loader", "spec", + "env", NULL}; + PyObject *module_name, *name, *package, *file, *loader, *spec, *env; + PyObject *log_obj, *m; + HPyMode hmode; + const char *s_module_name, *log_msg; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OOOOOOO", kwlist, + &module_name, &name, &package, &file, + &loader, &spec, &env)) { + return NULL; + } + + s_module_name = PyUnicode_AsUTF8AndSize(module_name, NULL); + if (s_module_name == NULL) { + return NULL; + } + + hmode = get_hpy_mode_from_environ(s_module_name, env); + if (hmode == MODE_INVALID) + return NULL; + + if (mapping_get_item(env, "HPY_LOG", &log_obj)) + return NULL; + + if (log_obj != NULL) { + Py_DECREF(log_obj); + switch (hmode) { + case MODE_DEBUG: + log_msg = " with a debug context"; + break; + case MODE_TRACE: + log_msg = " with a trace context"; + break; + case MODE_UNIVERSAL: + log_msg = ""; + break; + default: + // that's not possible but required for the compiler + return NULL; + } + PySys_FormatStdout("Loading '%.200s' in HPy universal mode%.200s\n", + s_module_name, log_msg); + } + + m = do_load(module_name, file, hmode, spec); + if (m == NULL) + return NULL; + + PyObject_SetAttrString(m, "__file__", file); + PyObject_SetAttrString(m, "__loader__", loader); + PyObject_SetAttrString(m, "__name__", name); + PyObject_SetAttrString(m, "__package__", package); + PyObject_SetAttrString(spec, "origin", file); + PyObject_SetAttrString(m, "__spec__", spec); + + return m; +} + +static PyObject *get_version(PyObject *self, PyObject *ignored) +{ + return Py_BuildValue("ss", HPY_VERSION, HPY_GIT_REVISION); +} + +PyDoc_STRVAR(load_bootstrap_doc, "Internal function intended to be used by " + "the stub loader. This function will honor env var 'HPY' and correctly" + " set the attributes of the module."); + +static PyMethodDef HPyMethods[] = { + {"load", (PyCFunction)load, METH_VARARGS | METH_KEYWORDS, + ("Load a ." HPY_ABI_TAG ".so file")}, + {"_load_bootstrap", (PyCFunction)load_bootstrap, + METH_VARARGS | METH_KEYWORDS, load_bootstrap_doc}, + {"get_version", (PyCFunction)get_version, METH_NOARGS, + "Return a tuple ('version', 'git revision')"}, + {NULL, NULL, 0, NULL} +}; + + +static int exec_module(PyObject *mod); +static PyModuleDef_Slot hpymodule_slots[] = { + {Py_mod_exec, exec_module}, + {0, NULL}, +}; + +static struct PyModuleDef hpy_pydef = { + PyModuleDef_HEAD_INIT, + .m_name = "hpy.universal", + .m_doc = "HPy universal runtime for CPython", + .m_size = 0, + .m_methods = HPyMethods, + .m_slots = hpymodule_slots, +}; + +static int initialize_module(HPyContext *ctx, PyObject *hpy_universal, const char* name, + const char *full_name, HPyModuleDef *hpydef, + PyObject* spec_from_file_and_location, PyObject *location) +{ + PyObject *spec = NULL, *new_mod = NULL; + int result = -1; + + spec = PyObject_CallFunction(spec_from_file_and_location, "sO", full_name, location); + PyModuleDef *pydef = _HPyModuleDef_CreatePyModuleDef(hpydef); + new_mod = PyModule_FromDefAndSpec(pydef, spec); + if (new_mod == NULL) + goto cleanup; + + if (PyModule_ExecDef(new_mod, pydef) != 0) + goto cleanup; + + Py_INCREF(new_mod); // PyModule_AddObject steals the reference + if (PyModule_AddObject(hpy_universal, name, new_mod) < 0) + goto cleanup; + + result = 0; + +cleanup: + Py_XDECREF(new_mod); + Py_XDECREF(spec); + return result; +} + +// module initialization function +int exec_module(PyObject* mod) { + HPyContext *ctx = &g_universal_ctx; + + PyObject *importlib_util = PyImport_ImportModule("importlib.util"); + if (importlib_util == NULL) + return -1; + + PyObject *spec_from_file_and_location = PyObject_GetAttrString(importlib_util, "spec_from_file_location"); + Py_DecRef(importlib_util); + if (spec_from_file_and_location == NULL) + return -1; + + PyObject *current_mod_spec = PyObject_GetAttrString(mod, "__spec__"); + if (current_mod_spec == NULL) + return -1; + + PyObject *location = PyObject_GetAttrString(current_mod_spec, "origin"); + if (location == NULL) + return -1; + + HPyInitGlobalContext__debug(ctx); + int result = initialize_module(ctx, mod, "_debug", "hpy.debug._debug", + HPyInit__debug(), spec_from_file_and_location, location); + if (result != 0) + return result; + + HPyInitGlobalContext__trace(ctx); + result = initialize_module(ctx, mod, "_trace", "hpy.trace._trace", + HPyInit__trace(), spec_from_file_and_location, location); + if (result != 0) + return result; + + for (int i=0; hpy_mode_names[i]; i++) + { + if (PyModule_AddIntConstant(mod, hpy_mode_names[i], i) < 0) + return -1; + } + + return 0; +} + +static void init_universal_ctx(HPyContext *ctx) +{ + if (!HPy_IsNull(ctx->h_None)) + // already initialized + return; + + // XXX this code is basically the same as found in cpython/hpy.h. We + // should probably share and/or autogenerate both versions + /* Constants */ + ctx->h_None = _py2h(Py_None); + ctx->h_True = _py2h(Py_True); + ctx->h_False = _py2h(Py_False); + ctx->h_NotImplemented = _py2h(Py_NotImplemented); + ctx->h_Ellipsis = _py2h(Py_Ellipsis); + /* Exceptions */ + ctx->h_BaseException = _py2h(PyExc_BaseException); + ctx->h_Exception = _py2h(PyExc_Exception); + ctx->h_StopAsyncIteration = _py2h(PyExc_StopAsyncIteration); + ctx->h_StopIteration = _py2h(PyExc_StopIteration); + ctx->h_GeneratorExit = _py2h(PyExc_GeneratorExit); + ctx->h_ArithmeticError = _py2h(PyExc_ArithmeticError); + ctx->h_LookupError = _py2h(PyExc_LookupError); + ctx->h_AssertionError = _py2h(PyExc_AssertionError); + ctx->h_AttributeError = _py2h(PyExc_AttributeError); + ctx->h_BufferError = _py2h(PyExc_BufferError); + ctx->h_EOFError = _py2h(PyExc_EOFError); + ctx->h_FloatingPointError = _py2h(PyExc_FloatingPointError); + ctx->h_OSError = _py2h(PyExc_OSError); + ctx->h_ImportError = _py2h(PyExc_ImportError); + ctx->h_ModuleNotFoundError = _py2h(PyExc_ModuleNotFoundError); + ctx->h_IndexError = _py2h(PyExc_IndexError); + ctx->h_KeyError = _py2h(PyExc_KeyError); + ctx->h_KeyboardInterrupt = _py2h(PyExc_KeyboardInterrupt); + ctx->h_MemoryError = _py2h(PyExc_MemoryError); + ctx->h_NameError = _py2h(PyExc_NameError); + ctx->h_OverflowError = _py2h(PyExc_OverflowError); + ctx->h_RuntimeError = _py2h(PyExc_RuntimeError); + ctx->h_RecursionError = _py2h(PyExc_RecursionError); + ctx->h_NotImplementedError = _py2h(PyExc_NotImplementedError); + ctx->h_SyntaxError = _py2h(PyExc_SyntaxError); + ctx->h_IndentationError = _py2h(PyExc_IndentationError); + ctx->h_TabError = _py2h(PyExc_TabError); + ctx->h_ReferenceError = _py2h(PyExc_ReferenceError); + ctx->h_SystemError = _py2h(PyExc_SystemError); + ctx->h_SystemExit = _py2h(PyExc_SystemExit); + ctx->h_TypeError = _py2h(PyExc_TypeError); + ctx->h_UnboundLocalError = _py2h(PyExc_UnboundLocalError); + ctx->h_UnicodeError = _py2h(PyExc_UnicodeError); + ctx->h_UnicodeEncodeError = _py2h(PyExc_UnicodeEncodeError); + ctx->h_UnicodeDecodeError = _py2h(PyExc_UnicodeDecodeError); + ctx->h_UnicodeTranslateError = _py2h(PyExc_UnicodeTranslateError); + ctx->h_ValueError = _py2h(PyExc_ValueError); + ctx->h_ZeroDivisionError = _py2h(PyExc_ZeroDivisionError); + ctx->h_BlockingIOError = _py2h(PyExc_BlockingIOError); + ctx->h_BrokenPipeError = _py2h(PyExc_BrokenPipeError); + ctx->h_ChildProcessError = _py2h(PyExc_ChildProcessError); + ctx->h_ConnectionError = _py2h(PyExc_ConnectionError); + ctx->h_ConnectionAbortedError = _py2h(PyExc_ConnectionAbortedError); + ctx->h_ConnectionRefusedError = _py2h(PyExc_ConnectionRefusedError); + ctx->h_ConnectionResetError = _py2h(PyExc_ConnectionResetError); + ctx->h_FileExistsError = _py2h(PyExc_FileExistsError); + ctx->h_FileNotFoundError = _py2h(PyExc_FileNotFoundError); + ctx->h_InterruptedError = _py2h(PyExc_InterruptedError); + ctx->h_IsADirectoryError = _py2h(PyExc_IsADirectoryError); + ctx->h_NotADirectoryError = _py2h(PyExc_NotADirectoryError); + ctx->h_PermissionError = _py2h(PyExc_PermissionError); + ctx->h_ProcessLookupError = _py2h(PyExc_ProcessLookupError); + ctx->h_TimeoutError = _py2h(PyExc_TimeoutError); + /* Warnings */ + ctx->h_Warning = _py2h(PyExc_Warning); + ctx->h_UserWarning = _py2h(PyExc_UserWarning); + ctx->h_DeprecationWarning = _py2h(PyExc_DeprecationWarning); + ctx->h_PendingDeprecationWarning = _py2h(PyExc_PendingDeprecationWarning); + ctx->h_SyntaxWarning = _py2h(PyExc_SyntaxWarning); + ctx->h_RuntimeWarning = _py2h(PyExc_RuntimeWarning); + ctx->h_FutureWarning = _py2h(PyExc_FutureWarning); + ctx->h_ImportWarning = _py2h(PyExc_ImportWarning); + ctx->h_UnicodeWarning = _py2h(PyExc_UnicodeWarning); + ctx->h_BytesWarning = _py2h(PyExc_BytesWarning); + ctx->h_ResourceWarning = _py2h(PyExc_ResourceWarning); + /* Types */ + ctx->h_BaseObjectType = _py2h((PyObject *)&PyBaseObject_Type); + ctx->h_TypeType = _py2h((PyObject *)&PyType_Type); + ctx->h_BoolType = _py2h((PyObject *)&PyBool_Type); + ctx->h_LongType = _py2h((PyObject *)&PyLong_Type); + ctx->h_FloatType = _py2h((PyObject *)&PyFloat_Type); + ctx->h_UnicodeType = _py2h((PyObject *)&PyUnicode_Type); + ctx->h_TupleType = _py2h((PyObject *)&PyTuple_Type); + ctx->h_ListType = _py2h((PyObject *)&PyList_Type); + ctx->h_ComplexType = _py2h((PyObject *)&PyComplex_Type); + ctx->h_BytesType = _py2h((PyObject *)&PyBytes_Type); + ctx->h_MemoryViewType = _py2h((PyObject *)&PyMemoryView_Type); + ctx->h_CapsuleType = _py2h((PyObject *)&PyCapsule_Type); + ctx->h_SliceType = _py2h((PyObject *)&PySlice_Type); + ctx->h_DictType = _py2h((PyObject *)&PyDict_Type); + /* Reflection */ + ctx->h_Builtins = _py2h(PyEval_GetBuiltins()); +} + + +PyMODINIT_FUNC +PyInit_universal(void) +{ + init_universal_ctx(&g_universal_ctx); + PyObject *mod = PyModuleDef_Init(&hpy_pydef); + return mod; +} diff --git a/graalpython/hpy/hpy/universal/src/misc_win32.h b/graalpython/hpy/hpy/universal/src/misc_win32.h new file mode 100644 index 0000000000..d267e758e7 --- /dev/null +++ b/graalpython/hpy/hpy/universal/src/misc_win32.h @@ -0,0 +1,55 @@ +/************************************************************/ +/* Emulate dlopen()&co. from the Windows API */ + +#define RTLD_LAZY 0 +#define RTLD_NOW 0 +#define RTLD_GLOBAL 0 +#define RTLD_LOCAL 0 + +static void *dlopen(const char *filename, int flag) +{ + return (void *)LoadLibraryA(filename); +} + +static void *dlopenW(const wchar_t *filename) +{ + return (void *)LoadLibraryW(filename); +} + +static void *dlsym(void *handle, const char *symbol) +{ + void *address = GetProcAddress((HMODULE)handle, symbol); +#ifndef MS_WIN64 + if (!address) { + /* If 'symbol' is not found, then try '_symbol@N' for N in + (0, 4, 8, 12, ..., 124). Unlike ctypes, we try to do that + for any symbol, although in theory it should only be done + for __stdcall functions. + */ + int i; + char mangled_name[1 + strlen(symbol) + 1 + 3 + 1]; + for (i = 0; i < 32; i++) { + sprintf(mangled_name, "_%s@%d", symbol, i * 4); + address = GetProcAddress((HMODULE)handle, mangled_name); + if (address) + break; + } + } +#endif + return address; +} + +static int dlclose(void *handle) +{ + return FreeLibrary((HMODULE)handle) ? 0 : -1; +} + +static const char *dlerror(void) +{ + static char buf[32]; + DWORD dw = GetLastError(); + if (dw == 0) + return NULL; + sprintf(buf, "error 0x%x", (unsigned int)dw); + return buf; +} diff --git a/graalpython/hpy/microbench/README.md b/graalpython/hpy/microbench/README.md new file mode 100644 index 0000000000..c25916088c --- /dev/null +++ b/graalpython/hpy/microbench/README.md @@ -0,0 +1,29 @@ +To run the microbenchmarks +-------------------------- + +1. You need to have `hpy` installed in your virtuanenv. The easiest way + to do it is: + + $ cd /path/to/hpy + $ python setup.py develop + +2. Build the extension modules needed for the microbenchmarks + + $ cd /path/to/hpy/microbench + $ pip install cffi # needed to build _valgrind + $ python setup.py build_ext --inplace + +2. `py.test -v` + +3. To run only cpy or hpy tests, use -m (to select markers): + + $ py.test -v -m hpy + $ py.test -v -m cpy + +4. Step (2) build `hpy_simple` using the CPython ABI by default. If you want + to benchmark the universal mode, you need to build it explicitly: + + $ cd /path/to/hpy/microbench + $ rm *.so # make sure to delete CPython-ABI versions + $ python setup.py --hpy-abi=universal build_ext --inplace + $ py.test -v diff --git a/graalpython/hpy/microbench/_valgrind_build.py b/graalpython/hpy/microbench/_valgrind_build.py new file mode 100644 index 0000000000..4cbbc1ec7d --- /dev/null +++ b/graalpython/hpy/microbench/_valgrind_build.py @@ -0,0 +1,29 @@ +from cffi import FFI +ffibuilder = FFI() + +ffibuilder.cdef(""" + void callgrind_start(void); + void callgrind_stop(void); + int is_running_on_valgrind(void); +""") + + +ffibuilder.set_source("_valgrind", """ +#include +void callgrind_start(void) { + CALLGRIND_START_INSTRUMENTATION; +} + +void callgrind_stop(void) { + CALLGRIND_STOP_INSTRUMENTATION; + CALLGRIND_DUMP_STATS; +} + +int is_running_on_valgrind(void) { + return RUNNING_ON_VALGRIND; +} +""", + libraries=[]) + +if __name__ == "__main__": + ffibuilder.compile(verbose=True) diff --git a/graalpython/hpy/microbench/conftest.py b/graalpython/hpy/microbench/conftest.py new file mode 100644 index 0000000000..8e553ffe76 --- /dev/null +++ b/graalpython/hpy/microbench/conftest.py @@ -0,0 +1,124 @@ +import re +import time +from collections import defaultdict +import pytest +import _valgrind + +class Timer: + + def __init__(self, nodeid): + self.nodeid = nodeid + self.start = None + self.stop = None + + def __enter__(self): + if self.start is not None: + raise ValueError('You cannot use "with timer:" more than once') + _valgrind.lib.callgrind_start() + self.start = time.time() + + def __exit__(self, etype, evalue, tb): + self.stop = time.time() + _valgrind.lib.callgrind_stop() + + def __str__(self): + if self.start is None: + return '[NO TIMING]' + if self.stop is None: + return '[IN-PROGRESS]' + usec = (self.stop - self.start) * 1000 + return f'{usec:.2f} us' + + @property + def elapsed(self): + if self.start is not None and self.stop is not None: + return self.stop - self.start + return None + + +class TimerSession: + + NODEID = re.compile(r'(.*)\[(.*)\]') + + def __init__(self): + self.apis = set() # ['cpy', 'hpy', ...] + self.table = defaultdict(dict) # {shortid: {api: timer}} + self.timers = {} # nodeid -> Timer + + def new_timer(self, nodeid): + shortid, api = self.split_nodeid(nodeid) + timer = Timer(nodeid) + self.apis.add(api) + self.table[shortid][api] = timer + self.timers[nodeid] = timer + return timer + + def get_timer(self, nodeid): + return self.timers.get(nodeid) + + def split_nodeid(self, nodeid): + shortid = '::'.join(nodeid.split('::')[-2:]) # take only class::function + m = self.NODEID.match(shortid) + if not m: + return shortid, '' + return m.group(1), m.group(2) + + def format_ratio(self, reference, value): + if reference and reference.elapsed and value and value.elapsed: + ratio = value.elapsed / reference.elapsed + return f'[{ratio:.2f}]' + return '' + + def display_summary(self, tr): + w = tr.write_line + w('') + tr.write_sep('=', 'BENCHMARKS', cyan=True) + w(' '*40 + ' cpy hpy') + w(' '*40 + '---------------- -------------------') + for shortid, timings in self.table.items(): + cpy = timings.get('cpy') + hpy = timings.get('hpy') + hpy_ratio = self.format_ratio(cpy, hpy) + cpy = cpy or '' + hpy = hpy or '' + w(f'{shortid:<40} {cpy!s:>15} {hpy!s:>15} {hpy_ratio}') + w('') + + + +@pytest.fixture +def timer(request, api): + nodeid = request.node.nodeid + return request.config._timersession.new_timer(nodeid) + +def pytest_configure(config): + config._timersession = TimerSession() + config.addinivalue_line("markers", "hpy: mark modules using the HPy API") + config.addinivalue_line("markers", "cpy: mark modules using the old Python/C API") + +def pytest_addoption(parser): + parser.addoption( + "--fast", action="store_true", default=False, help="run microbench faster" + ) + parser.addoption( + "--slow", action="store_true", default=False, help="run microbench slower" + ) + + +VERBOSE_TEST_NAME_LENGTH = 90 + +@pytest.hookimpl(hookwrapper=True) +def pytest_report_teststatus(report, config): + outcome = yield + category, letter, word = outcome.get_result() + timer = config._timersession.get_timer(report.nodeid) + if category == 'passed' and timer: + L = VERBOSE_TEST_NAME_LENGTH - len(report.nodeid) + word = str(timer).rjust(L) + markup = None + if timer.elapsed is None: + markup = {'yellow': True} + outcome.force_result((category, letter, (word, markup))) + +def pytest_terminal_summary(terminalreporter, config): + config._timersession.display_summary(terminalreporter) diff --git a/graalpython/hpy/microbench/pytest_valgrind.sh b/graalpython/hpy/microbench/pytest_valgrind.sh new file mode 100755 index 0000000000..8271381d0b --- /dev/null +++ b/graalpython/hpy/microbench/pytest_valgrind.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +valgrind --tool=callgrind --instr-atstart=no python3-dbg -m pytest "$@" diff --git a/graalpython/hpy/microbench/setup.py b/graalpython/hpy/microbench/setup.py new file mode 100644 index 0000000000..16988b0aef --- /dev/null +++ b/graalpython/hpy/microbench/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, Extension + +setup( + name="hpy.microbench", + setup_requires=['cffi', 'hpy'], + ext_modules = [ + Extension('cpy_simple', + ['src/cpy_simple.c'], + extra_compile_args=['-g']) + ], + hpy_ext_modules = [ + Extension('hpy_simple', + ['src/hpy_simple.c'], + extra_compile_args=['-g']), + ], + cffi_modules=["_valgrind_build.py:ffibuilder"], +) diff --git a/graalpython/hpy/microbench/src/cpy_simple.c b/graalpython/hpy/microbench/src/cpy_simple.c new file mode 100644 index 0000000000..dc754de64e --- /dev/null +++ b/graalpython/hpy/microbench/src/cpy_simple.c @@ -0,0 +1,174 @@ +#include + +/* module-level functions */ + +static PyObject* noargs(PyObject* self, PyObject* args) +{ + Py_RETURN_NONE; +} + +static PyObject* onearg(PyObject* self, PyObject* arg) +{ + Py_RETURN_NONE; +} + +static PyObject* varargs(PyObject* self, PyObject* args) +{ + Py_RETURN_NONE; +} + +static PyObject* call_with_tuple(PyObject* self, PyObject* args) +{ + PyObject* f; + PyObject* f_args; + f = PyTuple_GetItem(args, 0); + if (f == NULL) + Py_RETURN_NONE; + f_args = PyTuple_GetItem(args, 1); + if (f_args == NULL) + Py_RETURN_NONE; + return PyObject_CallObject(f, f_args); +} + +static PyObject* call_with_tuple_and_dict(PyObject* self, PyObject* args) +{ + PyObject* f; + PyObject* f_args; + PyObject* f_kw; + f = PyTuple_GetItem(args, 0); + if (f == NULL) + Py_RETURN_NONE; + f_args = PyTuple_GetItem(args, 1); + if (f_args == NULL) + Py_RETURN_NONE; + f_kw = PyTuple_GetItem(args, 2); + if (f_kw == NULL) + Py_RETURN_NONE; + return PyObject_Call(f, f_args, f_kw); +} + +static PyObject* allocate_int(PyObject* self, PyObject* args) +{ + return PyLong_FromLong(2048); +} + +static PyObject* allocate_tuple(PyObject* self, PyObject* args) +{ + return Py_BuildValue("ii", 2048, 2049); +} + +static PyObject * Foo_getitem(PyObject *self, Py_ssize_t i) +{ + Py_RETURN_NONE; +} + +static Py_ssize_t Foo_len(PyObject *self) +{ + return 42; +} + +static PyMethodDef SimpleMethods[] = { + {"noargs", (PyCFunction)noargs, METH_NOARGS, ""}, + {"onearg", (PyCFunction)onearg, METH_O, ""}, + {"varargs", (PyCFunction)varargs, METH_VARARGS, ""}, + {"call_with_tuple", (PyCFunction)call_with_tuple, METH_VARARGS, ""}, + {"call_with_tuple_and_dict", (PyCFunction)call_with_tuple_and_dict, METH_VARARGS, ""}, + {"allocate_int", (PyCFunction)allocate_int, METH_NOARGS, ""}, + {"allocate_tuple", (PyCFunction)allocate_tuple, METH_NOARGS, ""}, + {NULL, NULL, 0, NULL} +}; + + +static PySequenceMethods FooSequence = { + .sq_length = (lenfunc)Foo_len, + .sq_item = (ssizeargfunc)Foo_getitem, + NULL, +}; + + +/* types */ + +typedef struct { + PyObject_HEAD +} FooObject; + +static PyTypeObject Foo_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "cpy_simple.Foo", /* tp_name */ + sizeof(FooObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + 0, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &FooSequence, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "Foo objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + SimpleMethods, /* tp_methods, reuse the same functions */ +}; + + +static PyType_Slot HTFoo_slots[] = { + {Py_tp_doc, "HTFoo objects"}, + {Py_tp_methods, SimpleMethods}, + {Py_sq_item, Foo_getitem}, + {Py_sq_length, Foo_len}, + {0, 0} +}; + +static PyType_Spec HTFoo_Type_spec = { + .name = "cpy_simple.Foo", + .basicsize = sizeof(FooObject), + .itemsize = 0, + .flags = Py_TPFLAGS_DEFAULT, + .slots = HTFoo_slots +}; + +static struct PyModuleDef moduledef = { + PyModuleDef_HEAD_INIT, + "cpy_simple", + "Module Doc", + -1, + SimpleMethods, + NULL, + NULL, + NULL, + NULL, +}; + +PyMODINIT_FUNC +PyInit_cpy_simple(void) +{ + PyObject* m; + Foo_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&Foo_Type) < 0) + return NULL; + m = PyModule_Create(&moduledef); + if (m == NULL) + return NULL; + Py_INCREF(&Foo_Type); + PyModule_AddObject(m, "Foo", (PyObject *)&Foo_Type); + + PyObject *HTFoo_Type = PyType_FromSpec(&HTFoo_Type_spec); + if (HTFoo_Type == NULL) + return NULL; + PyModule_AddObject(m, "HTFoo", HTFoo_Type); + + return m; +} diff --git a/graalpython/hpy/microbench/src/hpy_simple.c b/graalpython/hpy/microbench/src/hpy_simple.c new file mode 100644 index 0000000000..a5dddadd63 --- /dev/null +++ b/graalpython/hpy/microbench/src/hpy_simple.c @@ -0,0 +1,135 @@ +#include "hpy.h" + +/* module-level functions */ + +HPyDef_METH(noargs, "noargs", HPyFunc_NOARGS) +static HPy noargs_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(onearg, "onearg", HPyFunc_O) +static HPy onearg_impl(HPyContext *ctx, HPy self, HPy arg) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(varargs, "varargs", HPyFunc_VARARGS) +static HPy varargs_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(call_with_tuple, "call_with_tuple", HPyFunc_VARARGS) +static HPy call_with_tuple_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy f, f_args; + if (nargs != 2) { + HPyErr_SetString(ctx, ctx->h_TypeError, "call_with_tuple requires two arguments"); + return HPy_NULL; + } + f = args[0]; + f_args = args[1]; + return HPy_CallTupleDict(ctx, f, f_args, HPy_NULL); +} + +HPyDef_METH(call_with_tuple_and_dict, "call_with_tuple_and_dict", HPyFunc_VARARGS) +static HPy call_with_tuple_and_dict_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + HPy f, f_args, f_kw; + if (nargs != 3) { + HPyErr_SetString(ctx, ctx->h_TypeError, "call_with_tuple_and_dict requires three arguments"); + return HPy_NULL; + } + f = args[0]; + f_args = args[1]; + f_kw = args[2]; + return HPy_CallTupleDict(ctx, f, f_args, f_kw); +} + +HPyDef_METH(allocate_int, "allocate_int", HPyFunc_NOARGS) +static HPy allocate_int_impl(HPyContext *ctx, HPy self) +{ + return HPyLong_FromLong(ctx, 2048); +} + +HPyDef_METH(allocate_tuple, "allocate_tuple", HPyFunc_NOARGS) +static HPy allocate_tuple_impl(HPyContext *ctx, HPy self) +{ + return HPy_BuildValue(ctx, "ii", 2048, 2049); +} + + +/* Foo type */ + +typedef struct { + long x; + long y; +} FooObject; + +HPyDef_SLOT(Foo_getitem, HPy_sq_item) +static HPy Foo_getitem_impl(HPyContext *ctx, HPy self, HPy_ssize_t i) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_SLOT(Foo_len, HPy_sq_length) +static HPy_ssize_t Foo_len_impl(HPyContext *ctx, HPy self) +{ + return 42; +} + + +// note that we can reuse the same HPyDef for both module-level and type-level +// methods +static HPyDef *foo_defines[] = { + &noargs, + &onearg, + &varargs, + &allocate_int, + &allocate_tuple, + &Foo_getitem, + &Foo_len, +}; + + +static HPyType_Spec Foo_spec = { + .name = "hpy_simple.Foo", + .basicsize = sizeof(FooObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = foo_defines +}; + + +/* Module defines */ + +HPyDef_SLOT(init_hpy_simple, HPy_mod_exec) +static int init_hpy_simple_impl(HPyContext *ctx, HPy m) +{ + HPy h_Foo = HPyType_FromSpec(ctx, &Foo_spec, NULL); + if (HPy_IsNull(h_Foo)) + return -1; + HPy_SetAttr_s(ctx, m, "Foo", h_Foo); + HPy_SetAttr_s(ctx, m, "HTFoo", h_Foo); + return 0; +} + +static HPyDef *module_defines[] = { + &noargs, + &onearg, + &varargs, + &call_with_tuple, + &call_with_tuple_and_dict, + &allocate_int, + &allocate_tuple, + &init_hpy_simple, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy microbenchmarks", + .size = 0, + .defines = module_defines, +}; + +HPy_MODINIT(hpy_simple, moduledef) diff --git a/graalpython/hpy/microbench/test_microbench.py b/graalpython/hpy/microbench/test_microbench.py new file mode 100644 index 0000000000..ed59a243f0 --- /dev/null +++ b/graalpython/hpy/microbench/test_microbench.py @@ -0,0 +1,226 @@ +""" +These are not real tests, but microbenchmarks. The machinery to record the +timing and display the results is inside conftest.py +""" + +import pytest +import _valgrind + +API_PARAMS = [ + pytest.param('cpy', marks=pytest.mark.cpy), + pytest.param('hpy', marks=pytest.mark.hpy) + ] + +@pytest.fixture(params=API_PARAMS) +def api(request): + return request.param + +@pytest.fixture +def simple(request, api): + if api == 'cpy': + import cpy_simple + return cpy_simple + elif api == 'hpy': + import hpy_simple + return hpy_simple + else: + assert False, 'Unkown param: %s' % request.param + +@pytest.fixture +def N(request): + n = 10000000 + if request.config.option.fast: + n //= 100 + if request.config.option.slow: + n *= 10 + if _valgrind.lib.is_running_on_valgrind(): + n //= 100 + return n + + +class TestModule: + + def test_noargs(self, simple, timer, N): + with timer: + for i in range(N): + simple.noargs() + + def test_onearg_None(self, simple, timer, N): + with timer: + for i in range(N): + simple.onearg(None) + + def test_onearg_int(self, simple, timer, N): + with timer: + for i in range(N): + simple.onearg(i) + + def test_varargs(self, simple, timer, N): + with timer: + for i in range(N): + simple.varargs(None, None) + + def test_call_with_tuple(self, simple, timer, N): + def f(a, b): + return a + b + + with timer: + for i in range(N): + simple.call_with_tuple(f, (1, 2)) + + def test_call_with_tuple_and_dict(self, simple, timer, N): + def f(a, b): + return a + b + + with timer: + for i in range(N): + simple.call_with_tuple_and_dict(f, (1,), {"b": 2}) + + def test_allocate_int(self, simple, timer, N): + with timer: + for i in range(N): + simple.allocate_int() + + def test_allocate_tuple(self, api, simple, timer, N): + with timer: + for i in range(N): + simple.allocate_tuple() + + +class TestType: + """ Compares the performance of operations on types. + + The kinds of type used are: + + * cpy: a static type + * hpy: a heap type (HPy only has heap types) + + The type is named `simple.Foo` in both cases. + """ + + def test_allocate_obj(self, simple, timer, N): + import gc + Foo = simple.Foo + objs = [None] * N + gc.collect() + with timer: + for i in range(N): + objs[i] = Foo() + del objs + gc.collect() + + def test_method_lookup(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + # note: here we are NOT calling it, we want to measure just + # the lookup + obj.noargs + + def test_noargs(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.noargs() + + def test_onearg_None(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.onearg(None) + + def test_onearg_int(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.onearg(i) + + def test_varargs(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj.varargs(None, None) + + def test_len(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + len(obj) + + def test_getitem(self, simple, timer, N): + obj = simple.Foo() + with timer: + for i in range(N): + obj[0] + + +class TestHeapType: + """ Compares the performance of operations on heap types. + + The type is named `simple.HTFoo` and is a heap type in all cases. + """ + + def test_allocate_obj_and_survive(self, simple, timer, N): + import gc + HTFoo = simple.HTFoo + objs = [None] * N + gc.collect() + with timer: + for i in range(N): + objs[i] = HTFoo() + del objs + gc.collect() + + def test_allocate_obj_and_die(self, simple, timer, N): + import gc + HTFoo = simple.HTFoo + gc.collect() + with timer: + for i in range(N): + obj = HTFoo() + obj.onearg(None) + gc.collect() + + def test_method_lookup(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + # note: here we are NOT calling it, we want to measure just + # the lookup + obj.noargs + + def test_noargs(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.noargs() + + def test_onearg_None(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.onearg(None) + + def test_onearg_int(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.onearg(i) + + def test_varargs(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj.varargs(None, None) + + def test_len(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + len(obj) + + def test_getitem(self, simple, timer, N): + obj = simple.HTFoo() + with timer: + for i in range(N): + obj[0] diff --git a/graalpython/hpy/proof-of-concept/pof.c b/graalpython/hpy/proof-of-concept/pof.c new file mode 100644 index 0000000000..672a9a0199 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pof.c @@ -0,0 +1,109 @@ +#include "hpy.h" +#include + +HPyDef_METH(do_nothing, "do_nothing", HPyFunc_NOARGS) +static HPy do_nothing_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(double_obj, "double", HPyFunc_O) +static HPy double_obj_impl(HPyContext *ctx, HPy self, HPy obj) +{ + return HPy_Add(ctx, obj, obj); +} + +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +HPyDef_METH(add_ints_kw, "add_ints_kw", HPyFunc_KEYWORDS) +static HPy add_ints_kw_impl(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames) +{ + long a, b; + const char* kwlist[] = {"a", "b", NULL}; + if (!HPyArg_ParseKeywords(ctx, NULL, args, nargs, kwnames, "ll", + kwlist, &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +typedef struct { + double x; + double y; +} PointObject; + +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl (HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kwnames) +{ + double x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "dd", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} + +HPyDef_SLOT(Point_repr, HPy_tp_repr) +static HPy Point_repr_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + char msg[256]; + snprintf(msg, 256, "Point(%g, %g)", point->x, point->y); + return HPyUnicode_FromString(ctx, msg); + //return HPyUnicode_FromFormat("Point(%g, %g)", point->x, point->y); +} + + +static HPyDef *point_type_defines[] = { + &Point_new, + &Point_repr, + NULL +}; +static HPyType_Spec point_type_spec = { + .name = "pof.Point", + .basicsize = sizeof(PointObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_type_defines +}; + +HPyDef_SLOT(mod_exec, HPy_mod_exec) +static int mod_exec_impl(HPyContext *ctx, HPy m) +{ + HPy h_point_type = HPyType_FromSpec(ctx, &point_type_spec, NULL); + if (HPy_IsNull(h_point_type)) + return -1; + HPy_SetAttr_s(ctx, m, "Point", h_point_type); + HPy_Close(ctx, h_point_type); + return 0; +} + +static HPyDef *module_defines[] = { + &do_nothing, + &double_obj, + &add_ints, + &add_ints_kw, + &mod_exec, + NULL +}; + +static HPyModuleDef moduledef = { + .doc = "HPy Proof of Concept", + .size = 0, + .defines = module_defines +}; + +HPy_MODINIT(pof, moduledef) diff --git a/graalpython/hpy/proof-of-concept/pofcpp.cpp b/graalpython/hpy/proof-of-concept/pofcpp.cpp new file mode 100644 index 0000000000..6214c5f87c --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofcpp.cpp @@ -0,0 +1,117 @@ +#include "hpy.h" +#include + +HPyDef_METH(do_nothing, "do_nothing", HPyFunc_NOARGS) +static HPy do_nothing_impl(HPyContext *ctx, HPy self) +{ + return HPy_Dup(ctx, ctx->h_None); +} + +HPyDef_METH(double_obj, "double", HPyFunc_O) +static HPy double_obj_impl(HPyContext *ctx, HPy self, HPy obj) +{ + return HPy_Add(ctx, obj, obj); +} + +HPyDef_METH(add_ints, "add_ints", HPyFunc_VARARGS) +static HPy add_ints_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) +{ + long a, b; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "ll", &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +HPyDef_METH(add_ints_kw, "add_ints_kw", HPyFunc_KEYWORDS) +static HPy add_ints_kw_impl(HPyContext *ctx, HPy self, const HPy *args, + size_t nargs, HPy kwnames) +{ + long a, b; + const char* kwlist[] = {"a", "b", NULL}; + if (!HPyArg_ParseKeywords(ctx, NULL, args, nargs, kwnames, "ll", + kwlist, &a, &b)) + return HPy_NULL; + return HPyLong_FromLong(ctx, a+b); +} + +typedef struct { + double x; + double y; +} PointObject; + +HPyType_HELPERS(PointObject) + +HPyDef_SLOT(Point_new, HPy_tp_new) +static HPy Point_new_impl (HPyContext *ctx, HPy cls, const HPy *args, + HPy_ssize_t nargs, HPy kwnames) +{ + double x, y; + if (!HPyArg_Parse(ctx, NULL, args, nargs, "dd", &x, &y)) + return HPy_NULL; + PointObject *point; + HPy h_point = HPy_New(ctx, cls, &point); + if (HPy_IsNull(h_point)) + return HPy_NULL; + point->x = x; + point->y = y; + return h_point; +} + +HPyDef_SLOT(Point_repr, HPy_tp_repr) +static HPy Point_repr_impl(HPyContext *ctx, HPy self) +{ + PointObject *point = PointObject_AsStruct(ctx, self); + char msg[256]; + snprintf(msg, 256, "Point(%g, %g)", point->x, point->y); + return HPyUnicode_FromString(ctx, msg); + //return HPyUnicode_FromFormat("Point(%g, %g)", point->x, point->y); +} + + +static HPyDef *point_type_defines[] = { + &Point_new, + &Point_repr, + NULL +}; + +static HPyType_Spec point_type_spec = { + .name = "pofcpp.Point", + .basicsize = sizeof(PointObject), + .flags = HPy_TPFLAGS_DEFAULT, + .defines = point_type_defines +}; + +HPyDef_SLOT(mod_exec, HPy_mod_exec) +static int mod_exec_impl(HPyContext *ctx, HPy m) +{ + HPy h_point_type = HPyType_FromSpec(ctx, &point_type_spec, NULL); + if (HPy_IsNull(h_point_type)) + return -1; + HPy_SetAttr_s(ctx, m, "Point", h_point_type); + HPy_Close(ctx, h_point_type); + return 0; +} + +static HPyDef *module_defines[] = { + &do_nothing, + &double_obj, + &add_ints, + &add_ints_kw, + &mod_exec, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy c++ Proof of Concept", + .size = 0, + .defines = module_defines +}; + +#ifdef __cplusplus +extern "C" { +#endif + +HPy_MODINIT(pofcpp, moduledef) + +#ifdef __cplusplus +} +#endif diff --git a/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp b/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp new file mode 100644 index 0000000000..01726d2d6f --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofpackage/bar.cpp @@ -0,0 +1,45 @@ +#include "hpy.h" + +class Bar +{ + int foo; + public: + Bar(int f) + { + foo = f; + } + + int boo(HPyContext *ctx, HPy obj) + { + return foo + HPyLong_AsLong(ctx, obj); + } +}; + +HPyDef_METH(hello, "hello", HPyFunc_O) +static HPy hello_impl(HPyContext *ctx, HPy self, HPy obj) +{ + Bar b(21); + return HPyLong_FromLong(ctx, b.boo(ctx, obj)); +} + + +static HPyDef *module_defines[] = { + &hello, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy C++ Proof of Concept", + .size = 0, + .defines = module_defines +}; + +#ifdef __cplusplus +extern "C" { +#endif + + +HPy_MODINIT(bar, moduledef) + +#ifdef __cplusplus +} +#endif diff --git a/graalpython/hpy/proof-of-concept/pofpackage/foo.c b/graalpython/hpy/proof-of-concept/pofpackage/foo.c new file mode 100644 index 0000000000..3a265efa25 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/pofpackage/foo.c @@ -0,0 +1,20 @@ +#include "hpy.h" + +HPyDef_METH(hello, "hello", HPyFunc_NOARGS) +static HPy hello_impl(HPyContext *ctx, HPy self) +{ + return HPyUnicode_FromString(ctx, "hello from pofpackage.foo"); +} + + +static HPyDef *module_defines[] = { + &hello, + NULL +}; +static HPyModuleDef moduledef = { + .doc = "HPy Proof of Concept", + .size = 0, + .defines = module_defines +}; + +HPy_MODINIT(foo, moduledef) diff --git a/graalpython/hpy/proof-of-concept/requirements.txt b/graalpython/hpy/proof-of-concept/requirements.txt new file mode 100644 index 0000000000..2009c86f18 --- /dev/null +++ b/graalpython/hpy/proof-of-concept/requirements.txt @@ -0,0 +1 @@ +hpy diff --git a/graalpython/hpy/proof-of-concept/setup.py b/graalpython/hpy/proof-of-concept/setup.py new file mode 100644 index 0000000000..8132e1f2dc --- /dev/null +++ b/graalpython/hpy/proof-of-concept/setup.py @@ -0,0 +1,36 @@ +from setuptools import setup, Extension +import platform + +cpp_compile_extra_args = [] + +if platform.system() == "Windows": + compile_extra_args = ['/WX'] + cpp_compile_extra_args = [ + "/std:c++latest", # MSVC C7555 + ] +else: + compile_extra_args = ['-Werror'] + + +setup( + name="hpy-pof", + packages = ['pofpackage'], + zip_safe=False, + hpy_ext_modules=[ + Extension('pof', + sources=['pof.c'], + extra_compile_args=compile_extra_args), + Extension('pofpackage.foo', + sources=['pofpackage/foo.c'], + extra_compile_args=compile_extra_args), + Extension('pofcpp', + sources=['pofcpp.cpp'], + language='C++', + extra_compile_args=compile_extra_args + cpp_compile_extra_args), + Extension('pofpackage.bar', + sources=['pofpackage/bar.cpp'], + language='C++', + extra_compile_args=compile_extra_args + cpp_compile_extra_args), + ], + setup_requires=['hpy'], +) diff --git a/graalpython/hpy/proof-of-concept/test_pof.py b/graalpython/hpy/proof-of-concept/test_pof.py new file mode 100644 index 0000000000..54c6851c3f --- /dev/null +++ b/graalpython/hpy/proof-of-concept/test_pof.py @@ -0,0 +1,44 @@ +import pof +import pofpackage.foo +import pofcpp +import pofpackage.bar + +def test_do_nothing(): + assert pof.do_nothing() is None + +def test_double(): + assert pof.double(21) == 42 + +def test_add_ints(): + assert pof.add_ints(30, 12) == 42 + +def test_add_ints_kw(): + assert pof.add_ints_kw(b=30, a=12) == 42 + +def test_point(): + p = pof.Point(1, 2) + assert repr(p) == 'Point(1, 2)' # fixme when we have HPyFloat_FromDouble + +def test_pofpackage(): + assert pofpackage.foo.__name__ == "pofpackage.foo" + assert pofpackage.foo.hello() == 'hello from pofpackage.foo' + +def test_cpp_do_nothing(): + assert pofcpp.do_nothing() is None + +def test_cpp_double(): + assert pofcpp.double(21) == 42 + +def test_cpp_add_ints(): + assert pofcpp.add_ints(30, 12) == 42 + +def test_cpp_add_ints_kw(): + assert pofcpp.add_ints_kw(b=30, a=12) == 42 + +def test_cpp_point(): + p = pofcpp.Point(1, 2) + assert repr(p) == 'Point(1, 2)' # fixme when we have HPyFloat_FromDouble + +def test_cpp_pofpackage(): + assert pofpackage.bar.__name__ == "pofpackage.bar" + assert pofpackage.bar.hello(21) == 42 diff --git a/graalpython/hpy/proof-of-concept/test_pof.sh b/graalpython/hpy/proof-of-concept/test_pof.sh new file mode 100755 index 0000000000..1ac800769c --- /dev/null +++ b/graalpython/hpy/proof-of-concept/test_pof.sh @@ -0,0 +1,177 @@ +#!/bin/bash +set -e +ROOT=`pwd` # we expect this script to be run from the repo root + +# Allow the caller to override the Python runtime used +PYTHON=${PYTHON:-python3} + +_install_hpy() { + echo "Installing hpy" + # at the moment this install hpy.devel and hpy.universal. Eventually, we + # will want to split those into two separate packages + local PYTHON="$1" + pushd ${ROOT} + ${PYTHON} -m pip install -U pip + ${PYTHON} -m pip install wheel + ${PYTHON} -m pip install . + popd +} + +_test_pof() { + echo "==== testing pof ====" + # this assumes that pof is already installed, e.g. after calling + # wheel or setup_py_install + ${PYTHON} -m pip install pytest pytest-azurepipelines + cd proof-of-concept + ${PYTHON} -m pytest +} + +_build_wheel() { + HPY_ABI="$1" + local VENV="venv/wheel_builder_$HPY_ABI" + # we use this venv just to build the wheel, and then we install the wheel + # in the currently active virtualenv + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + local PY_BUILDER="`pwd`/$VENV/bin/python3" + if [ -x "`pwd`/$VENV/Scripts/python.exe" ] + then + # Set the correct python executable for Windows + PY_BUILDER="`pwd`/$VENV/Scripts/python.exe" + fi + echo + echo "Installing hpy and requirements" + _install_hpy ${PY_BUILDER} + pushd proof-of-concept + ${PY_BUILDER} -m pip install -r requirements.txt + echo + echo "Building wheel" + ${PY_BUILDER} setup.py --hpy-abi="$HPY_ABI" bdist_wheel + popd +} + +_myrm() { + for path in "$@" + do + if [ -d "$path" -o -f "$path" ] + then + echo "rm $path" + rm -rf "$path" + else + echo "skipping $path" + fi + done +} + +clean() { + echo "=== cleaning up old stuff ===" + _myrm ${ROOT}/venv/wheel_builder_{cpython,universal} + _myrm ${ROOT}/venv/wheel_runner_{cpython,universal} + _myrm ${ROOT}/venv/setup_py_install_{cpython,universal} + _myrm ${ROOT}/venv/setup_py_build_ext_inplace_{cpython,universal} + _myrm ${ROOT}/build + _myrm ${ROOT}/proof-of-concept/build + _myrm ${ROOT}/proof-of-concept/dist + # remove files written by build_ext --inplace + _myrm ${ROOT}/proof-of-concept/pof*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofpackage/foo*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofcpp*{.so,.py} + _myrm ${ROOT}/proof-of-concept/pofpackage/bar*{.so,.py} + echo +} + +wheel() { + # build a wheel, install and test + HPY_ABI="$1" + local VENV="venv/wheel_runner_$HPY_ABI" + clean + echo "=== testing setup.py bdist_wheel" $HPY_ABI "===" + _build_wheel "$HPY_ABI" + WHEEL=`ls proof-of-concept/dist/*.whl` + echo "Wheel created: ${WHEEL}" + echo + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo "Installing wheel" + ${PYTHON} -m pip install $WHEEL + echo + _test_pof +} + +setup_py_install() { + # install proof-of-concept using setup.py install and run tests + HPY_ABI="$1" + VENV="venv/setup_py_install_$HPY_ABI" + clean + echo "=== testing setup.py --hpy-abi=$HPY_ABI install ===" + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo + echo "Running setup.py" + pushd proof-of-concept + ${PYTHON} setup.py --hpy-abi="$HPY_ABI" install + popd + echo + _test_pof +} + +setup_py_build_ext_inplace() { + # install proof-of-concept using setup.py install and run tests + HPY_ABI="$1" + VENV="venv/setup_py_build_ext_inplace_$HPY_ABI" + clean + echo "=== testing setup.py --hpy-abi=$HPY_ABI build_ext --inplace ===" + echo "Create venv: $VENV" + ${PYTHON} -m venv "$VENV" + if [ -e "$VENV/bin/activate" ] ; then + source "$VENV/bin/activate" + else + source "$VENV/Scripts/activate" + fi + _install_hpy ${PYTHON} + echo + echo "Running setup.py" + pushd proof-of-concept + echo python is $(which ${PYTHON}) + ${PYTHON} setup.py --hpy-abi="$HPY_ABI" build_ext --inplace + popd + echo + _test_pof +} + +# ======== main code ======= + +# validate arguments +if [[ "$#" -lt 1 || ( "$#" -lt 2 && "$1" != "clean") ]]; then + echo "Usage: $0 COMMAND [TARGET_ABI]" >&2 + echo "Commands:" >&2 + echo " wheel TARGET_ABI: build a wheel, install and test" + echo " setup_py_install TARGET_ABI: install poc using 'setup.py install' & run tests" >&2 + echo " clean: clean build artifacts" >&2 + echo "Target ABIs:" >&2 + echo " universal: Binary intended for any Python implementation" >&2 + echo " cpython : Binary optimized for CPython" >&2 + exit 1 +fi + +if [ ! -d "proof-of-concept" ] ; then + echo "Script must be run in the repo root" >&2 + exit 1 +fi + +# call the function mentioned as the first arg +COMMAND="$1" +shift +$COMMAND "$@" diff --git a/graalpython/hpy/pyproject.toml b/graalpython/hpy/pyproject.toml new file mode 100644 index 0000000000..a3194aec78 --- /dev/null +++ b/graalpython/hpy/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = [ "setuptools>=64.0", "setuptools-scm[toml]>=6.0", "wheel>=0.34.2",] +build-backend = "setuptools.build_meta" diff --git a/graalpython/hpy/requirements-autogen.txt b/graalpython/hpy/requirements-autogen.txt new file mode 100644 index 0000000000..8c9e2a7384 --- /dev/null +++ b/graalpython/hpy/requirements-autogen.txt @@ -0,0 +1,4 @@ +pycparser==2.21 +py==1.11.0 +packaging==19.2 +attrs==19.3.0 diff --git a/graalpython/hpy/setup.py b/graalpython/hpy/setup.py new file mode 100644 index 0000000000..8dd6962dd2 --- /dev/null +++ b/graalpython/hpy/setup.py @@ -0,0 +1,269 @@ +import sys +import os.path +from setuptools import setup, Extension +from setuptools.command.build_clib import build_clib +import platform + +# this package is supposed to be installed ONLY on CPython. Try to bail out +# with a meaningful error message in other cases. +if sys.implementation.name != 'cpython': + msg = 'ERROR: Cannot install and/or update hpy on this python implementation:\n' + msg += f' sys.implementation.name == {sys.implementation.name!r}\n\n' + if '_hpy_universal' in sys.builtin_module_names: + # this is a python which comes with its own hpy implementation + import _hpy_universal + if hasattr(_hpy_universal, 'get_version'): + hpy_version, git_rev = _hpy_universal.get_version() + msg += f'This python implementation comes with its own version of hpy=={hpy_version}\n' + msg += '\n' + msg += 'If you are trying to install hpy through pip, consider to put the\n' + msg += 'following in your requirements.txt, to make sure that pip will NOT\n' + msg += 'try to re-install it:\n' + msg += f' hpy=={hpy_version}' + else: + msg += 'This python implementation comes with its own version of hpy,\n' + msg += 'but the exact version could not be determined.\n' + # + else: + # this seems to be a python which does not support hpy + msg += 'This python implementation does not seem to support hpy:\n' + msg += '(built-in module _hpy_universal not found).\n' + msg += 'Please contact your vendor for more information.' + sys.exit(msg) + + +this_directory = os.path.abspath(os.path.dirname(__file__)) +with open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: + LONG_DESCRIPTION = f.read() + +if 'HPY_DEBUG_BUILD' in os.environ: + # -fkeep-inline-functions is needed to make sure that the stubs for HPy_* + # functions are available to call inside GDB + EXTRA_COMPILE_ARGS = [ + '-g', '-O0', '-UNDEBUG', + '-fkeep-inline-functions', + # + ## these flags are useful but don't work on all + ## platforms/compilers. Uncomment temporarily if you need them. + #'-Wfatal-errors', # stop after one error (unrelated to warnings) + #'-Werror', # turn warnings into errors + ] +else: + EXTRA_COMPILE_ARGS = [] + +if '_HPY_DEBUG_FORCE_DEFAULT_MEM_PROTECT' not in os.environ: + EXTRA_COMPILE_ARGS += ['-D_HPY_DEBUG_MEM_PROTECT_USEMMAP'] + +if platform.system() == "Windows": + EXTRA_COMPILE_ARGS += ['/WX'] +else: + EXTRA_COMPILE_ARGS += ['-Werror'] + + +def get_scm_config(): + """ + We use this function as a hook to generate version.h before building. + """ + import textwrap + import subprocess + import pathlib + import setuptools_scm + + version = setuptools_scm.get_version() + try: + gitrev = subprocess.check_output('git rev-parse --short HEAD'.split(), + encoding='utf-8') + gitrev = gitrev.strip() + except subprocess.CalledProcessError: + gitrev = "__UNKNOWN__" + + version_h = pathlib.Path('.').joinpath('hpy', 'devel', 'include', 'hpy', 'version.h') + version_h.write_text(textwrap.dedent(f""" + // automatically generated by setup.py:get_scm_config() + #define HPY_VERSION "{version}" + #define HPY_GIT_REVISION "{gitrev}" + """)) + + version_py = pathlib.Path('.').joinpath('hpy', 'devel', 'version.py') + version_py.write_text(textwrap.dedent(f""" + # automatically generated by setup.py:get_scm_config() + __version__ = "{version}" + __git_revision__ = "{gitrev}" + """)) + + return {} # use the default config + +HPY_EXTRA_SOURCES = [ + 'hpy/devel/src/runtime/argparse.c', + 'hpy/devel/src/runtime/buildvalue.c', + 'hpy/devel/src/runtime/format.c', + 'hpy/devel/src/runtime/helpers.c', + 'hpy/devel/src/runtime/structseq.c', +] + +HPY_CTX_SOURCES = [ + 'hpy/devel/src/runtime/ctx_bytes.c', + 'hpy/devel/src/runtime/ctx_call.c', + 'hpy/devel/src/runtime/ctx_capsule.c', + 'hpy/devel/src/runtime/ctx_err.c', + 'hpy/devel/src/runtime/ctx_eval.c', + 'hpy/devel/src/runtime/ctx_long.c', + 'hpy/devel/src/runtime/ctx_module.c', + 'hpy/devel/src/runtime/ctx_object.c', + 'hpy/devel/src/runtime/ctx_type.c', + 'hpy/devel/src/runtime/ctx_tracker.c', + 'hpy/devel/src/runtime/ctx_listbuilder.c', + 'hpy/devel/src/runtime/ctx_tuple.c', + 'hpy/devel/src/runtime/ctx_tuplebuilder.c', + 'hpy/devel/src/runtime/ctx_contextvar.c', +] + +HPY_INCLUDE_DIRS = [ + 'hpy/devel/include', + 'hpy/universal/src', + 'hpy/debug/src/include', + 'hpy/trace/src/include', +] + +HPY_EXTRA_UNIVERSAL_LIB_NAME = "hpy-extra-universal" +HPY_EXTRA_HYBRID_LIB_NAME = "hpy-extra-hybrid" +HPY_CTX_LIB_NAME = "hpy-ctx-cpython" + +HPY_BUILD_CLIB_ABI_ATTR = "hpy_abi" + +class build_clib_hpy(build_clib): + """ Special build_clib command for building HPy's static libraries defined + by 'STATIC_LIBS' below. The behavior differs in following points: + (1) Option 'force' is set such that static libs will always be renewed. + (2) Method 'get_library_names' always returns 'None'. This is because + we only use this command to build static libraries for testing. + That means, we only use them in-place. We don't need them for + linking here. + (3) This command consumes a custom build info key + HPY_BUILD_CLIB_ABI_ATTR that is used to create separate build + temp directories for each ABI. This is necessary to avoid + incorrect sharing of (temporary) build artifacts. + (4) This command will use the include directories from command + 'build_ext'. + """ + def finalize_options(self): + super().finalize_options() + # we overwrite the include dirs and use the ones from 'build_ext' + build_ext_includes = self.get_finalized_command('build_ext').include_dirs or [] + self.include_dirs = HPY_INCLUDE_DIRS + build_ext_includes + self.force = 1 + + def get_library_names(self): + # We only build static libraries for testing. We just use them + # in-place. We don't want that our extensions (i.e. 'hpy.universal' + # etc) link to these libs. + return None + + def build_libraries(self, libraries): + # we just inherit the 'inplace' option from 'build_ext' + build_ext = self.get_finalized_command('build_ext') + inplace = build_ext.inplace + if inplace: + # the inplace option requires to find the package directory + # using the build_py command for that + build_py = self.get_finalized_command('build_py') + lib_dir = os.path.abspath(build_py.get_package_dir('hpy.devel')) + else: + lib_dir = os.path.join(build_ext.build_lib, 'hpy', 'devel') + + import pathlib + for lib in libraries: + lib_name, build_info = lib + abi = build_info.get(HPY_BUILD_CLIB_ABI_ATTR) + # Call super's build_libraries with just one library in the list + # such that we can temporarily change the 'build_temp'. + orig_build_temp = self.build_temp + orig_build_clib = self.build_clib + self.build_temp = os.path.join(orig_build_temp, 'lib', abi) + self.build_clib = os.path.join(lib_dir, 'lib', abi) + # ensure that 'build_clib' directory exists + pathlib.Path(self.build_clib).mkdir(parents=True, exist_ok=True) + try: + super().build_libraries([lib]) + finally: + self.build_temp = orig_build_temp + self.build_clib = orig_build_clib + + +STATIC_LIBS = [(HPY_EXTRA_UNIVERSAL_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'universal', + 'macros': [('HPY_ABI_UNIVERSAL', None)]}), + (HPY_EXTRA_HYBRID_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'hybrid', + 'macros': [('HPY_ABI_HYBRID', None)]}), + (HPY_CTX_LIB_NAME, + {'sources': HPY_EXTRA_SOURCES + HPY_CTX_SOURCES, + HPY_BUILD_CLIB_ABI_ATTR: 'cpython', + 'macros': [('HPY_ABI_CPYTHON', None)]})] + +EXT_MODULES = [ + Extension('hpy.universal', + ['hpy/universal/src/hpymodule.c', + 'hpy/universal/src/ctx.c', + 'hpy/universal/src/ctx_meth.c', + 'hpy/universal/src/ctx_misc.c', + 'hpy/debug/src/debug_ctx.c', + 'hpy/debug/src/debug_ctx_cpython.c', + 'hpy/debug/src/debug_handles.c', + 'hpy/debug/src/dhqueue.c', + 'hpy/debug/src/memprotect.c', + 'hpy/debug/src/stacktrace.c', + 'hpy/debug/src/_debugmod.c', + 'hpy/debug/src/autogen_debug_wrappers.c', + 'hpy/trace/src/trace_ctx.c', + 'hpy/trace/src/_tracemod.c', + 'hpy/trace/src/autogen_trace_wrappers.c', + 'hpy/trace/src/autogen_trace_func_table.c'] + + HPY_EXTRA_SOURCES + + HPY_CTX_SOURCES, + include_dirs=HPY_INCLUDE_DIRS, + extra_compile_args=[ + # so we need to enable the HYBRID ABI in order to implement + # the legacy features + '-DHPY_ABI_HYBRID', + '-DHPY_DEBUG_ENABLE_UHPY_SANITY_CHECK', + '-DHPY_EMBEDDED_MODULES', + ] + EXTRA_COMPILE_ARGS + ) + ] + +DEV_REQUIREMENTS = [ + "pytest", + "pytest-xdist", + "filelock", +] + +setup( + name="hpy", + author='The HPy team', + author_email='hpy-dev@python.org', + url='https://hpyproject.org', + license='MIT', + description='A better C API for Python', + long_description=LONG_DESCRIPTION, + long_description_content_type='text/markdown', + packages=['hpy.devel', 'hpy.debug', 'hpy.trace'], + include_package_data=True, + extras_require={ + "dev": DEV_REQUIREMENTS, + }, + libraries=STATIC_LIBS, + ext_modules=EXT_MODULES, + entry_points={ + "distutils.setup_keywords": [ + "hpy_ext_modules = hpy.devel:handle_hpy_ext_modules", + ], + }, + cmdclass={"build_clib": build_clib_hpy}, + use_scm_version=get_scm_config, + setup_requires=['setuptools_scm'], + install_requires=['setuptools>=64.0'], + python_requires='>=3.8', +) diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py b/graalpython/hpy/test/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/__init__.py rename to graalpython/hpy/test/__init__.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/check_py27_compat.py b/graalpython/hpy/test/check_py27_compat.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/check_py27_compat.py rename to graalpython/hpy/test/check_py27_compat.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py b/graalpython/hpy/test/conftest.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/conftest.py rename to graalpython/hpy/test/conftest.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py b/graalpython/hpy/test/debug/__init__.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/__init__.py rename to graalpython/hpy/test/debug/__init__.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_builder_invalid.py b/graalpython/hpy/test/debug/test_builder_invalid.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_builder_invalid.py rename to graalpython/hpy/test/debug/test_builder_invalid.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py b/graalpython/hpy/test/debug/test_charptr.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_charptr.py rename to graalpython/hpy/test/debug/test_charptr.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_context_reuse.py b/graalpython/hpy/test/debug/test_context_reuse.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_context_reuse.py rename to graalpython/hpy/test/debug/test_context_reuse.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py b/graalpython/hpy/test/debug/test_handles_invalid.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_invalid.py rename to graalpython/hpy/test/debug/test_handles_invalid.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_leak.py b/graalpython/hpy/test/debug/test_handles_leak.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_handles_leak.py rename to graalpython/hpy/test/debug/test_handles_leak.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py b/graalpython/hpy/test/debug/test_misc.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/debug/test_misc.py rename to graalpython/hpy/test/debug/test_misc.py diff --git a/graalpython/hpy/test/hpy_devel/__init__.py b/graalpython/hpy/test/hpy_devel/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_abitag.py b/graalpython/hpy/test/hpy_devel/test_abitag.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_abitag.py rename to graalpython/hpy/test/hpy_devel/test_abitag.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py b/graalpython/hpy/test/hpy_devel/test_distutils.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/hpy_devel/test_distutils.py rename to graalpython/hpy/test/hpy_devel/test_distutils.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py b/graalpython/hpy/test/support.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/support.py rename to graalpython/hpy/test/support.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py b/graalpython/hpy/test/test_00_basic.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_00_basic.py rename to graalpython/hpy/test/test_00_basic.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py b/graalpython/hpy/test/test_argparse.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_argparse.py rename to graalpython/hpy/test/test_argparse.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_call.py b/graalpython/hpy/test/test_call.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_call.py rename to graalpython/hpy/test/test_call.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py b/graalpython/hpy/test/test_capsule.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule.py rename to graalpython/hpy/test/test_capsule.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule_legacy.py b/graalpython/hpy/test/test_capsule_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_capsule_legacy.py rename to graalpython/hpy/test/test_capsule_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_contextvar.py b/graalpython/hpy/test/test_contextvar.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_contextvar.py rename to graalpython/hpy/test/test_contextvar.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_cpy_compat.py b/graalpython/hpy/test/test_cpy_compat.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_cpy_compat.py rename to graalpython/hpy/test/test_cpy_compat.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py b/graalpython/hpy/test/test_eval.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_eval.py rename to graalpython/hpy/test/test_eval.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_helpers.py b/graalpython/hpy/test/test_helpers.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_helpers.py rename to graalpython/hpy/test/test_helpers.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybuildvalue.py b/graalpython/hpy/test/test_hpybuildvalue.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybuildvalue.py rename to graalpython/hpy/test/test_hpybuildvalue.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybytes.py b/graalpython/hpy/test/test_hpybytes.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpybytes.py rename to graalpython/hpy/test/test_hpybytes.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpydict.py b/graalpython/hpy/test/test_hpydict.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpydict.py rename to graalpython/hpy/test/test_hpydict.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyerr.py b/graalpython/hpy/test/test_hpyerr.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyerr.py rename to graalpython/hpy/test/test_hpyerr.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py b/graalpython/hpy/test/test_hpyfield.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyfield.py rename to graalpython/hpy/test/test_hpyfield.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyglobal.py b/graalpython/hpy/test/test_hpyglobal.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyglobal.py rename to graalpython/hpy/test/test_hpyglobal.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyimport.py b/graalpython/hpy/test/test_hpyimport.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyimport.py rename to graalpython/hpy/test/test_hpyimport.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py b/graalpython/hpy/test/test_hpyiter.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyiter.py rename to graalpython/hpy/test/test_hpyiter.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py b/graalpython/hpy/test/test_hpylist.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylist.py rename to graalpython/hpy/test/test_hpylist.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylong.py b/graalpython/hpy/test/test_hpylong.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpylong.py rename to graalpython/hpy/test/test_hpylong.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py b/graalpython/hpy/test/test_hpymodule.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpymodule.py rename to graalpython/hpy/test/test_hpymodule.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py b/graalpython/hpy/test/test_hpyslice.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyslice.py rename to graalpython/hpy/test/test_hpyslice.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpystructseq.py b/graalpython/hpy/test/test_hpystructseq.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpystructseq.py rename to graalpython/hpy/test/test_hpystructseq.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytuple.py b/graalpython/hpy/test/test_hpytuple.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytuple.py rename to graalpython/hpy/test/test_hpytuple.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py b/graalpython/hpy/test/test_hpytype.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype.py rename to graalpython/hpy/test/test_hpytype.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py b/graalpython/hpy/test/test_hpytype_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpytype_legacy.py rename to graalpython/hpy/test/test_hpytype_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py b/graalpython/hpy/test/test_hpyunicode.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_hpyunicode.py rename to graalpython/hpy/test/test_hpyunicode.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py b/graalpython/hpy/test/test_importing.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_importing.py rename to graalpython/hpy/test/test_importing.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py b/graalpython/hpy/test/test_legacy_forbidden.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_legacy_forbidden.py rename to graalpython/hpy/test/test_legacy_forbidden.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_number.py b/graalpython/hpy/test/test_number.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_number.py rename to graalpython/hpy/test/test_number.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py b/graalpython/hpy/test/test_object.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_object.py rename to graalpython/hpy/test/test_object.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py b/graalpython/hpy/test/test_slots.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots.py rename to graalpython/hpy/test/test_slots.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots_legacy.py b/graalpython/hpy/test/test_slots_legacy.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_slots_legacy.py rename to graalpython/hpy/test/test_slots_legacy.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_support.py b/graalpython/hpy/test/test_support.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_support.py rename to graalpython/hpy/test/test_support.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_tracker.py b/graalpython/hpy/test/test_tracker.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/test_tracker.py rename to graalpython/hpy/test/test_tracker.py diff --git a/graalpython/com.oracle.graal.python.hpy.test/src/hpytest/trace/test_trace.py b/graalpython/hpy/test/trace/test_trace.py similarity index 100% rename from graalpython/com.oracle.graal.python.hpy.test/src/hpytest/trace/test_trace.py rename to graalpython/hpy/test/trace/test_trace.py diff --git a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c b/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c deleted file mode 100644 index 17a3909f8e..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/devel/src/runtime/ctx_tracker.c +++ /dev/null @@ -1,155 +0,0 @@ -/** - * A manager for HPy handles, allowing handles to be tracked - * and closed as a group. - * - * Note:: - * Calling HPyTracker_New(ctx, n) will ensure that at least n handles - * can be tracked without a call to HPyTracker_Add failing. - * - * If a call to HPyTracker_Add fails, the tracker still guarantees that - * the handle passed to it has been tracked (internally it does this by - * maintaining space for one more handle). - * - * After HPyTracker_Add fails, HPyTracker_Close should be called without - * any further calls to HPyTracker_Add. Calling HPyTracker_Close will close - * all the tracked handles, including the handled passed to the failed call - * to HPyTracker_Add. - * - * Example usage (inside an HPyDef_METH function):: - * - * long i; - * HPy key, value; - * HPyTracker ht; - * - * ht = HPyTracker_New(ctx, 0); // track the key-value pairs - * if (HPy_IsNull(ht)) - * return HPy_NULL; - * - * HPy dict = HPyDict_New(ctx); - * if (HPy_IsNull(dict)) - * goto error; - * - * for (i=0; i<5; i++) { - * key = HPyLong_FromLong(ctx, i); - * if (HPy_IsNull(key)) - * goto error; - * if (HPyTracker_Add(ctx, ht, key) < 0) - * goto error; - * value = HPyLong_FromLong(ctx, i * i); - * if (HPy_IsNull(value)) { - * goto error; - * } - * if (HPyTracker_Add(ctx, ht, value) < 0) - * goto error; - * result = HPy_SetItem(ctx, dict, key, value); - * if (result < 0) - * goto error; - * } - * - * success: - * HPyTracker_Close(ctx, ht); - * return dict; - * - * error: - * HPyTracker_Close(ctx, ht); - * HPy_Close(ctx, dict); - * // HPyErr will already have been set by the error that occurred. - * return HPy_NULL; - */ - -#include "hpy.h" - -static const HPy_ssize_t HPYTRACKER_INITIAL_CAPACITY = 5; - -typedef struct { - HPy_ssize_t capacity; // allocated handles - HPy_ssize_t length; // used handles - HPy *handles; -} _HPyTracker_s; - - -static inline _HPyTracker_s *_ht2hp(HPyTracker ht) { - return (_HPyTracker_s *) (ht)._i; -} -static inline HPyTracker _hp2ht(_HPyTracker_s *hp) { - return (HPyTracker) {(HPy_ssize_t) (hp)}; -} - - -_HPy_HIDDEN HPyTracker -ctx_Tracker_New(HPyContext *ctx, HPy_ssize_t capacity) -{ - _HPyTracker_s *hp; - if (capacity == 0) { - capacity = HPYTRACKER_INITIAL_CAPACITY; - } - capacity++; // always reserve space for an extra handle, see the docs - - hp = (_HPyTracker_s*)malloc(sizeof(_HPyTracker_s)); - if (hp == NULL) { - HPyErr_NoMemory(ctx); - return _hp2ht(0); - } - hp->handles = (HPy*)calloc(capacity, sizeof(HPy)); - if (hp->handles == NULL) { - free(hp); - HPyErr_NoMemory(ctx); - return _hp2ht(0); - } - hp->capacity = capacity; - hp->length = 0; - return _hp2ht(hp); -} - -static int -tracker_resize(HPyContext *ctx, _HPyTracker_s *hp, HPy_ssize_t capacity) -{ - HPy *new_handles; - capacity++; - - if (capacity <= hp->length) { - // refuse a resize that would either 1) lose handles or 2) not leave - // space for one new handle - HPyErr_SetString(ctx, ctx->h_ValueError, "HPyTracker resize would lose handles"); - return -1; - } - new_handles = (HPy*)realloc(hp->handles, capacity * sizeof(HPy)); - if (new_handles == NULL) { - HPyErr_NoMemory(ctx); - return -1; - } - hp->capacity = capacity; - hp->handles = new_handles; - return 0; -} - -_HPy_HIDDEN int -ctx_Tracker_Add(HPyContext *ctx, HPyTracker ht, HPy h) -{ - _HPyTracker_s *hp = _ht2hp(ht); - hp->handles[hp->length++] = h; - if (hp->capacity <= hp->length) { - if (tracker_resize(ctx, hp, hp->capacity * 2 - 1) < 0) - return -1; - } - return 0; -} - -_HPy_HIDDEN void -ctx_Tracker_ForgetAll(HPyContext *ctx, HPyTracker ht) -{ - _HPyTracker_s *hp = _ht2hp(ht); - hp->length = 0; -} - -_HPy_HIDDEN void -ctx_Tracker_Close(HPyContext *ctx, HPyTracker ht) -{ - _HPyTracker_s *hp = _ht2hp(ht); - HPy_ssize_t i; - for (i=0; ilength; i++) { - HPy_Close(ctx, hp->handles[i]); - } - free(hp->handles); - free(hp); -} diff --git a/graalpython/lib-graalpython/modules/hpy/devel/version.py b/graalpython/lib-graalpython/modules/hpy/devel/version.py deleted file mode 100644 index 0cd1dbcc22..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/devel/version.py +++ /dev/null @@ -1,4 +0,0 @@ - -# automatically generated by setup.py:get_scm_config() -__version__ = "0.9.1.dev79+gb0fbdf73" -__git_revision__ = "b0fbdf73" diff --git a/graalpython/lib-graalpython/modules/hpy/trace/__init__.py b/graalpython/lib-graalpython/modules/hpy/trace/__init__.py deleted file mode 100644 index 71ece53923..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from .leakdetector import HPyDebugError, HPyLeakError, LeakDetector - - -def set_handle_stack_trace_limit(limit): - from hpy.universal import _debug - _debug.set_handle_stack_trace_limit(limit) - - -def disable_handle_stack_traces(): - from hpy.universal import _debug - _debug.set_handle_stack_trace_limit(None) diff --git a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py b/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py deleted file mode 100644 index 56a68d344c..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/leakdetector.py +++ /dev/null @@ -1,43 +0,0 @@ -from hpy.universal import _debug - -class HPyDebugError(Exception): - pass - -class HPyLeakError(HPyDebugError): - def __init__(self, leaks): - super().__init__() - self.leaks = leaks - - def __str__(self): - lines = [] - n = len(self.leaks) - s = 's' if n != 1 else '' - lines.append(f'{n} unclosed handle{s}:') - for dh in self.leaks: - lines.append(' %r' % dh) - return '\n'.join(lines) - - -class LeakDetector: - - def __init__(self): - self.generation = None - - def start(self): - if self.generation is not None: - raise ValueError('LeakDetector already started') - self.generation = _debug.new_generation() - - def stop(self): - if self.generation is None: - raise ValueError('LeakDetector not started yet') - leaks = _debug.get_open_handles(self.generation) - if leaks: - raise HPyLeakError(leaks) - - def __enter__(self): - self.start() - return self - - def __exit__(self, etype, evalue, tb): - self.stop() diff --git a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py b/graalpython/lib-graalpython/modules/hpy/trace/pytest.py deleted file mode 100644 index 9a33c51343..0000000000 --- a/graalpython/lib-graalpython/modules/hpy/trace/pytest.py +++ /dev/null @@ -1,31 +0,0 @@ -# hpy.debug / pytest integration - -import pytest -from .leakdetector import LeakDetector - -# For now "hpy_debug" just does leak detection, but in the future it might -# grows extra features: that's why it's called generically "hpy_debug" instead -# of "detect_leaks". - -# NOTE: the fixture itself is currently untested :(. It turns out that testing -# that the fixture raises during the teardown is complicated and probably -# requires to write a full-fledged plugin. We might want to turn this into a -# real plugin in the future, but for now I think this is enough. - -# pypy still uses a very ancient version of pytest, 2.9.2: pytest<3.x needs to -# use @yield_fixture, which is deprecated in newer version of pytest (where -# you can just use @fixture) -if pytest.__version__ < '3': - fixture = pytest.yield_fixture -else: - fixture = pytest.fixture - -@fixture -def hpy_debug(request): - """ - pytest fixture which makes it possible to control hpy.debug from within a test. - - In particular, it automatically check that the test doesn't leak. - """ - with LeakDetector() as ld: - yield ld From 56103bff65acc629c507ccbaecc48d0d2a1b6d92 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Thu, 10 Apr 2025 13:41:37 +0200 Subject: [PATCH 3/3] Update to hpy 79571e2c558318f10be6f04040e6781ebbadb86f --- graalpython/hpy/.github/workflows/valgrind-tests.yml | 2 +- graalpython/hpy/hpy/universal/src/hpymodule.c | 3 --- graalpython/hpy/setup.py | 6 +++--- graalpython/hpy/test/conftest.py | 8 ++++++++ graalpython/hpy/test/debug/test_charptr.py | 6 +++++- graalpython/hpy/test/debug/test_context_reuse.py | 5 +++++ graalpython/hpy/test/debug/test_handles_invalid.py | 11 ++++++++++- graalpython/hpy/test/debug/test_misc.py | 8 +++++++- graalpython/hpy/test/hpy_devel/test_distutils.py | 11 +++++++---- graalpython/hpy/test/support.py | 3 +++ graalpython/hpy/test/test_cpy_compat.py | 7 ++++++- graalpython/hpy/test/test_hpyerr.py | 5 ++++- graalpython/hpy/test/test_hpyfield.py | 7 ++++++- graalpython/hpy/test/test_hpytype.py | 3 +++ graalpython/hpy/test/test_object.py | 6 ++++++ 15 files changed, 74 insertions(+), 17 deletions(-) diff --git a/graalpython/hpy/.github/workflows/valgrind-tests.yml b/graalpython/hpy/.github/workflows/valgrind-tests.yml index badbc9da6e..ab739252eb 100644 --- a/graalpython/hpy/.github/workflows/valgrind-tests.yml +++ b/graalpython/hpy/.github/workflows/valgrind-tests.yml @@ -23,7 +23,7 @@ jobs: python-version: 3.9 - name: Install / Upgrade Python dependencies - run: python -m pip install --upgrade pip wheel + run: python -m pip install --upgrade pip wheel setuptools - name: Build run: | diff --git a/graalpython/hpy/hpy/universal/src/hpymodule.c b/graalpython/hpy/hpy/universal/src/hpymodule.c index b86e418e61..6ae8b54859 100644 --- a/graalpython/hpy/hpy/universal/src/hpymodule.c +++ b/graalpython/hpy/hpy/universal/src/hpymodule.c @@ -22,9 +22,6 @@ #ifdef PYPY_VERSION # error "Cannot build hpy.universal on top of PyPy. PyPy comes with its own version of it" #endif -#ifdef GRAALVM_PYTHON -# error "Cannot build hpy.universal on top of GraalPy. GraalPy comes with its own version of it" -#endif static const char *hpy_mode_names[] = { "MODE_UNIVERSAL", diff --git a/graalpython/hpy/setup.py b/graalpython/hpy/setup.py index 8dd6962dd2..9a1cb9f2eb 100644 --- a/graalpython/hpy/setup.py +++ b/graalpython/hpy/setup.py @@ -4,9 +4,9 @@ from setuptools.command.build_clib import build_clib import platform -# this package is supposed to be installed ONLY on CPython. Try to bail out -# with a meaningful error message in other cases. -if sys.implementation.name != 'cpython': +# this package is supposed to be installed ONLY on CPython and GraalPy. Try to +# bail out with a meaningful error message in other cases. +if sys.implementation.name not in ('cpython', 'graalpy'): msg = 'ERROR: Cannot install and/or update hpy on this python implementation:\n' msg += f' sys.implementation.name == {sys.implementation.name!r}\n\n' if '_hpy_universal' in sys.builtin_module_names: diff --git a/graalpython/hpy/test/conftest.py b/graalpython/hpy/test/conftest.py index 372843459f..01c5a61103 100644 --- a/graalpython/hpy/test/conftest.py +++ b/graalpython/hpy/test/conftest.py @@ -1,4 +1,5 @@ import pytest +import sys from .support import ExtensionCompiler, DefaultExtensionTemplate,\ PythonSubprocessRunner, HPyDebugCapture, make_hpy_abi_fixture from pathlib import Path @@ -32,6 +33,13 @@ def pytest_configure(config): "markers", "syncgc: Mark tests that rely on a synchronous GC." ) + +def pytest_runtest_setup(item): + if any(item.iter_markers(name="syncgc")): + if sys.implementation.name in ("pypy", "graalpy"): + pytest.skip("requires synchronous garbage collector") + + # this is the default set of hpy_abi for all the tests. Individual files and # classes can override it. hpy_abi = make_hpy_abi_fixture('default') diff --git a/graalpython/hpy/test/debug/test_charptr.py b/graalpython/hpy/test/debug/test_charptr.py index 7df8d8f2c3..cc1fb9406f 100644 --- a/graalpython/hpy/test/debug/test_charptr.py +++ b/graalpython/hpy/test/debug/test_charptr.py @@ -1,6 +1,7 @@ import os import pytest -from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +import sys +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION, IS_GRAALPY # Tests detection of usage of char pointers associated with invalid already # closed handles. For now, the debug mode does not provide any hook for this @@ -13,6 +14,7 @@ def hpy_abi(): yield "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_charptr_use_after_implicit_arg_handle_close(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -70,6 +72,7 @@ def test_charptr_use_after_implicit_arg_handle_close(compiler, python_subprocess assert b"UnicodeDecodeError" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_charptr_use_after_handle_close(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -121,6 +124,7 @@ def test_charptr_use_after_handle_close(compiler, python_subprocess): assert b"UnicodeDecodeError" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="transiently fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_MEM_PROTECTION, reason= "Could be implemented by checking the contents on close.") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") diff --git a/graalpython/hpy/test/debug/test_context_reuse.py b/graalpython/hpy/test/debug/test_context_reuse.py index e3a57d3163..164505cf31 100644 --- a/graalpython/hpy/test/debug/test_context_reuse.py +++ b/graalpython/hpy/test/debug/test_context_reuse.py @@ -1,10 +1,15 @@ import pytest +import sys + +from ..support import IS_GRAALPY + @pytest.fixture def hpy_abi(): return "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="Hangs on GraalPy") def test_reuse_context_from_global_variable(compiler, python_subprocess): mod = compiler.compile_module(""" #include diff --git a/graalpython/hpy/test/debug/test_handles_invalid.py b/graalpython/hpy/test/debug/test_handles_invalid.py index 36622eece9..355470c088 100644 --- a/graalpython/hpy/test/debug/test_handles_invalid.py +++ b/graalpython/hpy/test/debug/test_handles_invalid.py @@ -1,7 +1,7 @@ import pytest import sys from hpy.debug.leakdetector import LeakDetector -from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD +from ..support import SUPPORTS_SYS_EXECUTABLE, IS_PYTHON_DEBUG_BUILD, IS_GRAALPY from ..conftest import IS_VALGRIND_RUN @pytest.fixture @@ -12,6 +12,8 @@ def hpy_abi(): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_no_invalid_handle(compiler, hpy_debug_capture): # Basic sanity check that valid code does not trigger any error reports mod = compiler.make_module(""" @@ -38,6 +40,8 @@ def test_no_invalid_handle(compiler, hpy_debug_capture): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_cant_use_closed_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPyDef_METH(f, "f", HPyFunc_O, .doc="double close") @@ -113,6 +117,8 @@ def test_cant_use_closed_handle(compiler, hpy_debug_capture): @pytest.mark.skipif(sys.implementation.name == 'pypy', reason="Cannot recover from use-after-close on pypy") +@pytest.mark.skipif(IS_GRAALPY, + reason="This corrupts process memory on GraalPy and crashes later") def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): mod = compiler.make_module(""" HPy keep; @@ -142,6 +148,7 @@ def test_keeping_and_reusing_argument_handle(compiler, hpy_debug_capture): assert hpy_debug_capture.invalid_handles_count == 1 +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_return_ctx_constant_without_dup(compiler, python_subprocess, fatal_exit_code): # Since this puts the context->h_None into an inconsistent state, we run # this test in a subprocess and check fatal error instead @@ -163,6 +170,7 @@ def test_return_ctx_constant_without_dup(compiler, python_subprocess, fatal_exit assert b"Invalid usage of already closed handle" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_close_ctx_constant(compiler, python_subprocess, fatal_exit_code): # Since this puts the context->h_True into an inconsistent state, we run # this test in a subprocess and check fatal error instead @@ -185,6 +193,7 @@ def test_close_ctx_constant(compiler, python_subprocess, fatal_exit_code): assert b"Invalid usage of already closed handle" in result.stderr +@pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_invalid_handle_crashes_python_if_no_hook(compiler, python_subprocess, fatal_exit_code): if not SUPPORTS_SYS_EXECUTABLE: pytest.skip("no sys.executable") diff --git a/graalpython/hpy/test/debug/test_misc.py b/graalpython/hpy/test/debug/test_misc.py index 2994da4b3e..3640459798 100644 --- a/graalpython/hpy/test/debug/test_misc.py +++ b/graalpython/hpy/test/debug/test_misc.py @@ -1,11 +1,13 @@ import pytest -from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION +import sys +from ..support import SUPPORTS_SYS_EXECUTABLE, SUPPORTS_MEM_PROTECTION, IS_GRAALPY @pytest.fixture def hpy_abi(): return "debug" +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_use_invalid_as_struct(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -38,6 +40,7 @@ def test_use_invalid_as_struct(compiler, python_subprocess): assert "Invalid usage of _HPy_AsStruct_Object" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_typecheck(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -60,6 +63,7 @@ def test_typecheck(compiler, python_subprocess): assert "HPy_TypeCheck arg 2 must be a type" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_MEM_PROTECTION, reason= "Could be implemented by checking the contents on close.") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") @@ -119,6 +123,7 @@ def test_type_getname(compiler, python_subprocess): assert result.returncode != 0 +@pytest.mark.skipif(IS_GRAALPY, reason="hangs on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_type_issubtype(compiler, python_subprocess): mod = compiler.compile_module(""" @@ -143,6 +148,7 @@ def test_type_issubtype(compiler, python_subprocess): assert "HPyType_IsSubtype arg 1 must be a type" in result.stderr.decode("utf-8") +@pytest.mark.skipif(IS_GRAALPY, reason="transiently fails on GraalPy") @pytest.mark.skipif(not SUPPORTS_SYS_EXECUTABLE, reason="needs subprocess") def test_unicode_substring(compiler, python_subprocess): mod = compiler.compile_module(""" diff --git a/graalpython/hpy/test/hpy_devel/test_distutils.py b/graalpython/hpy/test/hpy_devel/test_distutils.py index f6b2daae39..62904f0957 100644 --- a/graalpython/hpy/test/hpy_devel/test_distutils.py +++ b/graalpython/hpy/test/hpy_devel/test_distutils.py @@ -16,7 +16,7 @@ import py import pytest -from ..support import atomic_run, HPY_ROOT +from ..support import atomic_run, HPY_ROOT, IS_GRAALPY # ====== IMPORTANT DEVELOPMENT TIP ===== # You can use py.test --reuse-venv to speed up local testing. @@ -43,7 +43,7 @@ def print_CalledProcessError(p): @pytest.fixture(scope='session') def venv_template(request, tmpdir_factory): - if request.config.option.reuse_venv: + if getattr(request.config.option, "reuse_venv", False): d = py.path.local('/tmp/venv-for-hpytest') if d.check(dir=True): # if it exists, we assume it's correct. If you want to recreate, @@ -59,7 +59,7 @@ def venv_template(request, tmpdir_factory): # it's just easier to use e.g. python -m pip attach_python_to_venv(d) for script in d.bin.listdir(): - if script.basename.startswith('python'): + if script.basename.startswith(('python', 'graalpy')): continue script.remove() # @@ -95,7 +95,7 @@ def initargs(self, pytestconfig, tmpdir, venv_template): self.tmpdir = tmpdir # create a fresh venv by copying the template self.venv = tmpdir.join('venv') - shutil.copytree(venv_template, self.venv) + shutil.copytree(venv_template, self.venv, symlinks=True) attach_python_to_venv(self.venv) # create the files for our test project self.hpy_test_project = tmpdir.join('hpy_test_project').ensure(dir=True) @@ -290,6 +290,7 @@ def test_hpymod_wheel(self, hpy_abi): doc = self.get_docstring('hpymod') assert doc == f'hpymod with HPy ABI: {hpy_abi}' + @pytest.mark.skipif(IS_GRAALPY, reason='not supported on GraalPy') def test_dont_mix_cpython_and_universal_abis(self): """ See issue #322 @@ -328,6 +329,8 @@ def test_dont_mix_cpython_and_universal_abis(self): def test_hpymod_legacy(self, hpy_abi): if hpy_abi == 'universal': pytest.skip('only for cpython and hybrid ABIs') + if IS_GRAALPY: + pytest.skip('not supported on GraalPy') self.gen_setup_py(""" setup(name = "hpy_test_project", hpy_ext_modules = [hpymod_legacy], diff --git a/graalpython/hpy/test/support.py b/graalpython/hpy/test/support.py index 250640548f..d102e398e7 100644 --- a/graalpython/hpy/test/support.py +++ b/graalpython/hpy/test/support.py @@ -19,6 +19,8 @@ # True if we are running on the CPython debug build IS_PYTHON_DEBUG_BUILD = hasattr(sys, 'gettotalrefcount') +IS_GRAALPY = getattr(getattr(sys, "implementation", None), "name", None) == 'graalpy' + # pytest marker to run tests only on linux ONLY_LINUX = pytest.mark.skipif(sys.platform!='linux', reason='linux only') @@ -441,6 +443,7 @@ def run(self, mod, code): @pytest.mark.usefixtures('initargs') class HPyTest: + is_graalpy = IS_GRAALPY # Exposed here for tests running as PyPy apptests ExtensionTemplate = DefaultExtensionTemplate @pytest.fixture() diff --git a/graalpython/hpy/test/test_cpy_compat.py b/graalpython/hpy/test/test_cpy_compat.py index 5a7866c335..8151be0dd9 100644 --- a/graalpython/hpy/test/test_cpy_compat.py +++ b/graalpython/hpy/test/test_cpy_compat.py @@ -1,4 +1,5 @@ -from .support import HPyTest, make_hpy_abi_fixture +import pytest +from .support import HPyTest, make_hpy_abi_fixture, IS_GRAALPY class TestCPythonCompatibility(HPyTest): @@ -30,6 +31,7 @@ def test_abi(self): expected = 'hybrid' assert hpy_abi == expected + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_frompyobject(self): mod = self.make_module(""" #include @@ -132,6 +134,7 @@ def foo(self): obj = MyClass() assert mod.f(obj) == 1234 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_hpy_close(self): mod = self.make_module(""" #include @@ -156,6 +159,7 @@ def test_hpy_close(self): if self.supports_refcounts(): assert x == -1 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_hpy_dup(self): mod = self.make_module(""" #include @@ -182,6 +186,7 @@ def test_hpy_dup(self): if self.supports_refcounts(): assert x == +1 + @pytest.mark.skipif(IS_GRAALPY, reason="Crashes on GraalPy") def test_many_handles(self): mod = self.make_module(""" #include diff --git a/graalpython/hpy/test/test_hpyerr.py b/graalpython/hpy/test/test_hpyerr.py index 6adc8d51df..20426aaa31 100644 --- a/graalpython/hpy/test/test_hpyerr.py +++ b/graalpython/hpy/test/test_hpyerr.py @@ -1,4 +1,5 @@ -from .support import HPyTest, SUPPORTS_SYS_EXECUTABLE, trampoline +import pytest +from .support import HPyTest, SUPPORTS_SYS_EXECUTABLE, trampoline, IS_GRAALPY class TestErr(HPyTest): @@ -17,6 +18,7 @@ def test_NoMemory(self): with pytest.raises(MemoryError): mod.f() + @pytest.mark.skipif(IS_GRAALPY, reason="Fails transiently on GraalPy especially with xdist") def test_FatalError(self, python_subprocess, fatal_exit_code): mod = self.compile_module(""" HPyDef_METH(f, "f", HPyFunc_NOARGS) @@ -672,6 +674,7 @@ def do_not_raise_exception(*args): with pytest.raises(DummyException): mod.f(raise_exception, (DummyException, ), exc_types) + @pytest.mark.skipif(IS_GRAALPY, reason="Fails transiently on GraalPy especially with xdist") def test_HPyErr_WriteUnraisable(self, python_subprocess): mod = self.compile_module(""" HPyDef_METH(f, "f", HPyFunc_NOARGS) diff --git a/graalpython/hpy/test/test_hpyfield.py b/graalpython/hpy/test/test_hpyfield.py index 29ef99dba7..5e67515a13 100644 --- a/graalpython/hpy/test/test_hpyfield.py +++ b/graalpython/hpy/test/test_hpyfield.py @@ -145,7 +145,9 @@ def test_gc_track_no_gc_flag(self): assert not gc.is_tracked(p) def test_tp_traverse(self): - import sys + if self.is_graalpy: + import pytest + pytest.skip("Crashes on GraalPy") import gc mod = self.make_module(""" @DEFINE_PairObject @@ -308,6 +310,9 @@ def count_pairs(): assert count_pairs() == 0 def test_tp_finalize(self): + if self.is_graalpy: + import pytest + pytest.skip("Crashes on GraalPy") # Tests the contract of tp_finalize: what it should see # if called from within HPyField_Store mod = self.make_module(""" diff --git a/graalpython/hpy/test/test_hpytype.py b/graalpython/hpy/test/test_hpytype.py index 3b8469fd16..4d39fbcd0a 100644 --- a/graalpython/hpy/test/test_hpytype.py +++ b/graalpython/hpy/test/test_hpytype.py @@ -1594,6 +1594,9 @@ class TestPureHPyType(HPyTest): ExtensionTemplate = PointTemplate def test_builtin_shape(self): + if self.is_graalpy: + import pytest + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" @DEFINE_PointObject(HPyType_BuiltinShape_Long) @DEFINE_Point_xy diff --git a/graalpython/hpy/test/test_object.py b/graalpython/hpy/test/test_object.py index 8e50450457..369b1500ad 100644 --- a/graalpython/hpy/test/test_object.py +++ b/graalpython/hpy/test/test_object.py @@ -532,6 +532,8 @@ def test_getitem_s(self): def test_getslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) @@ -653,6 +655,8 @@ def test_setitem_s(self): def test_setslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs) @@ -786,6 +790,8 @@ def test_delitem(self): def test_delslice(self): import pytest + if self.is_graalpy: + pytest.skip("Not yet implemented on GraalPy") mod = self.make_module(""" HPyDef_METH(f, "f", HPyFunc_VARARGS) static HPy f_impl(HPyContext *ctx, HPy self, const HPy *args, size_t nargs)