Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
77dd9fa
feat(profiling): python 3.14 support
taegyunkim Dec 4, 2025
c42034b
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 7, 2025
be86eea
check existence of header to decide rebuild
taegyunkim Dec 7, 2025
52e00ba
remove protobuf==4.22.0 variant for 3.14
taegyunkim Dec 8, 2025
4391872
uwsgi<2.0.30 is not compatible with python 3.14
taegyunkim Dec 8, 2025
8690dbc
update serverless import test
taegyunkim Dec 8, 2025
28a956f
fix internal telemetry test
taegyunkim Dec 8, 2025
ed01ba7
update test
taegyunkim Dec 8, 2025
a97c618
simplify code a bit
taegyunkim Dec 8, 2025
f86b3eb
reduce diff - these are not necessary
taegyunkim Dec 8, 2025
c9117cf
collapse idential branches
taegyunkim Dec 8, 2025
b03e5b4
remove redundant code
taegyunkim Dec 8, 2025
f9676e4
use nullptr instead of NULL
taegyunkim Dec 8, 2025
f9ff0eb
split into multiple lines for readability
taegyunkim Dec 8, 2025
514bfe9
reduce diff
taegyunkim Dec 8, 2025
95be056
remove 3.14 special handling which is actually not needed
taegyunkim Dec 8, 2025
434ab99
reduce diff
taegyunkim Dec 8, 2025
17f3f89
remove code thats not doing anything
taegyunkim Dec 8, 2025
a6b070f
reduce diff
taegyunkim Dec 8, 2025
3d01189
tidy-up includes following include what you use principle
taegyunkim Dec 8, 2025
88a8e36
udpate comment and include
taegyunkim Dec 8, 2025
ab58a60
clear LSB to get the PyCodeObject
taegyunkim Dec 8, 2025
52f8a9b
ignore frame owned by interpreter
taegyunkim Dec 8, 2025
1ab4609
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 8, 2025
4b92ca2
cosmetic change
taegyunkim Dec 8, 2025
d163dfc
update comments and checks for PyGen_yf
taegyunkim Dec 8, 2025
7104cb1
simplify code by not passing around uintptr tstate_addr
taegyunkim Dec 8, 2025
d1a10ef
clean up some code and comments
taegyunkim Dec 8, 2025
d765720
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 9, 2025
ea99d06
update test_asyncio_as_completed
taegyunkim Dec 9, 2025
3c17bd2
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 9, 2025
bc24c6a
update lineno
taegyunkim Dec 9, 2025
49444d7
update to 3
taegyunkim Dec 9, 2025
cc21de7
Update comment
taegyunkim Dec 9, 2025
afae620
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 10, 2025
7104d62
remove unnecessary comments
taegyunkim Dec 10, 2025
7f3be8a
use 1<<16
taegyunkim Dec 10, 2025
1fa3d96
adopt review suggestions
taegyunkim Dec 10, 2025
a2971f7
use return<void>
taegyunkim Dec 10, 2025
9f762f3
revert lines
taegyunkim Dec 10, 2025
6135053
avoid copying over large struct, _PyThreadStateImpl, and copy only ne…
taegyunkim Dec 10, 2025
8429a24
Would this work with the previous commit?
taegyunkim Dec 10, 2025
6c7975d
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 11, 2025
1cf23bb
somehow this is now fixed?
taegyunkim Dec 11, 2025
e61b878
simplify includes
taegyunkim Dec 11, 2025
a14a17c
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 11, 2025
9eadc94
revert unnecessary changes
taegyunkim Dec 11, 2025
76eabaa
remove unnecessary comments
taegyunkim Dec 11, 2025
c49290f
keep the comment
taegyunkim Dec 11, 2025
870e787
Update riotfile.py
taegyunkim Dec 11, 2025
67590b4
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 11, 2025
c99ec52
Merge branch 'main' into taegyunkim/prof-12435-py314
taegyunkim Dec 11, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions .riot/requirements/173f5b3.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
#
# This file is autogenerated by pip-compile with Python 3.14
# by the following command:
#
# pip-compile --allow-unsafe --no-annotate .riot/requirements/173f5b3.in
#
attrs==25.4.0
coverage[toml]==7.12.0
gevent==25.9.1
greenlet==3.3.0
gunicorn[gevent]==23.0.0
hypothesis==6.45.0
iniconfig==2.3.0
jsonschema==4.25.1
jsonschema-specifications==2025.9.1
mock==5.2.0
opentracing==2.4.0
packaging==25.0
pluggy==1.6.0
protobuf==6.33.1
py-cpuinfo==8.0.0
pygments==2.19.2
pytest==9.0.1
pytest-asyncio==0.21.1
pytest-benchmark==5.2.3
pytest-cov==7.0.0
pytest-cpp==2.6.0
pytest-mock==3.15.1
pytest-randomly==4.0.1
referencing==0.37.0
rpds-py==0.30.0
sortedcontainers==2.4.0
uwsgi==2.0.31
zope-event==6.1
zope-interface==8.1.1
zstandard==0.25.0
31 changes: 31 additions & 0 deletions .riot/requirements/1a4c947.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#
# This file is autogenerated by pip-compile with Python 3.14
# by the following command:
#
# pip-compile --allow-unsafe --no-annotate .riot/requirements/1a4c947.in
#
attrs==25.4.0
coverage[toml]==7.12.0
gunicorn==23.0.0
hypothesis==6.45.0
iniconfig==2.3.0
jsonschema==4.25.1
jsonschema-specifications==2025.9.1
mock==5.2.0
opentracing==2.4.0
packaging==25.0
pluggy==1.6.0
protobuf==6.33.1
py-cpuinfo==8.0.0
pygments==2.19.2
pytest==9.0.1
pytest-asyncio==0.21.1
pytest-benchmark==5.2.3
pytest-cov==7.0.0
pytest-cpp==2.6.0
pytest-mock==3.15.1
pytest-randomly==4.0.1
referencing==0.37.0
rpds-py==0.30.0
sortedcontainers==2.4.0
zstandard==0.25.0
32 changes: 32 additions & 0 deletions .riot/requirements/72ed1ec.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#
# This file is autogenerated by pip-compile with Python 3.14
# by the following command:
#
# pip-compile --allow-unsafe --no-annotate .riot/requirements/72ed1ec.in
#
attrs==25.4.0
coverage[toml]==7.12.0
gunicorn==23.0.0
hypothesis==6.45.0
iniconfig==2.3.0
jsonschema==4.25.1
jsonschema-specifications==2025.9.1
mock==5.2.0
opentracing==2.4.0
packaging==25.0
pluggy==1.6.0
protobuf==6.33.1
py-cpuinfo==8.0.0
pygments==2.19.2
pytest==9.0.1
pytest-asyncio==0.21.1
pytest-benchmark==5.2.3
pytest-cov==7.0.0
pytest-cpp==2.6.0
pytest-mock==3.15.1
pytest-randomly==4.0.1
referencing==0.37.0
rpds-py==0.30.0
sortedcontainers==2.4.0
uwsgi==2.0.31
zstandard==0.25.0
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,18 @@
#include <cpython/genobject.h>

#define Py_BUILD_CORE
#if PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030e0000
#include <cstddef>
#include <internal/pycore_frame.h>
#include <internal/pycore_interpframe.h>
#include <internal/pycore_interpframe_structs.h>
#include <internal/pycore_llist.h>
#include <internal/pycore_stackref.h>
#include <opcode.h>
#elif PY_VERSION_HEX >= 0x030d0000
#include <opcode.h>
#else
#include <cstddef>
#include <internal/pycore_frame.h>
#include <internal/pycore_opcode.h>
#endif // PY_VERSION_HEX >= 0x030d0000
Expand All @@ -38,7 +47,32 @@ extern "C"
STATE_FINISHED
} fut_state;

#if PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030e0000
// Python 3.14+: New fields added (awaited_by, is_task, awaited_by_is_set)
#define FutureObj_HEAD(prefix) \
PyObject_HEAD PyObject* prefix##_loop; \
PyObject* prefix##_callback0; \
PyObject* prefix##_context0; \
PyObject* prefix##_callbacks; \
PyObject* prefix##_exception; \
PyObject* prefix##_exception_tb; \
PyObject* prefix##_result; \
PyObject* prefix##_source_tb; \
PyObject* prefix##_cancel_msg; \
PyObject* prefix##_cancelled_exc; \
PyObject* prefix##_awaited_by; \
fut_state prefix##_state; \
/* Used by profilers to make traversing the stack from an external \
process faster. */ \
char prefix##_is_task; \
char prefix##_awaited_by_is_set; \
/* These bitfields need to be at the end of the struct \
so that these and bitfields from TaskObj are contiguous. \
*/ \
unsigned prefix##_log_tb : 1; \
unsigned prefix##_blocking : 1;

#elif PY_VERSION_HEX >= 0x030d0000
#define FutureObj_HEAD(prefix) \
PyObject_HEAD PyObject* prefix##_loop; \
PyObject* prefix##_callback0; \
Expand Down Expand Up @@ -131,7 +165,24 @@ extern "C"
FutureObj_HEAD(future)
} FutureObj;

#if PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030e0000
// Python 3.14+: TaskObj includes task_node for linked-list storage
typedef struct
{
FutureObj_HEAD(task) unsigned task_must_cancel : 1;
unsigned task_log_destroy_pending : 1;
int task_num_cancels_requested;
PyObject* task_fut_waiter;
PyObject* task_coro;
PyObject* task_name;
PyObject* task_context;
struct llist_node task_node;
#ifdef Py_GIL_DISABLED
// thread id of the thread where this task was created
uintptr_t task_tid;
#endif
} TaskObj;
#elif PY_VERSION_HEX >= 0x030d0000
typedef struct
{
FutureObj_HEAD(task) unsigned task_must_cancel : 1;
Expand Down Expand Up @@ -173,7 +224,67 @@ extern "C"
#define RESUME_QUICK INSTRUMENTED_RESUME
#endif

#if PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030e0000
// Python 3.14+: Use stackpointer and _PyStackRef

inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr)
{
if (gen->gi_frame_state != FRAME_SUSPENDED_YIELD_FROM) {
return nullptr;
}

_PyInterpreterFrame frame;
if (copy_type(frame_addr, frame)) {
return nullptr;
}

// CPython asserts the following:
// assert(f->stackpointer > f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus);
// assert(!PyStackRef_IsNull(f->stackpointer[-1]));

// Though we have to pay the price of copying the code object, we need
// to do this to catch the case where the stack is empty, as accessing
// frame.stackpointer[-1] would be an undefined behavior.
// This is necessary as frame.stacktop is removed in 3.14.
PyCodeObject code;
auto code_addr = reinterpret_cast<PyCodeObject*>(BITS_TO_PTR_MASKED(frame.f_executable));
if (copy_type(code_addr, code)) {
return nullptr;
}

uintptr_t frame_addr_uint = reinterpret_cast<uintptr_t>(frame_addr);
uintptr_t localsplus_addr = frame_addr_uint + offsetof(_PyInterpreterFrame, localsplus);
// This computes f->localsplus + code.co_nlocalsplus.
uintptr_t stackbase_addr = localsplus_addr + code.co_nlocalsplus * sizeof(_PyStackRef);

uintptr_t stackpointer_addr = reinterpret_cast<uintptr_t>(frame.stackpointer);
// We want stackpointer_addr to be greater than the stackbase_addr,
// that is, the stack is not empty.
if (stackpointer_addr <= stackbase_addr) {
return nullptr;
}

// We can also calculate stacktop and check that it is within a reasonable range.
// Similar to 3.13's stacktop check below.
int stacktop = (int)((stackpointer_addr - stackbase_addr) / sizeof(_PyStackRef));

if (stacktop < 1 || stacktop > MAX_STACK_SIZE) {
return nullptr;
}

// Read the top of stack directly from remote memory
// This is equivalent to CPython's frame.stackpointer[-1].
_PyStackRef top_ref;
if (copy_type(reinterpret_cast<void*>(stackpointer_addr - sizeof(_PyStackRef)), top_ref)) {
return nullptr;
}

// Extract PyObject* from _PyStackRef.bits
// Per Python 3.14 release notes (gh-123923): clear LSB to recover PyObject* pointer
return BITS_TO_PTR_MASKED(top_ref);
}

#elif PY_VERSION_HEX >= 0x030d0000

inline PyObject* PyGen_yf(PyGenObject* gen, PyObject* frame_addr)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@
#undef _PyGC_FINALIZED
#endif
#include <frameobject.h>
#if PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030e0000
#define Py_BUILD_CORE
#include <internal/pycore_code.h>
#endif // PY_VERSION_HEX >= 0x030d0000
#if PY_VERSION_HEX >= 0x030b0000
#include <internal/pycore_interpframe_structs.h>
#elif PY_VERSION_HEX >= 0x030b0000
#define Py_BUILD_CORE
#include <internal/pycore_frame.h>
#endif
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
#include <Python.h>
#define Py_BUILD_CORE

#if PY_VERSION_HEX >= 0x030e0000
#include <internal/pycore_frame.h>
#endif

#include <echion/stacks.h>
#include <echion/strings.h>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
#endif
#define Py_BUILD_CORE
#include <internal/pycore_pystate.h>
#if PY_VERSION_HEX >= 0x030e0000
#include <internal/pycore_runtime.h>
#endif

#include <thread>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,22 @@

#pragma once

#include <optional>

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <weakrefobject.h>
#include <frameobject.h>

#if PY_VERSION_HEX >= 0x030b0000
#include <cpython/genobject.h>

#define Py_BUILD_CORE
#if PY_VERSION_HEX >= 0x030d0000
#include <cstddef>
#if PY_VERSION_HEX >= 0x030e0000
#include <internal/pycore_frame.h>
#include <opcode.h>
#elif PY_VERSION_HEX >= 0x030d0000
#include <opcode.h>
#else
#include <internal/pycore_frame.h>
#include <internal/pycore_opcode.h>
#endif // PY_VERSION_HEX >= 0x030d0000
#else
Expand Down Expand Up @@ -275,67 +278,6 @@ TaskInfo::current(PyObject* loop)
return TaskInfo::create(reinterpret_cast<TaskObj*>(task));
}

// ----------------------------------------------------------------------------
// TODO: Make this a "for_each_task" function?
[[nodiscard]] inline Result<std::vector<TaskInfo::Ptr>>
get_all_tasks(PyObject* loop)
{
std::vector<TaskInfo::Ptr> tasks;
if (loop == NULL)
return tasks;

auto maybe_scheduled_tasks_set = MirrorSet::create(asyncio_scheduled_tasks);
if (!maybe_scheduled_tasks_set) {
return ErrorKind::TaskInfoError;
}

auto scheduled_tasks_set = std::move(*maybe_scheduled_tasks_set);
auto maybe_scheduled_tasks = scheduled_tasks_set.as_unordered_set();
if (!maybe_scheduled_tasks) {
return ErrorKind::TaskInfoError;
}

auto scheduled_tasks = std::move(*maybe_scheduled_tasks);
for (auto task_wr_addr : scheduled_tasks) {
PyWeakReference task_wr;
if (copy_type(task_wr_addr, task_wr))
continue;

auto maybe_task_info = TaskInfo::create(reinterpret_cast<TaskObj*>(task_wr.wr_object));
if (maybe_task_info) {
if ((*maybe_task_info)->loop == loop) {
tasks.push_back(std::move(*maybe_task_info));
}
}
}

if (asyncio_eager_tasks != NULL) {
auto maybe_eager_tasks_set = MirrorSet::create(asyncio_eager_tasks);
if (!maybe_eager_tasks_set) {
return ErrorKind::TaskInfoError;
}

auto eager_tasks_set = std::move(*maybe_eager_tasks_set);

auto maybe_eager_tasks = eager_tasks_set.as_unordered_set();
if (!maybe_eager_tasks) {
return ErrorKind::TaskInfoError;
}

auto eager_tasks = std::move(*maybe_eager_tasks);
for (auto task_addr : eager_tasks) {
auto maybe_task_info = TaskInfo::create(reinterpret_cast<TaskObj*>(task_addr));
if (maybe_task_info) {
if ((*maybe_task_info)->loop == loop) {
tasks.push_back(std::move(*maybe_task_info));
}
}
}
}

return tasks;
}

// ----------------------------------------------------------------------------

inline std::vector<std::unique_ptr<StackInfo>> current_tasks;
Expand Down
Loading
Loading