Skip to content

GH-93841: Turn stats on and off, clear and dump them at runtime.#93843

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
markshannon merged 8 commits into from
Jun 21, 2022
Merged

GH-93841: Turn stats on and off, clear and dump them at runtime. #93843

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4b142e6
Allow stats recording to be turned on or off. WIP.
markshannon Jun 14, 2022
daa5487
Allow stats to be turned on and off, and be cleared.
markshannon Jun 15, 2022
6bc5fe2
Turn on stats
markshannon Feb 2, 2022
d6ce8e6
Fixups and add dump function.
markshannon Jun 15, 2022
cc0f61e
Turn stats back off.
markshannon Jun 15, 2022
e301cda
Add news
markshannon Jun 15, 2022
4409aff
Tidy up doc strings.
markshannon Jun 15, 2022
ca65b2e
Merge branch 'main' into on-off-stats
markshannon Jun 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions Include/internal/pycore_code.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -259,16 +259,16 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co);
#ifdef Py_STATS


#define STAT_INC(opname, name) _py_stats.opcode_stats[(opname)].specialization.name++
#define STAT_DEC(opname, name) _py_stats.opcode_stats[(opname)].specialization.name--
#define OPCODE_EXE_INC(opname) _py_stats.opcode_stats[(opname)].execution_count++
#define CALL_STAT_INC(name) _py_stats.call_stats.name++
#define OBJECT_STAT_INC(name) _py_stats.object_stats.name++
#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
#define OBJECT_STAT_INC_COND(name, cond) \
do { if (cond) _py_stats.object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) _py_stats.call_stats.eval_calls[(name)]++
do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
do { if (PyFunction_Check(callable)) _py_stats.call_stats.eval_calls[(name)]++; } while (0)
do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)

// Used by the _opcode extension which is built as a shared library
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
Expand Down
13 changes: 8 additions & 5 deletions Include/pystats.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -73,19 +73,22 @@ typedef struct _stats {
ObjectStats object_stats;
} PyStats;

PyAPI_DATA(PyStats) _py_stats;

PyAPI_DATA(PyStats) _py_stats_struct;
PyAPI_DATA(PyStats *) _py_stats;

extern void _Py_StatsClear(void);
extern void _Py_PrintSpecializationStats(int to_file);

#ifdef _PY_INTERPRETER

#define _Py_INCREF_STAT_INC() _py_stats.object_stats.interpreter_increfs++
#define _Py_DECREF_STAT_INC() _py_stats.object_stats.interpreter_decrefs++
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0)

#else

#define _Py_INCREF_STAT_INC() _py_stats.object_stats.increfs++
#define _Py_DECREF_STAT_INC() _py_stats.object_stats.decrefs++
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0)

#endif

Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2022-06-15-11-16-13.gh-issue-93841.06zqX3.rst
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
When built with ``-enable-pystats``, ``sys._stats_on()``,
``sys._stats_off()``, ``sys._stats_clear()`` and ``sys._stats_dump()``
functions have been added to enable gathering stats for parts of programs.
6 changes: 3 additions & 3 deletions Python/ceval.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1310,7 +1310,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
do { \
frame->prev_instr = next_instr++; \
OPCODE_EXE_INC(op); \
_py_stats.opcode_stats[lastopcode].pair_count[op]++; \
if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \
lastopcode = op; \
} while (0)
#else
Expand DownExpand Up@@ -7790,7 +7790,7 @@ _Py_GetDXProfile(PyObject *self, PyObject *args)
PyObject *l = PyList_New(257);
if (l == NULL) return NULL;
for (i = 0; i < 256; i++) {
PyObject *x = getarray(_py_stats.opcode_stats[i].pair_count);
PyObject *x = getarray(_py_stats_struct.opcode_stats[i].pair_count);
if (x == NULL) {
Py_DECREF(l);
return NULL;
Expand All@@ -7804,7 +7804,7 @@ _Py_GetDXProfile(PyObject *self, PyObject *args)
}
for (i = 0; i < 256; i++) {
PyObject *x = PyLong_FromUnsignedLongLong(
_py_stats.opcode_stats[i].execution_count);
_py_stats_struct.opcode_stats[i].execution_count);
if (x == NULL) {
Py_DECREF(counts);
Py_DECREF(l);
Expand Down
106 changes: 105 additions & 1 deletion Python/clinic/sysmodule.c.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

24 changes: 19 additions & 5 deletions Python/specialize.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -33,7 +33,8 @@ uint8_t _PyOpcode_Adaptive[256] = {

Py_ssize_t _Py_QuickenedCount = 0;
#ifdef Py_STATS
PyStats _py_stats = { 0 };
PyStats _py_stats_struct = { 0 };
PyStats *_py_stats = &_py_stats_struct;

#define ADD_STAT_TO_DICT(res, field) \
do { \
Expand DownExpand Up@@ -93,7 +94,7 @@ add_stat_dict(
int opcode,
const char *name) {

SpecializationStats *stats = &_py_stats.opcode_stats[opcode].specialization;
SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;
PyObject *d = stats_to_dict(stats);
if (d == NULL) {
return -1;
Expand DownExpand Up@@ -209,9 +210,18 @@ print_stats(FILE *out, PyStats *stats) {
print_object_stats(out, &stats->object_stats);
}

void
_Py_StatsClear(void)
{
_py_stats_struct = (PyStats) { 0 };
}

void
_Py_PrintSpecializationStats(int to_file)
{
if (_py_stats == NULL) {
return;
}
FILE *out = stderr;
if (to_file) {
/* Write to a file instead of stderr. */
Expand DownExpand Up@@ -242,16 +252,20 @@ _Py_PrintSpecializationStats(int to_file)
else {
fprintf(out, "Specialization stats:\n");
}
print_stats(out, &_py_stats);
print_stats(out, _py_stats);
if (out != stderr) {
fclose(out);
}
}

#ifdef Py_STATS

#define SPECIALIZATION_FAIL(opcode, kind) _py_stats.opcode_stats[opcode].specialization.failure_kinds[kind]++

#define SPECIALIZATION_FAIL(opcode, kind) \
do { \
if (_py_stats) { \
_py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
} \
} while (0)

#endif
#endif
Expand Down
66 changes: 66 additions & 0 deletions Python/sysmodule.c
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -1909,6 +1909,66 @@ sys_is_finalizing_impl(PyObject *module)
return PyBool_FromLong(_Py_IsFinalizing());
}

#ifdef Py_STATS
/*[clinic input]
sys._stats_on

Turns on stats gathering (stats gathering is on by default).
[clinic start generated code]*/

static PyObject *
sys__stats_on_impl(PyObject *module)
/*[clinic end generated code: output=aca53eafcbb4d9fe input=8ddc6df94e484f3a]*/
{
_py_stats = &_py_stats_struct;
Py_RETURN_NONE;
}

/*[clinic input]
sys._stats_off

Turns off stats gathering (stats gathering is on by default).
[clinic start generated code]*/

static PyObject *
sys__stats_off_impl(PyObject *module)
/*[clinic end generated code: output=1534c1ee63812214 input=b3e50e71ecf29f66]*/
{
_py_stats = NULL;

@arhadthedevarhadthedevJun 21, 2022

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use a hidden, dummy brother of _py_stats_struct here instead of NULL? It'll allow to omit if (_py_stats) checks.

@markshannonmarkshannonJun 21, 2022

Copy link
Copy Markdown
MemberAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did consider it, but the stats object is about 600k and is likely to get larger. So I think the extra NULL checks are less inefficient than wasting the memory.

Py_RETURN_NONE;
}

/*[clinic input]
sys._stats_clear

Clears the stats.
[clinic start generated code]*/

static PyObject *
sys__stats_clear_impl(PyObject *module)
/*[clinic end generated code: output=fb65a2525ee50604 input=3e03f2654f44da96]*/
{
_Py_StatsClear();
Py_RETURN_NONE;
}

/*[clinic input]
sys._stats_dump

Dump stats to file, and clears the stats.
[clinic start generated code]*/

static PyObject *
sys__stats_dump_impl(PyObject *module)
/*[clinic end generated code: output=79f796fb2b4ddf05 input=92346f16d64f6f95]*/
{
_Py_PrintSpecializationStats(1);
_Py_StatsClear();
Py_RETURN_NONE;
}

#endif

#ifdef ANDROID_API_LEVEL
/*[clinic input]
sys.getandroidapilevel
Expand DownExpand Up@@ -1978,6 +2038,12 @@ static PyMethodDef sys_methods[] = {
SYS_GET_ASYNCGEN_HOOKS_METHODDEF
SYS_GETANDROIDAPILEVEL_METHODDEF
SYS_UNRAISABLEHOOK_METHODDEF
#ifdef Py_STATS
SYS__STATS_ON_METHODDEF
SYS__STATS_OFF_METHODDEF
SYS__STATS_CLEAR_METHODDEF
SYS__STATS_DUMP_METHODDEF
#endif
{NULL, NULL} // sentinel
};

Expand Down