Skip to content

Commit 342b93f

Browse files
markshannonmarkshannon
authored
bpo-46072: Add --with-pystats configure option to simplify gathering of VM stats (GH-30116)
* Simplify specialization stats collection macros. * Add --enable-pystats option to configure. * Update specialization summary script to handle larger number of kinds
1 parent 3a60bfe commit 342b93f

File tree

9 files changed

+74
-45
lines changed

9 files changed

+74
-45
lines changed

‎Include/internal/pycore_code.h

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -276,22 +276,11 @@ void _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
276276
SpecializedCacheEntry *cache);
277277
void _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr, SpecializedCacheEntry *cache);
278278

279-
#define PRINT_SPECIALIZATION_STATS 0
280-
#define PRINT_SPECIALIZATION_STATS_DETAILED 0
281-
#define PRINT_SPECIALIZATION_STATS_TO_FILE 0
282279

283-
#ifdef Py_DEBUG
284-
#define COLLECT_SPECIALIZATION_STATS 1
285-
#define COLLECT_SPECIALIZATION_STATS_DETAILED 1
286-
#else
287-
#define COLLECT_SPECIALIZATION_STATS PRINT_SPECIALIZATION_STATS
288-
#define COLLECT_SPECIALIZATION_STATS_DETAILED PRINT_SPECIALIZATION_STATS_DETAILED
289-
#endif
280+
#ifdef Py_STATS
290281

291282
#define SPECIALIZATION_FAILURE_KINDS 30
292283

293-
#if COLLECT_SPECIALIZATION_STATS
294-
295284
typedef struct _stats {
296285
uint64_t specialization_success;
297286
uint64_t specialization_failure;
@@ -300,15 +289,13 @@ typedef struct _stats {
300289
uint64_t miss;
301290
uint64_t deopt;
302291
uint64_t unquickened;
303-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
304292
uint64_t specialization_failure_kinds[SPECIALIZATION_FAILURE_KINDS];
305-
#endif
306293
} SpecializationStats;
307294

308295
extern SpecializationStats _specialization_stats[256];
309296
#define STAT_INC(opname, name) _specialization_stats[opname].name++
310297
#define STAT_DEC(opname, name) _specialization_stats[opname].name--
311-
void _Py_PrintSpecializationStats(void);
298+
void _Py_PrintSpecializationStats(int to_file);
312299

313300
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
314301

‎Misc/NEWS.d/next/Build/2021-12-15-10-37-44.bpo-46072.GgeAU3.rst

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Add a --with-pystats configure option to turn on internal statistics
2+
gathering.

‎Modules/_opcode.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ static PyObject *
8585
_opcode_get_specialization_stats_impl(PyObject *module)
8686
/*[clinic end generated code: output=fcbc32fdfbec5c17 input=e1f60db68d8ce5f6]*/
8787
{
88-
#if COLLECT_SPECIALIZATION_STATS
88+
#ifdef Py_STATS
8989
return _Py_GetSpecializationStats();
9090
#else
9191
Py_RETURN_NONE;

‎Python/ceval.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,8 +349,8 @@ PyEval_InitThreads(void)
349349
void
350350
_PyEval_Fini(void)
351351
{
352-
#if PRINT_SPECIALIZATION_STATS
353-
_Py_PrintSpecializationStats();
352+
#ifdef Py_STATS
353+
_Py_PrintSpecializationStats(1);
354354
#endif
355355
}
356356

‎Python/specialize.c

Lines changed: 22 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
*/
4040

4141
Py_ssize_t _Py_QuickenedCount = 0;
42-
#if COLLECT_SPECIALIZATION_STATS
42+
#ifdef Py_STATS
4343
SpecializationStats _specialization_stats[256] = { 0 };
4444

4545
#define ADD_STAT_TO_DICT(res, field) \
@@ -71,7 +71,6 @@ stats_to_dict(SpecializationStats *stats)
7171
ADD_STAT_TO_DICT(res, miss);
7272
ADD_STAT_TO_DICT(res, deopt);
7373
ADD_STAT_TO_DICT(res, unquickened);
74-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
7574
PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);
7675
if (failure_kinds == NULL) {
7776
Py_DECREF(res);
@@ -92,7 +91,6 @@ stats_to_dict(SpecializationStats *stats)
9291
return NULL;
9392
}
9493
Py_DECREF(failure_kinds);
95-
#endif
9694
return res;
9795
}
9896
#undef ADD_STAT_TO_DICT
@@ -113,7 +111,7 @@ add_stat_dict(
113111
return err;
114112
}
115113

116-
#if COLLECT_SPECIALIZATION_STATS
114+
#ifdef Py_STATS
117115
PyObject*
118116
_Py_GetSpecializationStats(void) {
119117
PyObject *stats = PyDict_New();
@@ -151,35 +149,34 @@ print_stats(FILE *out, SpecializationStats *stats, const char *name)
151149
PRINT_STAT(name, miss);
152150
PRINT_STAT(name, deopt);
153151
PRINT_STAT(name, unquickened);
154-
#if PRINT_SPECIALIZATION_STATS_DETAILED
155152
for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {
156153
fprintf(out, " %s.specialization_failure_kinds[%d] : %" PRIu64 "\n",
157154
name, i, stats->specialization_failure_kinds[i]);
158155
}
159-
#endif
160156
}
161157
#undef PRINT_STAT
162158

163159
void
164-
_Py_PrintSpecializationStats(void)
160+
_Py_PrintSpecializationStats(int to_file)
165161
{
166162
FILE *out = stderr;
167-
#if PRINT_SPECIALIZATION_STATS_TO_FILE
168-
/* Write to a file instead of stderr. */
163+
if (to_file) {
164+
/* Write to a file instead of stderr. */
169165
# ifdef MS_WINDOWS
170-
const char *dirname = "c:\\temp\\py_stats\\";
166+
const char *dirname = "c:\\temp\\py_stats\\";
171167
# else
172-
const char *dirname = "/tmp/py_stats/";
168+
const char *dirname = "/tmp/py_stats/";
173169
# endif
174-
char buf[48];
175-
sprintf(buf, "%s%u_%u.txt", dirname, (unsigned)clock(), (unsigned)rand());
176-
FILE *fout = fopen(buf, "w");
177-
if (fout) {
178-
out = fout;
179-
}
180-
#else
181-
fprintf(out, "Specialization stats:\n");
182-
#endif
170+
char buf[48];
171+
sprintf(buf, "%s%u_%u.txt", dirname, (unsigned)clock(), (unsigned)rand());
172+
FILE *fout = fopen(buf, "w");
173+
if (fout) {
174+
out = fout;
175+
}
176+
}
177+
else {
178+
fprintf(out, "Specialization stats:\n");
179+
}
183180
print_stats(out, &_specialization_stats[LOAD_ATTR], "load_attr");
184181
print_stats(out, &_specialization_stats[LOAD_GLOBAL], "load_global");
185182
print_stats(out, &_specialization_stats[LOAD_METHOD], "load_method");
@@ -194,7 +191,7 @@ _Py_PrintSpecializationStats(void)
194191
}
195192
}
196193

197-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
194+
#ifdef Py_STATS
198195

199196
#define SPECIALIZATION_FAIL(opcode, kind) _specialization_stats[opcode].specialization_failure_kinds[kind]++
200197

@@ -860,7 +857,7 @@ _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name, S
860857
}
861858

862859

863-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
860+
#ifdef Py_STATS
864861
static int
865862
load_method_fail_kind(DesciptorClassification kind)
866863
{
@@ -1086,7 +1083,7 @@ _Py_Specialize_LoadGlobal(
10861083
return 0;
10871084
}
10881085

1089-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
1086+
#ifdef Py_STATS
10901087
static int
10911088
binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
10921089
{
@@ -1380,7 +1377,7 @@ specialize_py_call(
13801377
return 0;
13811378
}
13821379

1383-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
1380+
#ifdef Py_STATS
13841381
static int
13851382
builtin_call_fail_kind(int ml_flags)
13861383
{
@@ -1459,7 +1456,7 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs,
14591456
}
14601457
}
14611458

1462-
#if COLLECT_SPECIALIZATION_STATS_DETAILED
1459+
#ifdef Py_STATS
14631460
static int
14641461
call_fail_kind(PyObject *callable)
14651462
{

‎Tools/scripts/summarize_specialization_stats.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ def print_stats(name, family_stats):
2424
for key in ("specialization_success", "specialization_failure"):
2525
print(f" {key}:{family_stats[key]:>12}")
2626
total_failures = family_stats["specialization_failure"]
27-
failure_kinds = [ 0 ] * 20
27+
failure_kinds = [ 0 ] * 30
2828
for key in family_stats:
2929
if not key.startswith("specialization_failure_kind"):
3030
continue

‎configure

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1006,6 +1006,7 @@ enable_shared
10061006
enable_profiling
10071007
with_pydebug
10081008
with_trace_refs
1009+
enable_pystats
10091010
with_assertions
10101011
enable_optimizations
10111012
with_lto
@@ -1713,6 +1714,7 @@ Optional Features:
17131714
no)
17141715
--enable-profiling enable C-level code profiling with gprof (default is
17151716
no)
1717+
--enable-pystats enable internal statistics gathering (default is no)
17161718
--enable-optimizations enable expensive, stable optimizations (PGO, etc.)
17171719
(default is no)
17181720
--enable-loadable-sqlite-extensions
@@ -6913,6 +6915,29 @@ then
69136915

69146916
$as_echo "#define Py_TRACE_REFS 1" >>confdefs.h
69156917

6918+
fi
6919+
6920+
6921+
# Check for --enable-pystats
6922+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for --enable-pystats" >&5
6923+
$as_echo_n "checking for --enable-pystats... " >&6; }
6924+
# Check whether --enable-pystats was given.
6925+
if test "${enable_pystats+set}" = set; then :
6926+
enableval=$enable_pystats;
6927+
else
6928+
enable_pystats=no
6929+
6930+
fi
6931+
6932+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $enable_pystats" >&5
6933+
$as_echo "$enable_pystats" >&6; }
6934+
6935+
if test "x$enable_pystats" = xyes; then :
6936+
6937+
6938+
$as_echo "#define Py_STATS 1" >>confdefs.h
6939+
6940+
69166941
fi
69176942

69186943
# Check for --with-assertions.

‎configure.ac

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1387,6 +1387,21 @@ then
13871387
AC_DEFINE(Py_TRACE_REFS, 1, [Define if you want to enable tracing references for debugging purpose])
13881388
fi
13891389

1390+
1391+
# Check for --enable-pystats
1392+
AC_MSG_CHECKING([for --enable-pystats])
1393+
AC_ARG_ENABLE([pystats],
1394+
[AS_HELP_STRING(
1395+
[--enable-pystats],
1396+
[enable internal statistics gathering (default is no)])],,
1397+
[enable_pystats=no]
1398+
)
1399+
AC_MSG_RESULT([$enable_pystats])
1400+
1401+
AS_VAR_IF([enable_pystats], [yes], [
1402+
AC_DEFINE([Py_STATS], [1], [Define if you want to enable internal statistics gathering.])
1403+
])
1404+
13901405
# Check for --with-assertions.
13911406
# This allows enabling assertions without Py_DEBUG.
13921407
assertions='false'
@@ -6323,7 +6338,7 @@ AC_DEFUN([PY_STDLIB_MOD], [
63236338
])
63246339

63256340
dnl Define simple stdlib extension module
6326-
dnl Always enable unless the module is listed in py_stdlib_not_available
6341+
dnl Always enable unless the module is listed in py_stdlib_not_available
63276342
dnl PY_STDLIB_MOD_SIMPLE([NAME], [CFLAGS], [LDFLAGS])
63286343
dnl cflags and ldflags are optional
63296344
AC_DEFUN([PY_STDLIB_MOD_SIMPLE], [

‎pyconfig.h.in

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,9 @@
14961496
SipHash13: 3, externally defined: 0 */
14971497
#undef Py_HASH_ALGORITHM
14981498

1499+
/* Define if you want to enable internal statistics gathering. */
1500+
#undef Py_STATS
1501+
14991502
/* Define if you want to enable tracing references for debugging purpose */
15001503
#undef Py_TRACE_REFS
15011504

0 commit comments

Comments
 (0)