Skip to content

Commit 5716cc3

Browse files
colesburycolesbury
authored
gh-100240: Use a consistent implementation for freelists (#121934)
This combines and updates our freelist handling to use a consistent implementation. Objects in the freelist are linked together using the first word of memory block. If configured with freelists disabled, these operations are essentially no-ops.
1 parent 2408a8a commit 5716cc3

27 files changed

+290
-700
lines changed

‎Include/internal/pycore_context.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
# error "this header requires Py_BUILD_CORE define"
66
#endif
77

8-
#include "pycore_freelist.h" // _PyFreeListState
98
#include "pycore_hamt.h" // PyHamtObject
109

1110

‎Include/internal/pycore_dict.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "pycore_freelist.h" // _PyFreeListState
1211
#include "pycore_object.h" // PyManagedDictPointer
1312
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_ACQUIRE
1413

‎Include/internal/pycore_floatobject.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "pycore_freelist.h" // _PyFreeListState
1211
#include "pycore_unicodeobject.h" // _PyUnicodeWriter
1312

1413
/* runtime lifecycle */

‎Include/internal/pycore_freelist.h

Lines changed: 92 additions & 127 deletions
Original file line numberDiff line numberDiff line change
@@ -8,144 +8,109 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
// PyTuple_MAXSAVESIZE - largest tuple to save on free list
12-
// PyTuple_MAXFREELIST - maximum number of tuples of each size to save
13-
14-
#ifdef WITH_FREELISTS
15-
// with freelists
16-
# define PyTuple_MAXSAVESIZE 20
17-
# define PyTuple_NFREELISTS PyTuple_MAXSAVESIZE
18-
# define PyTuple_MAXFREELIST 2000
19-
# define PyList_MAXFREELIST 80
20-
# define PyDict_MAXFREELIST 80
21-
# define PyFloat_MAXFREELIST 100
22-
# define PyContext_MAXFREELIST 255
23-
# define _PyAsyncGen_MAXFREELIST 80
24-
# define _PyObjectStackChunk_MAXFREELIST 4
25-
#else
26-
# define PyTuple_NFREELISTS 0
27-
# define PyTuple_MAXFREELIST 0
28-
# define PyList_MAXFREELIST 0
29-
# define PyDict_MAXFREELIST 0
30-
# define PyFloat_MAXFREELIST 0
31-
# define PyContext_MAXFREELIST 0
32-
# define _PyAsyncGen_MAXFREELIST 0
33-
# define _PyObjectStackChunk_MAXFREELIST 0
34-
#endif
35-
36-
struct _Py_list_freelist {
37-
#ifdef WITH_FREELISTS
38-
PyListObject *items[PyList_MAXFREELIST];
39-
int numfree;
11+
#include "pycore_freelist_state.h" // struct _Py_freelists
12+
#include "pycore_object.h" // _PyObject_IS_GC
13+
#include "pycore_pystate.h" // _PyThreadState_GET
14+
#include "pycore_code.h" // OBJECT_STAT_INC
15+
16+
static inline struct _Py_freelists *
17+
_Py_freelists_GET(void)
18+
{
19+
PyThreadState *tstate = _PyThreadState_GET();
20+
#ifdef Py_DEBUG
21+
_Py_EnsureTstateNotNULL(tstate);
4022
#endif
41-
};
42-
43-
struct _Py_tuple_freelist {
44-
#if WITH_FREELISTS
45-
/* There is one freelist for each size from 1 to PyTuple_MAXSAVESIZE.
46-
The empty tuple is handled separately.
4723

48-
Each tuple stored in the array is the head of the linked list
49-
(and the next available tuple) for that size. The actual tuple
50-
object is used as the linked list node, with its first item
51-
(ob_item[0]) pointing to the next node (i.e. the previous head).
52-
Each linked list is initially NULL. */
53-
PyTupleObject *items[PyTuple_NFREELISTS];
54-
int numfree[PyTuple_NFREELISTS];
24+
#ifdef Py_GIL_DISABLED
25+
return &((_PyThreadStateImpl*)tstate)->freelists;
5526
#else
56-
char _unused; // Empty structs are not allowed.
57-
#endif
58-
};
59-
60-
struct _Py_float_freelist {
61-
#ifdef WITH_FREELISTS
62-
/* Special free list
63-
free_list is a singly-linked list of available PyFloatObjects,
64-
linked via abuse of their ob_type members. */
65-
int numfree;
66-
PyFloatObject *items;
67-
#endif
68-
};
69-
70-
struct _Py_dict_freelist {
71-
#ifdef WITH_FREELISTS
72-
/* Dictionary reuse scheme to save calls to malloc and free */
73-
PyDictObject *items[PyDict_MAXFREELIST];
74-
int numfree;
27+
return &tstate->interp->object_state.freelists;
7528
#endif
76-
};
29+
}
7730

78-
struct _Py_dictkeys_freelist {
79-
#ifdef WITH_FREELISTS
80-
/* Dictionary keys reuse scheme to save calls to malloc and free */
81-
PyDictKeysObject *items[PyDict_MAXFREELIST];
82-
int numfree;
83-
#endif
84-
};
31+
#ifndef WITH_FREELISTS
32+
#define _Py_FREELIST_FREE(NAME, op, freefunc) freefunc(op)
33+
#define _Py_FREELIST_PUSH(NAME, op, limit) (0)
34+
#define _Py_FREELIST_POP(TYPE, NAME) (NULL)
35+
#define _Py_FREELIST_POP_MEM(NAME) (NULL)
36+
#define _Py_FREELIST_SIZE(NAME) (0)
37+
#else
38+
// Pushes `op` to the freelist, calls `freefunc` if the freelist is full
39+
#define _Py_FREELIST_FREE(NAME, op, freefunc) \
40+
_PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
41+
Py_ ## NAME ## _MAXFREELIST, freefunc)
42+
// Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
43+
#define _Py_FREELIST_PUSH(NAME, op, limit) \
44+
_PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
45+
46+
// Pops a PyObject from the freelist, returns NULL if the freelist is empty.
47+
#define _Py_FREELIST_POP(TYPE, NAME) \
48+
_Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
49+
50+
// Pops a non-PyObject data structure from the freelist, returns NULL if the
51+
// freelist is empty.
52+
#define _Py_FREELIST_POP_MEM(NAME) \
53+
_PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
54+
55+
#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
56+
57+
static inline int
58+
_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
59+
{
60+
if (fl->size < maxsize && fl->size >= 0) {
61+
*(void **)obj = fl->freelist;
62+
fl->freelist = obj;
63+
fl->size++;
64+
OBJECT_STAT_INC(to_freelist);
65+
return 1;
66+
}
67+
return 0;
68+
}
8569

86-
struct _Py_slice_freelist {
87-
#ifdef WITH_FREELISTS
88-
/* Using a cache is very effective since typically only a single slice is
89-
created and then deleted again. */
90-
PySliceObject *slice_cache;
91-
#endif
92-
};
70+
static inline void
71+
_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
72+
freefunc dofree)
73+
{
74+
if (!_PyFreeList_Push(fl, obj, maxsize)) {
75+
dofree(obj);
76+
}
77+
}
9378

94-
struct _Py_context_freelist {
95-
#ifdef WITH_FREELISTS
96-
// List of free PyContext objects
97-
PyContext *items;
98-
int numfree;
99-
#endif
100-
};
79+
static inline void *
80+
_PyFreeList_PopNoStats(struct _Py_freelist *fl)
81+
{
82+
void *obj = fl->freelist;
83+
if (obj != NULL) {
84+
assert(fl->size > 0);
85+
fl->freelist = *(void **)obj;
86+
fl->size--;
87+
}
88+
return obj;
89+
}
10190

102-
struct _Py_async_gen_freelist {
103-
#ifdef WITH_FREELISTS
104-
/* Freelists boost performance 6-10%; they also reduce memory
105-
fragmentation, as _PyAsyncGenWrappedValue and PyAsyncGenASend
106-
are short-living objects that are instantiated for every
107-
__anext__() call. */
108-
struct _PyAsyncGenWrappedValue* items[_PyAsyncGen_MAXFREELIST];
109-
int numfree;
110-
#endif
111-
};
91+
static inline PyObject *
92+
_PyFreeList_Pop(struct _Py_freelist *fl)
93+
{
94+
PyObject *op = _PyFreeList_PopNoStats(fl);
95+
if (op != NULL) {
96+
OBJECT_STAT_INC(from_freelist);
97+
_Py_NewReference(op);
98+
}
99+
return op;
100+
}
112101

113-
struct _Py_async_gen_asend_freelist {
114-
#ifdef WITH_FREELISTS
115-
struct PyAsyncGenASend* items[_PyAsyncGen_MAXFREELIST];
116-
int numfree;
102+
static inline void *
103+
_PyFreeList_PopMem(struct _Py_freelist *fl)
104+
{
105+
void *op = _PyFreeList_PopNoStats(fl);
106+
if (op != NULL) {
107+
OBJECT_STAT_INC(from_freelist);
108+
}
109+
return op;
110+
}
117111
#endif
118-
};
119-
120-
struct _PyObjectStackChunk;
121-
122-
struct _Py_object_stack_freelist {
123-
struct _PyObjectStackChunk *items;
124-
Py_ssize_t numfree;
125-
};
126-
127-
struct _Py_object_freelists {
128-
struct _Py_float_freelist floats;
129-
struct _Py_tuple_freelist tuples;
130-
struct _Py_list_freelist lists;
131-
struct _Py_dict_freelist dicts;
132-
struct _Py_dictkeys_freelist dictkeys;
133-
struct _Py_slice_freelist slices;
134-
struct _Py_context_freelist contexts;
135-
struct _Py_async_gen_freelist async_gens;
136-
struct _Py_async_gen_asend_freelist async_gen_asends;
137-
struct _Py_object_stack_freelist object_stacks;
138-
};
139112

140-
extern void _PyObject_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
141-
extern void _PyTuple_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
142-
extern void _PyFloat_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
143-
extern void _PyList_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
144-
extern void _PySlice_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
145-
extern void _PyDict_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
146-
extern void _PyAsyncGen_ClearFreeLists(struct _Py_object_freelists *freelists, int is_finalization);
147-
extern void _PyContext_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
148-
extern void _PyObjectStackChunk_ClearFreeList(struct _Py_object_freelists *freelists, int is_finalization);
113+
extern void _PyObject_ClearFreeLists(struct _Py_freelists *freelists, int is_finalization);
149114

150115
#ifdef __cplusplus
151116
}

‎Include/internal/pycore_freelist_state.h

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#ifndef Py_INTERNAL_FREELIST_STATE_H
2+
#define Py_INTERNAL_FREELIST_STATE_H
3+
#ifdef __cplusplus
4+
extern "C" {
5+
#endif
6+
7+
#ifndef Py_BUILD_CORE
8+
# error "this header requires Py_BUILD_CORE define"
9+
#endif
10+
11+
#ifdef WITH_FREELISTS
12+
// with freelists
13+
# define PyTuple_MAXSAVESIZE 20 // Largest tuple to save on freelist
14+
# define Py_tuple_MAXFREELIST 2000 // Maximum number of tuples of each size to save
15+
# define Py_lists_MAXFREELIST 80
16+
# define Py_dicts_MAXFREELIST 80
17+
# define Py_dictkeys_MAXFREELIST 80
18+
# define Py_floats_MAXFREELIST 100
19+
# define Py_slices_MAXFREELIST 1
20+
# define Py_contexts_MAXFREELIST 255
21+
# define Py_async_gens_MAXFREELIST 80
22+
# define Py_async_gen_asends_MAXFREELIST 80
23+
# define Py_object_stack_chunks_MAXFREELIST 4
24+
#else
25+
# define PyTuple_MAXSAVESIZE 0
26+
#endif
27+
28+
// A generic freelist of either PyObjects or other data structures.
29+
struct _Py_freelist {
30+
// Entries are linked together using the first word of the the object.
31+
// For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
32+
// field.
33+
void *freelist;
34+
35+
// The number of items in the free list or -1 if the free list is disabled
36+
Py_ssize_t size;
37+
};
38+
39+
struct _Py_freelists {
40+
#ifdef WITH_FREELISTS
41+
struct _Py_freelist floats;
42+
struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
43+
struct _Py_freelist lists;
44+
struct _Py_freelist dicts;
45+
struct _Py_freelist dictkeys;
46+
struct _Py_freelist slices;
47+
struct _Py_freelist contexts;
48+
struct _Py_freelist async_gens;
49+
struct _Py_freelist async_gen_asends;
50+
struct _Py_freelist object_stack_chunks;
51+
#else
52+
char _unused; // Empty structs are not allowed.
53+
#endif
54+
};
55+
56+
#ifdef __cplusplus
57+
}
58+
#endif
59+
#endif /* !Py_INTERNAL_FREELIST_STATE_H */

‎Include/internal/pycore_gc.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "pycore_freelist.h" // _PyFreeListState
12-
1311
/* GC information is stored BEFORE the object structure. */
1412
typedef struct {
1513
// Pointer to next object in the list.

‎Include/internal/pycore_list.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "pycore_freelist.h" // _PyFreeListState
12-
1311
PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *);
1412
extern void _PyList_DebugMallocStats(FILE *out);
1513

‎Include/internal/pycore_object_stack.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
#ifndef Py_INTERNAL_OBJECT_STACK_H
22
#define Py_INTERNAL_OBJECT_STACK_H
33

4-
#include "pycore_freelist.h" // _PyFreeListState
5-
64
#ifdef __cplusplus
75
extern "C" {
86
#endif

‎Include/internal/pycore_object_state.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@ extern "C" {
88
# error "this header requires Py_BUILD_CORE define"
99
#endif
1010

11-
#include "pycore_freelist.h" // _PyObject_freelists
12-
#include "pycore_hashtable.h" // _Py_hashtable_t
11+
#include "pycore_freelist_state.h" // _Py_freelists
12+
#include "pycore_hashtable.h" // _Py_hashtable_t
1313

1414
struct _py_object_runtime_state {
1515
#ifdef Py_REF_DEBUG
@@ -20,7 +20,7 @@ struct _py_object_runtime_state {
2020

2121
struct _py_object_state {
2222
#if !defined(Py_GIL_DISABLED)
23-
struct _Py_object_freelists freelists;
23+
struct _Py_freelists freelists;
2424
#endif
2525
#ifdef Py_REF_DEBUG
2626
Py_ssize_t reftotal;

0 commit comments

Comments
 (0)