前言
最近在看 python GC 这块，主要参考了武老师的是视频和博客，自己再总结一下。
我的 python 源码版本 3.9.0。
知识点
python GC 主要分为引用计数和分带的标记清除两种 GC。
- 引用计数会一直占用系统资源，需要持续监控对象的引用
- 标记清除有 STW（stop the world）
两个基础的数据结构
python 所有的内存对象都保存在一个双向循环链表中。
Include/object.h:
/* Define pointers to support a doubly-linked list of all live heap objects. */
#define _PyObject_HEAD_EXTRA \
struct _object *_ob_next; \
struct _object *_ob_prev;
/* PyObject_HEAD defines the initial segment of every PyObject. */
#define PyObject_HEAD PyObject ob_base;
typedef struct _object {
_PyObject_HEAD_EXTRA // 双向循环链表
Py_ssize_t ob_refcnt; // 引用计数的个数
PyTypeObject *ob_type;
} PyObject;
typedef struct {
PyObject ob_base;
Py_ssize_t ob_size; /* Number of items in variable part */
} PyVarObject;
list 类型创建，引用，销毁
创建
Objects/listobject.c:
PyObject *
PyList_New(Py_ssize_t size)
{
PyListObject *op;
if (size < 0) {
PyErr_BadInternalCall();
return NULL;
}
if (numfree) { // 看 free_list 中是否有对象
numfree--;
op = free_list[numfree];
_Py_NewReference((PyObject *)op);
} else { // 缓存中没有需要新开辟内存
op = PyObject_GC_New(PyListObject, &PyList_Type);
if (op == NULL)
return NULL;
}
if (size <= 0)
op->ob_item = NULL;
else {
op->ob_item = (PyObject **) PyMem_Calloc(size, sizeof(PyObject *));
if (op->ob_item == NULL) {
Py_DECREF(op);
return PyErr_NoMemory();
}
}
Py_SET_SIZE(op, size);
op->allocated = size;
_PyObject_GC_TRACK(op); // 将对象放到0代链表中
return (PyObject *) op;
}
Include/objimpl.h:
#define PyObject_GC_New(type, typeobj) \
( (type *) _PyObject_GC_New(typeobj) )
Modules/gcmodule.c:
PyObject *
_PyObject_GC_New(PyTypeObject *tp)
{
PyObject *op = _PyObject_GC_Malloc(_PyObject_SIZE(tp)); // 为对象分配内存
if (op != NULL)
op = PyObject_INIT(op, tp); // 初始化对象，并把对象加入 refchain 中
return op;
}
PyObject *
_PyObject_GC_Malloc(size_t basicsize)
{
return _PyObject_GC_Alloc(0, basicsize);
}
static PyObject *
_PyObject_GC_Alloc(int use_calloc, size_t basicsize)
{
PyThreadState *tstate = _PyThreadState_GET();
GCState *gcstate = &tstate->interp->gc;
if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) {
return _PyErr_NoMemory(tstate);
}
size_t size = sizeof(PyGC_Head) + basicsize;
PyGC_Head *g;
if (use_calloc) {
g = (PyGC_Head *)PyObject_Calloc(1, size);
}
else {
g = (PyGC_Head *)PyObject_Malloc(size);
}
if (g == NULL) {
return _PyErr_NoMemory(tstate);
}
assert(((uintptr_t)g & 3) == 0); // g must be aligned 4bytes boundary
g->_gc_next = 0;
g->_gc_prev = 0;
gcstate->generations[0].count++; /* number of allocated GC objects */ // 分代回收0代+1
if (gcstate->generations[0].count > gcstate->generations[0].threshold &&
gcstate->enabled &&
gcstate->generations[0].threshold &&
!gcstate->collecting &&
!_PyErr_Occurred(tstate)) // 0代超过阈值就进行GC
{
gcstate->collecting = 1;
collect_generations(tstate);
gcstate->collecting = 0;
}
PyObject *op = FROM_GC(g);
return op;
}
/* Get the object given the GC head */
#define FROM_GC(g) ((PyObject *)(((PyGC_Head *)g)+1))
static Py_ssize_t
collect_generations(PyThreadState *tstate)
{
GCState *gcstate = &tstate->interp->gc;
/* Find the oldest generation (highest numbered) where the count
* exceeds the threshold. Objects in the that generation and
* generations younger than it will be collected. */
Py_ssize_t n = 0;
for (int i = NUM_GENERATIONS-1; i >= 0; i--) {
if (gcstate->generations[i].count > gcstate->generations[i].threshold) {
/* Avoid quadratic performance degradation in number
of tracked objects (see also issue #4074):
To limit the cost of garbage collection, there are two strategies;
- make each collection faster, e.g. by scanning fewer objects
- do less collections
This heuristic is about the latter strategy.
In addition to the various configurable thresholds, we only trigger a
full collection if the ratio
long_lived_pending / long_lived_total
is above a given value (hardwired to 25%).
The reason is that, while "non-full" collections (i.e., collections of
the young and middle generations) will always examine roughly the same
number of objects -- determined by the aforementioned thresholds --,
the cost of a full collection is proportional to the total number of
long-lived objects, which is virtually unbounded.
Indeed, it has been remarked that doing a full collection every
<constant number> of object creations entails a dramatic performance
degradation in workloads which consist in creating and storing lots of
long-lived objects (e.g. building a large list of GC-tracked objects would
show quadratic performance, instead of linear as expected: see issue #4074).
Using the above ratio, instead, yields amortized linear performance in
the total number of objects (the effect of which can be summarized
thusly: "each full garbage collection is more and more costly as the
number of objects grows, but we do fewer and fewer of them").
This heuristic was suggested by Martin von Löwis on python-dev in
June 2008. His original analysis and proposal can be found at:
http://mail.python.org/pipermail/python-dev/2008-June/080579.html
*/
if (i == NUM_GENERATIONS - 1
&& gcstate->long_lived_pending < gcstate->long_lived_total / 4)
continue; // 如果是最高代并且 long_lived_pending / long_lived_total < 25% 就跳过
n = collect_with_callback(tstate, i);
break;
}
}
return n;
}
/* Perform garbage collection of a generation and invoke
* progress callbacks.
*/
static Py_ssize_t
collect_with_callback(PyThreadState *tstate, int generation)
{
assert(!_PyErr_Occurred(tstate)); // 这段应该是检查python的线程状态的
Py_ssize_t result, collected, uncollectable;
invoke_gc_callback(tstate, "start", generation, 0, 0); // 调用进程回调以通知客户端垃圾回收正在开始或停止
result = collect(tstate, generation, &collected, &uncollectable, 0); // GC 主流程
invoke_gc_callback(tstate, "stop", generation, collected, uncollectable);
assert(!_PyErr_Occurred(tstate));
return result;
}
/* This is the main function. Read this to understand how the
* collection process works. */
static Py_ssize_t
collect(PyThreadState *tstate, int generation,
Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail)
{
int i;
Py_ssize_t m = 0; /* # objects collected */
Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */
PyGC_Head *young; /* the generation we are examining */
PyGC_Head *old; /* next older generation */
PyGC_Head unreachable; /* non-problematic unreachable trash */
PyGC_Head finalizers; /* objects with, & reachable from, __del__ */
PyGC_Head *gc;
_PyTime_t t1 = 0; /* initialize to prevent a compiler warning */
GCState *gcstate = &tstate->interp->gc; // 下面会介绍这个结构体
if (gcstate->debug & DEBUG_STATS) {
PySys_WriteStderr("gc: collecting generation %d...\n", generation);
show_stats_each_generations(gcstate);
t1 = _PyTime_GetMonotonicClock();
}
if (PyDTrace_GC_START_ENABLED())
PyDTrace_GC_START(generation);
/* update collection and allocation counters */
if (generation+1 < NUM_GENERATIONS)
gcstate->generations[generation+1].count += 1; // 当前代如果小于最高代，当前代+1的代 count+1
for (i = 0; i <= generation; i++)
gcstate->generations[i].count = 0; // 当前代和小于当前代的 count 都置为0
/* merge younger generations with one we are currently collecting */
for (i = 0; i < generation; i++) {
gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation));
} // 把小于当前代的都合并到当前代，并且清空小于当前代的
/* handy references */
young = GEN_HEAD(gcstate, generation);
if (generation < NUM_GENERATIONS-1)
old = GEN_HEAD(gcstate, generation+1);
else
old = young;
validate_list(old, collecting_clear_unreachable_clear);
deduce_unreachable(young, &unreachable);
untrack_tuples(young);
/* Move reachable objects to next generation. */
if (young != old) {
if (generation == NUM_GENERATIONS - 2) {
gcstate->long_lived_pending += gc_list_size(young);
} // 只有1代会把 list_size 给 gcstate->long_lived_pending
gc_list_merge(young, old);
}
else { // 第2代会执行的步骤
/* We only un-track dicts in full collections, to avoid quadratic
dict build-up. See issue #14775. */
untrack_dicts(young);
gcstate->long_lived_pending = 0;
gcstate->long_lived_total = gc_list_size(young);
}
/* All objects in unreachable are trash, but objects reachable from
* legacy finalizers (e.g. tp_del) can't safely be deleted.
*/
gc_list_init(&finalizers); // 循环所有具有__del__方法的不可达对象，放到finalizers链表中
// NEXT_MASK_UNREACHABLE is cleared here.
// After move_legacy_finalizers(), unreachable is normal list.
move_legacy_finalizers(&unreachable, &finalizers);
/* finalizers contains the unreachable objects with a legacy finalizer;
* unreachable objects reachable *from* those are also uncollectable,
* and we move those into the finalizers list too.
*/
move_legacy_finalizer_reachable(&finalizers);
validate_list(&finalizers, collecting_clear_unreachable_clear);
validate_list(&unreachable, collecting_set_unreachable_clear);
/* Print debugging information. */
if (gcstate->debug & DEBUG_COLLECTABLE) {
for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) {
debug_cycle("collectable", FROM_GC(gc));
}
}
/* Clear weakrefs and invoke callbacks as necessary. */
m += handle_weakrefs(&unreachable, old); // 处理弱引用
validate_list(old, collecting_clear_unreachable_clear);
validate_list(&unreachable, collecting_set_unreachable_clear);
/* Call tp_finalize on objects which have one. */
finalize_garbage(tstate, &unreachable);
/* Handle any objects that may have resurrected after the call
* to 'finalize_garbage' and continue the collection with the
* objects that are still unreachable */
PyGC_Head final_unreachable;
handle_resurrected_objects(&unreachable, &final_unreachable, old);
/* Call tp_clear on objects in the final_unreachable set. This will cause
* the reference cycles to be broken. It may also cause some objects
* in finalizers to be freed.
*/
m += gc_list_size(&final_unreachable);
delete_garbage(tstate, gcstate, &final_unreachable, old);
/* Collect statistics on uncollectable objects found and print
* debugging information. */
for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) {
n++;
if (gcstate->debug & DEBUG_UNCOLLECTABLE)
debug_cycle("uncollectable", FROM_GC(gc));
}
if (gcstate->debug & DEBUG_STATS) {
double d = _PyTime_AsSecondsDouble(_PyTime_GetMonotonicClock() - t1);
PySys_WriteStderr(
"gc: done, %" PY_FORMAT_SIZE_T "d unreachable, "
"%" PY_FORMAT_SIZE_T "d uncollectable, %.4fs elapsed\n",
n+m, n, d);
}
/* Append instances in the uncollectable set to a Python
* reachable list of garbage. The programmer has to deal with
* this if they insist on creating this type of structure.
*/
handle_legacy_finalizers(tstate, gcstate, &finalizers, old);
validate_list(old, collecting_clear_unreachable_clear);
/* Clear free list only during the collection of the highest
* generation */
if (generation == NUM_GENERATIONS-1) {
clear_freelists();
}
if (_PyErr_Occurred(tstate)) {
if (nofail) {
_PyErr_Clear(tstate);
}
else {
_PyErr_WriteUnraisableMsg("in garbage collection", NULL);
}
}
/* Update stats */
if (n_collected) {
*n_collected = m;
}
if (n_uncollectable) {
*n_uncollectable = n;
}
struct gc_generation_stats *stats = &gcstate->generation_stats[generation];
stats->collections++;
stats->collected += m;
stats->uncollectable += n;
if (PyDTrace_GC_DONE_ENABLED()) {
PyDTrace_GC_DONE(n + m);
}
assert(!_PyErr_Occurred(tstate));
return n + m;
}
Include/internal/pycore_gc.h:
/* If we change this, we need to change the default value in the
signature of gc.collect. */
#define NUM_GENERATIONS 3
引用
Include/object.h:
static inline void _Py_INCREF(PyObject *op)
{
#ifdef Py_REF_DEBUG
_Py_RefTotal++;
#endif
op->ob_refcnt++;
}
#define Py_INCREF(op) _Py_INCREF(_PyObject_CAST(op))
销毁
Objects/listobject.c:
PyTypeObject PyList_Type = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"list",
sizeof(PyListObject),
0,
(destructor)list_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
(reprfunc)list_repr, /* tp_repr */
0, /* tp_as_number */
&list_as_sequence, /* tp_as_sequence */
&list_as_mapping, /* tp_as_mapping */
PyObject_HashNotImplemented, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC |
Py_TPFLAGS_BASETYPE | Py_TPFLAGS_LIST_SUBCLASS, /* tp_flags */
list___init____doc__, /* tp_doc */
(traverseproc)list_traverse, /* tp_traverse */
(inquiry)_list_clear, /* tp_clear */
list_richcompare, /* tp_richcompare */
0, /* tp_weaklistoffset */
list_iter, /* tp_iter */
0, /* tp_iternext */
list_methods, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
(initproc)list___init__, /* tp_init */
PyType_GenericAlloc, /* tp_alloc */
PyType_GenericNew, /* tp_new */
PyObject_GC_Del, /* tp_free */
.tp_vectorcall = list_vectorcall,
};
/* Empty list reuse scheme to save calls to malloc and free */
#ifndef PyList_MAXFREELIST
# define PyList_MAXFREELIST 80
#endif
#if PyDict_MAXFREELIST > 0
static PyDictObject *free_list[PyDict_MAXFREELIST];
static int numfree = 0;
static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
static int numfreekeys = 0;
#endif
static void
list_dealloc(PyListObject *op)
{
Py_ssize_t i;
PyObject_GC_UnTrack(op); // 从GC中移除
Py_TRASHCAN_BEGIN(op, list_dealloc)
if (op->ob_item != NULL) {
/* Do it backwards, for Christian Tismer.
There's a simple test case where somehow this reduces
thrashing when a *very* large list is created and
immediately deleted. */
i = Py_SIZE(op);
while (--i >= 0) {
Py_XDECREF(op->ob_item[i]);
}
PyMem_FREE(op->ob_item);
}
// 如果不满80就缓存，满了80就释放对应的内存
if (numfree < PyList_MAXFREELIST && PyList_CheckExact(op))
free_list[numfree++] = op;
else
Py_TYPE(op)->tp_free((PyObject *)op);
Py_TRASHCAN_END
}
GC 分代知识点
python GC 一共分3代，从 Collecting the oldest generation 可以看出最高代 GC 只在下面的情况发生，和老师说的不太一样，我还在研究中。除了最高代，每次 GC 发生都会把当前代的 long-lived 对象放到下一代中。
In addition to the various configurable thresholds, the GC only triggers a full collection of the oldest generation if the ratio long_lived_pending / long_lived_total is above a given value (hardwired to 25%).
查看 GC threshold
>>> import gc
>>> gc.get_threshold()
(700, 10, 10)
三代 GC 的解释：
Doc/library/gc.rst:
The GC classifies objects into three generations depending on how many
collection sweeps they have survived. New objects are placed in the youngest
generation (generation ``0``). If an object survives a collection it is moved
into the next older generation. Since generation ``2`` is the oldest
generation, objects in that generation remain there after a collection. In
order to decide when to run, the collector keeps track of the number object
allocations and deallocations since the last collection. When the number of
allocations minus the number of deallocations exceeds *threshold0*, collection
starts. Initially only generation ``0`` is examined. If generation ``0`` has
been examined more than *threshold1* times since generation ``1`` has been
examined, then generation ``1`` is examined as well.
With the third generation, things are a bit more complicated,
see `Collecting the oldest generation <https://devguide.python.org/garbage_collector/#collecting-the-oldest-generation>`_ for more information.
可以总结 GC 的默认触发条件：
- 0代，generations[0].count 超过700（这个700是个数）
- 1代，generations[1].count 超过10（这个10是次数）
- 2代，generations[2].count 超过10 并且 long_lived_pending / long_lived_total 超过 25%
GC 分代的对象：
Include/internal/pycore_gc.h:
struct gc_generation {
PyGC_Head head;
int threshold; /* collection threshold */
int count; /* count of allocations or collections of younger
generations */
};
/* GC information is stored BEFORE the object structure. */
typedef struct {
// Pointer to next object in the list.
// 0 means the object is not tracked
uintptr_t _gc_next;
// Pointer to previous object in the list.
// Lowest two bits are used for flags documented later.
uintptr_t _gc_prev;
} PyGC_Head;
struct _gc_runtime_state {
/* List of objects that still need to be cleaned up, singly linked
* via their gc headers' gc_prev pointers. */
PyObject *trash_delete_later;
/* Current call-stack depth of tp_dealloc calls. */
int trash_delete_nesting;
int enabled;
int debug;
/* linked lists of container objects */
struct gc_generation generations[NUM_GENERATIONS];
PyGC_Head *generation0;
/* a permanent generation which won't be collected */
struct gc_generation permanent_generation;
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
/* true if we are currently running the collector */
int collecting;
/* list of uncollectable objects */
PyObject *garbage;
/* a list of callbacks to be invoked when collection is performed */
PyObject *callbacks;
/* This is the number of objects that survived the last full
collection. It approximates the number of long lived objects
tracked by the GC.
(by "full collection", we mean a collection of the oldest
generation). */
Py_ssize_t long_lived_total;
/* This is the number of objects that survived all "non-full"
collections, and are awaiting to undergo a full collection for
the first time. */
Py_ssize_t long_lived_pending;
};
GC 缓存知识点
int 有常驻内存的值，如果是 -5 到 257 范围内的值会直接链接到已有的内存中，不会新分配内存。
Include/internal/pycore_interp.h:
#define _PY_NSMALLPOSINTS 257
#define _PY_NSMALLNEGINTS 5
Objects/longobject.c:
#define NSMALLPOSINTS _PY_NSMALLPOSINTS
#define NSMALLNEGINTS _PY_NSMALLNEGINTS
#if NSMALLNEGINTS + NSMALLPOSINTS > 0
#define IS_SMALL_INT(ival) (-NSMALLNEGINTS <= (ival) && (ival) < NSMALLPOSINTS)
#define IS_SMALL_UINT(ival) ((ival) < NSMALLPOSINTS)
list，dict 等的 free_list 有最大个数
Objects/listobject.c:
/* Empty list reuse scheme to save calls to malloc and free */
#ifndef PyList_MAXFREELIST
# define PyList_MAXFREELIST 80
#endif
#if PyDict_MAXFREELIST > 0
static PyDictObject *free_list[PyDict_MAXFREELIST];
static int numfree = 0;
static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
static int numfreekeys = 0;
#endif
Objects/dictobject.c:
/* Dictionary reuse scheme to save calls to malloc and free */
#ifndef PyDict_MAXFREELIST
#define PyDict_MAXFREELIST 80
#endif
#if PyDict_MAXFREELIST > 0
static PyDictObject *free_list[PyDict_MAXFREELIST];
static int numfree = 0;
static PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
static int numfreekeys = 0;
#endif
元组的 free_list 只存元组元素个数 <= 20 个的，每个个数的最多存 2000 个元组。
Objects/tupleobject.c:
/* Speed optimization to avoid frequent malloc/free of small tuples */
#ifndef PyTuple_MAXSAVESIZE
#define PyTuple_MAXSAVESIZE 20 /* Largest tuple to save on free list */
#endif
#ifndef PyTuple_MAXFREELIST
#define PyTuple_MAXFREELIST 2000 /* Maximum number of tuples of each size to save */
#endif
#if PyTuple_MAXSAVESIZE > 0
/* Entries 1 up to PyTuple_MAXSAVESIZE are free lists, entry 0 is the empty
tuple () of which at most one instance will be allocated.
*/
static PyTupleObject *free_list[PyTuple_MAXSAVESIZE];
static int numfree[PyTuple_MAXSAVESIZE];
#endif
实际操作
最开始用了 pycharm 内部的 python console，结果和预期的不一样，应该是 pycharm 内部的 python console 启动的时候就已经分配了很多内存，可能有些 free_list 已经满了。下面是直接从命令行启动的 python。
>>> s=[1,2]
>>> id(s)
140203065141832
>>> del s
>>> n=[3]
>>> id(n)
140203065141832
后记
调试内存泄漏方面 objgraph 是个很好用的工具
总结
python 的 GC 过程相对 java 来讲可能简单了很多，但是实际看下来还是很多细节，很多需要推敲的地方，不过这个过程自己也学习了不少东西。