pythongh-127635: Use flexible array in tracemalloc

Replace frames[1] with frames[] in tracemalloc_traceback structure.

Author: vstinner

Include/internal/pycore_tracemalloc.h

@@ -30,8 +30,8 @@ struct _PyTraceMalloc_Config {
 };
 
 
-/* Pack the frame_t structure to reduce the memory footprint on 64-bit
-   architectures: 12 bytes instead of 16. */
+/* Pack the tracemalloc_frame and tracemalloc_traceback structures to reduce
+   the memory footprint on 64-bit architectures: 12 bytes instead of 16. */
 #if defined(_MSC_VER)
 #pragma pack(push, 4)
 #endif
@@ -46,18 +46,22 @@ tracemalloc_frame {
     PyObject *filename;
     unsigned int lineno;
 };
-#ifdef _MSC_VER
-#pragma pack(pop)
-#endif
 
-struct tracemalloc_traceback {
+struct
+#ifdef __GNUC__
+__attribute__((packed))
+#endif
+tracemalloc_traceback {
     Py_uhash_t hash;
     /* Number of frames stored */
     uint16_t nframe;
     /* Total number of frames the traceback had */
     uint16_t total_nframe;
-    struct tracemalloc_frame frames[1];
+    struct tracemalloc_frame frames[];
 };
+#ifdef _MSC_VER
+#pragma pack(pop)
+#endif
 
 
 struct _tracemalloc_runtime_state {
@@ -95,7 +99,7 @@ struct _tracemalloc_runtime_state {
        Protected by TABLES_LOCK(). */
     _Py_hashtable_t *domains;
 
-    struct tracemalloc_traceback empty_traceback;
+    struct tracemalloc_traceback *empty_traceback;
 
     Py_tss_t reentrant_key;
 };

Python/tracemalloc.c

@@ -46,7 +46,7 @@ typedef struct tracemalloc_frame frame_t;
 typedef struct tracemalloc_traceback traceback_t;
 
 #define TRACEBACK_SIZE(NFRAME) \
-        (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1))
+        (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME))
 
 static const int MAX_NFRAME = UINT16_MAX;
 
@@ -329,8 +329,9 @@ traceback_new(void)
     traceback->nframe = 0;
     traceback->total_nframe = 0;
     traceback_get_frames(traceback);
-    if (traceback->nframe == 0)
-        return &tracemalloc_empty_traceback;
+    if (traceback->nframe == 0) {
+        return tracemalloc_empty_traceback;
+    }
     traceback->hash = traceback_hash(traceback);
 
     /* intern the traceback */
@@ -754,12 +755,18 @@ _PyTraceMalloc_Init(void)
         return _PyStatus_NO_MEMORY();
     }
 
-    tracemalloc_empty_traceback.nframe = 1;
-    tracemalloc_empty_traceback.total_nframe = 1;
+    assert(tracemalloc_empty_traceback == NULL);
+    tracemalloc_empty_traceback = raw_malloc(TRACEBACK_SIZE(1));
+    if (tracemalloc_empty_traceback  == NULL) {
+        return _PyStatus_NO_MEMORY();
+    }
+
+    tracemalloc_empty_traceback->nframe = 1;
+    tracemalloc_empty_traceback->total_nframe = 1;
     /* borrowed reference */
-    tracemalloc_empty_traceback.frames[0].filename = &_Py_STR(anon_unknown);
-    tracemalloc_empty_traceback.frames[0].lineno = 0;
-    tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback);
+    tracemalloc_empty_traceback->frames[0].filename = &_Py_STR(anon_unknown);
+    tracemalloc_empty_traceback->frames[0].lineno = 0;
+    tracemalloc_empty_traceback->hash = traceback_hash(tracemalloc_empty_traceback);
 
     tracemalloc_config.initialized = TRACEMALLOC_INITIALIZED;
     return _PyStatus_OK();
@@ -782,6 +789,9 @@ tracemalloc_deinit(void)
     _Py_hashtable_destroy(tracemalloc_filenames);
 
     PyThread_tss_delete(&tracemalloc_reentrant_key);
+
+    raw_free(tracemalloc_empty_traceback);
+    tracemalloc_empty_traceback = NULL;
 }