diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 3c8d82958fd7c8..f96f5b03fa8187 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -2,6 +2,7 @@ from test import list_tests from test.support import cpython_only import pickle +import struct import unittest class ListTest(list_tests.CommonTest): @@ -196,6 +197,63 @@ def test_preallocation(self): self.assertEqual(iter_size, sys.getsizeof(list([0] * 10))) self.assertEqual(iter_size, sys.getsizeof(list(range(10)))) + @cpython_only + def test_overallocation(self): + # bpo-33234: Don't overallocate when initialized from known lengths + # bpo-38373: Allows list over-allocation to be zero for some lengths + # bpo-43574: Don't overallocate for list-literals + sizeof = sys.getsizeof + + # First handle empty list and empty list-literal cases. Should have no + # overallocation, including init from iterable of unknown length. + self.assertEqual(sizeof([]), sizeof(list())) + self.assertEqual(sizeof([]), sizeof(list(tuple()))) + self.assertEqual(sizeof([]), sizeof(list(x for x in []))) + + # Must use actual list-literals to test the overallocation behavior of + # compiled list-literals as well as those initialized from them. + test_literals = [ + [1], + [1,2], + [1,2,3], # Literals of length > 2 are special-cased in compile + [1,2,3,4], + [1,2,3,4,5,6,7], + [1,2,3,4,5,6,7,8], # bpo-38373: Length 8 init won't over-alloc + [1,2,3,4,5,6,7,8,9], + ] + + overalloc_amts = [] + for literal in test_literals: + # Direct check that list-literals do not over-allocate, by + # calculating the total size of used pointers. + total_ptr_size = len(literal) * struct.calcsize('P') + self.assertEqual(sizeof(literal), sizeof([]) + total_ptr_size) + + # Ensure that both list literals, and lists made from an iterable + # of known size, use the same amount of allocation. + self.assertEqual(sizeof(literal), sizeof(list(literal))) + self.assertEqual(sizeof(literal), sizeof(list(tuple(literal)))) + + # By contrast, confirm that non-empty lists initialized from an + # iterable where the length is unknown at the time of + # initialization, can be overallocated. + iterated_list = list(x for x in literal) + overalloc_amts.append(sizeof(iterated_list) - sizeof(literal)) + self.assertGreaterEqual(sizeof(iterated_list), sizeof(literal)) + + # bpo-38373: initialized or grown lists are not always over-allocated. + # Confirm that over-allocation occurs at least some of the time. + self.assertEqual(True, any(x>0 for x in overalloc_amts)) + + # Empty lists should overallocate on initial append/insert (unlike + # list-literals) + l1 = [] + l1.append(1) + self.assertGreater(sizeof(l1), sizeof([1])) + l2 = [] + l2.insert(0, 1) + self.assertGreater(sizeof(l2), sizeof([1])) + def test_count_index_remove_crashes(self): # bpo-38610: The count(), index(), and remove() methods were not # holding strong references to list elements while calling diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-03-21-18-51-30.bpo-43574.mteI-I.rst b/Misc/NEWS.d/next/Core and Builtins/2021-03-21-18-51-30.bpo-43574.mteI-I.rst new file mode 100644 index 00000000000000..818ea4465f4731 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-03-21-18-51-30.bpo-43574.mteI-I.rst @@ -0,0 +1,3 @@ +``list`` objects don't overallocate when starting empty and then extended, or +when set to be empty. This effectively restores previous ``list`` memory +behavior for lists initialized from literals. diff --git a/Objects/listobject.c b/Objects/listobject.c index e7987a6d352bfa..05b0de144c71a9 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -58,25 +58,33 @@ list_resize(PyListObject *self, Py_ssize_t newsize) return 0; } - /* This over-allocates proportional to the list size, making room - * for additional growth. The over-allocation is mild, but is - * enough to give linear-time amortized behavior over a long - * sequence of appends() in the presence of a poorly-performing - * system realloc(). - * Add padding to make the allocated size multiple of 4. - * The growth pattern is: 0, 4, 8, 16, 24, 32, 40, 52, 64, 76, ... - * Note: new_allocated won't overflow because the largest possible value - * is PY_SSIZE_T_MAX * (9 / 8) + 6 which always fits in a size_t. - */ - new_allocated = ((size_t)newsize + (newsize >> 3) + 6) & ~(size_t)3; - /* Do not overallocate if the new size is closer to overallocated size - * than to the old size. - */ - if (newsize - Py_SIZE(self) > (Py_ssize_t)(new_allocated - newsize)) - new_allocated = ((size_t)newsize + 3) & ~(size_t)3; + if (newsize == 0 || (Py_SIZE(self) == 0 && newsize > 1)) { + /* Don't overallocate empty lists that are extended by more than 1 + * element. This helps ensure that list-literals aren't + * over-allocated, but still allows it for empty-list append/insert. + */ + new_allocated = newsize; + } + else { + /* This over-allocates proportional to the list size, making room + * for additional growth. The over-allocation is mild, but is + * enough to give linear-time amortized behavior over a long + * sequence of appends() in the presence of a poorly-performing + * system realloc(). + * Add padding to make the allocated size multiple of 4. + * The growth pattern is: 0, 4, 8, 16, 24, 32, 40, 52, 64, 76, ... + * Note: new_allocated won't overflow because the largest possible value + * is PY_SSIZE_T_MAX * (9 / 8) + 6 which always fits in a size_t. + */ + new_allocated = ((size_t)newsize + (newsize >> 3) + 6) & ~(size_t)3; + /* Do not overallocate if the new size is closer to overallocated size + * than to the old size. + */ + if (newsize - Py_SIZE(self) > (Py_ssize_t)(new_allocated - newsize)) { + new_allocated = ((size_t)newsize + 3) & ~(size_t)3; + } + } - if (newsize == 0) - new_allocated = 0; num_allocated_bytes = new_allocated * sizeof(PyObject *); items = (PyObject **)PyMem_Realloc(self->ob_item, num_allocated_bytes); if (items == NULL) {