diff --git a/src/backend/utils/mmgr/aset.c b/src/backend/utils/mmgr/aset.c index 743455e..3ef9814 100644 --- a/src/backend/utils/mmgr/aset.c +++ b/src/backend/utils/mmgr/aset.c @@ -1,148 +1,56 @@ /*------------------------------------------------------------------------- - * - * aset.c - * Allocation set definitions. - * - * AllocSet is our standard implementation of the abstract MemoryContext - * type. - * - * - * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * - * IDENTIFICATION - * src/backend/utils/mmgr/aset.c - * - * NOTE: - * This is a new (Feb. 05, 1999) implementation of the allocation set - * routines. AllocSet...() does not use OrderedSet...() any more. - * Instead it manages allocations in a block pool by itself, combining - * many small allocations in a few bigger blocks. AllocSetFree() normally - * doesn't free() memory really. It just add's the free'd area to some - * list for later reuse by AllocSetAlloc(). All memory blocks are free()'d - * at once on AllocSetReset(), which happens when the memory context gets - * destroyed. - * Jan Wieck - * - * Performance improvement from Tom Lane, 8/99: for extremely large request - * sizes, we do want to be able to give the memory back to free() as soon - * as it is pfree()'d. Otherwise we risk tying up a lot of memory in - * freelist entries that might never be usable. This is specially needed - * when the caller is repeatedly repalloc()'ing a block bigger and bigger; - * the previous instances of the block were guaranteed to be wasted until - * AllocSetReset() under the old way. - * - * Further improvement 12/00: as the code stood, request sizes in the - * midrange between "small" and "large" were handled very inefficiently, - * because any sufficiently large free chunk would be used to satisfy a - * request, even if it was much larger than necessary. This led to more - * and more wasted space in allocated chunks over time. To fix, get rid - * of the midrange behavior: we now handle only "small" power-of-2-size - * chunks as chunks. Anything "large" is passed off to malloc(). Change - * the number of freelists to change the small/large boundary. - * - * - * About CLOBBER_FREED_MEMORY: - * - * If this symbol is defined, all freed memory is overwritten with 0x7F's. - * This is useful for catching places that reference already-freed memory. - * - * About MEMORY_CONTEXT_CHECKING: - * - * Since we usually round request sizes up to the next power of 2, there - * is often some unused space immediately after a requested data area. - * Thus, if someone makes the common error of writing past what they've - * requested, the problem is likely to go unnoticed ... until the day when - * there *isn't* any wasted space, perhaps because of different memory - * alignment on a new platform, or some other effect. To catch this sort - * of problem, the MEMORY_CONTEXT_CHECKING option stores 0x7E just beyond - * the requested space whenever the request is less than the actual chunk - * size, and verifies that the byte is undamaged when the chunk is freed. - * - * - * About USE_VALGRIND and Valgrind client requests: - * - * Valgrind provides "client request" macros that exchange information with - * the host Valgrind (if any). Under !USE_VALGRIND, memdebug.h stubs out - * currently-used macros. - * - * When running under Valgrind, we want a NOACCESS memory region both before - * and after the allocation. The chunk header is tempting as the preceding - * region, but mcxt.c expects to able to examine the standard chunk header - * fields. Therefore, we use, when available, the requested_size field and - * any subsequent padding. requested_size is made NOACCESS before returning - * a chunk pointer to a caller. However, to reduce client request traffic, - * it is kept DEFINED in chunks on the free list. - * - * The rounded-up capacity of the chunk usually acts as a post-allocation - * NOACCESS region. If the request consumes precisely the entire chunk, - * there is no such region; another chunk header may immediately follow. In - * that case, Valgrind will not detect access beyond the end of the chunk. - * - * See also the cooperating Valgrind client requests in mcxt.c. - * - *------------------------------------------------------------------------- - */ +* +* Experimental replacement / alternative to allocation set, which is the +* default memory context implementation. +* +* The default AllocSet implementation allocates large blocks of memory +* at once (aka blocks) using malloc() and then splits them up into small +* pieces (aka chunks) to handle palloc() chunks. +* +* This default implementation has a number of advantages - for example it +* lowers the number of malloc() calls, and replaces them with a trivial +* pointer arithmetics (when getting chunk from an existing block). +* +* It also has some disadvantages, though. The obvious one is that it's +* impossible to free() individual chunks (because it's the whole block +* that was allocated using malloc), and the blocks are split in a static +* manner - it's impossible to merge the chunks and split them in a +* different way. So AllocSet does not free memory until the whole context +* is destroyed (not exactly true for over-sized requests, but let's +* ignore that for now). +* +* This disadvantage is mitigated by reusing chunks that are not needed +* (and pfree() was called on them), and it works quite well in most cases. +* But there are examples of workloads where this approach fails (e.g. +* continuously growing requests). +* +* Another question is how much better the generic allocators (e.g. the +* one provided by your kernel) got over the time, and whether it would +* be better to use a "trivial allocator" calling malloc/free directly. +* And that is what this experimental memory context does. +* +* There needs to be some sort of registry though, because reducing the +* malloc/free overhead is only one of the goals. The other one is making +* memory management easier by handling lifecycle for each small piece. +* +* For the default implementation this is done by tracking blocks, but this +* implementation has no blocks, and so it has to do that for each chunk +* by organizing the chunks into a doubly-linked list (to support pfree). +* +* This makes the AllocSetContext much simpler (no freelists, no blocks). +* It however makes the AllocChunkData (chunk header) larger, as it needs +* to track links to the prev/next chunk for the doubly-linked list. +* +*------------------------------------------------------------------------- +*/ #include "postgres.h" #include "utils/memdebug.h" #include "utils/memutils.h" -/* Define this to detail debug alloc information */ -/* #define HAVE_ALLOCINFO */ - -/*-------------------- - * Chunk freelist k holds chunks of size 1 << (k + ALLOC_MINBITS), - * for k = 0 .. ALLOCSET_NUM_FREELISTS-1. - * - * Note that all chunks in the freelists have power-of-2 sizes. This - * improves recyclability: we may waste some space, but the wasted space - * should stay pretty constant as requests are made and released. - * - * A request too large for the last freelist is handled by allocating a - * dedicated block from malloc(). The block still has a block header and - * chunk header, but when the chunk is freed we'll return the whole block - * to malloc(), not put it on our freelists. - * - * CAUTION: ALLOC_MINBITS must be large enough so that - * 1< head) */ + AllocChunk next; /* next item in the list (NULL -> last) */ + + /* aset is the owning aset if allocated */ void *aset; + /* size is always the size of the usable space in the chunk */ Size size; + #ifdef MEMORY_CONTEXT_CHECKING /* when debugging memory usage, also store actual requested size */ /* this is zero in a free chunk */ Size requested_size; #endif + } AllocChunkData; /* @@ -277,18 +162,6 @@ static MemoryContextMethods AllocSetMethods = { #endif }; -/* - * Table for AllocSetFreeIndex - */ -#define LT16(n) n, n, n, n, n, n, n, n, n, n, n, n, n, n, n, n - -static const unsigned char LogTable256[256] = -{ - 0, 1, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, - LT16(5), LT16(6), LT16(6), LT16(7), LT16(7), LT16(7), LT16(7), - LT16(8), LT16(8), LT16(8), LT16(8), LT16(8), LT16(8), LT16(8), LT16(8) -}; - /* ---------- * Debug macros * ---------- @@ -306,44 +179,20 @@ static const unsigned char LogTable256[256] = #endif /* ---------- - * AllocSetFreeIndex - - * - * Depending on the size of an allocation compute which freechunk - * list of the alloc set it belongs to. Caller must have verified - * that size <= ALLOC_CHUNK_LIMIT. + * AllocChunkSize. * ---------- */ static inline int -AllocSetFreeIndex(Size size) +AllocChunkSize(Size size) { - int idx; - unsigned int t, - tsize; - - if (size > (1 << ALLOC_MINBITS)) - { - tsize = (size - 1) >> ALLOC_MINBITS; - - /* - * At this point we need to obtain log2(tsize)+1, ie, the number of - * not-all-zero bits at the right. We used to do this with a - * shift-and-count loop, but this function is enough of a hotspot to - * justify micro-optimization effort. The best approach seems to be - * to use a lookup table. Note that this code assumes that - * ALLOCSET_NUM_FREELISTS <= 17, since we only cope with two bytes of - * the tsize value. - */ - t = tsize >> 8; - idx = t ? LogTable256[t] + 8 : LogTable256[tsize]; - - Assert(idx < ALLOCSET_NUM_FREELISTS); - } - else - idx = 0; + if (size < ALLOC_MINSIZE) + return ALLOC_MINSIZE; - return idx; + /* round to 4 bytes */ + return ((size + 3) / 4) * 4; } + #ifdef CLOBBER_FREED_MEMORY /* Wipe freed memory for debugging purposes */ @@ -427,9 +276,9 @@ randomize_mem(char *ptr, size_t size) * * parent: parent context, or NULL if top-level context * name: name of context (for debugging --- string will be copied) - * minContextSize: minimum context size - * initBlockSize: initial allocation block size - * maxBlockSize: maximum allocation block size + * minContextSize: (ignored) + * initBlockSize: (ignored) + * maxBlockSize: (ignored) */ MemoryContext AllocSetContextCreate(MemoryContext parent, @@ -447,71 +296,7 @@ AllocSetContextCreate(MemoryContext parent, parent, name); - /* - * Make sure alloc parameters are reasonable, and save them. - * - * We somewhat arbitrarily enforce a minimum 1K block size. - */ - initBlockSize = MAXALIGN(initBlockSize); - if (initBlockSize < 1024) - initBlockSize = 1024; - maxBlockSize = MAXALIGN(maxBlockSize); - if (maxBlockSize < initBlockSize) - maxBlockSize = initBlockSize; - Assert(AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */ - context->initBlockSize = initBlockSize; - context->maxBlockSize = maxBlockSize; - context->nextBlockSize = initBlockSize; - - /* - * Compute the allocation chunk size limit for this context. It can't be - * more than ALLOC_CHUNK_LIMIT because of the fixed number of freelists. - * If maxBlockSize is small then requests exceeding the maxBlockSize, or - * even a significant fraction of it, should be treated as large chunks - * too. For the typical case of maxBlockSize a power of 2, the chunk size - * limit will be at most 1/8th maxBlockSize, so that given a stream of - * requests that are all the maximum chunk size we will waste at most - * 1/8th of the allocated space. - * - * We have to have allocChunkLimit a power of two, because the requested - * and actually-allocated sizes of any chunk must be on the same side of - * the limit, else we get confused about whether the chunk is "big". - */ - context->allocChunkLimit = ALLOC_CHUNK_LIMIT; - while ((Size) (context->allocChunkLimit + ALLOC_CHUNKHDRSZ) > - (Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION)) - context->allocChunkLimit >>= 1; - - /* - * Grab always-allocated space, if requested - */ - if (minContextSize > ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ) - { - Size blksize = MAXALIGN(minContextSize); - AllocBlock block; - - block = (AllocBlock) malloc(blksize); - if (block == NULL) - { - MemoryContextStats(TopMemoryContext); - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"), - errdetail("Failed while creating memory context \"%s\".", - name))); - } - block->aset = context; - block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; - block->endptr = ((char *) block) + blksize; - block->next = context->blocks; - context->blocks = block; - /* Mark block as not to be released at reset time */ - context->keeper = block; - - /* Mark unallocated space NOACCESS; leave the block header alone. */ - VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, - blksize - ALLOC_BLOCKHDRSZ); - } + context->chunks = NULL; return (MemoryContext) context; } @@ -541,18 +326,13 @@ AllocSetInit(MemoryContext context) * AllocSetReset * Frees all memory which is allocated in the given set. * - * Actually, this routine has some discretion about what to do. - * It should mark all allocated chunks freed, but it need not necessarily - * give back all the resources the set owns. Our actual implementation is - * that we hang onto any "keeper" block specified for the set. In this way, - * we don't thrash malloc() when a context is repeatedly reset after small - * allocations, which is typical behavior for per-tuple contexts. + * Free all the chunks in the memory context. */ static void AllocSetReset(MemoryContext context) { AllocSet set = (AllocSet) context; - AllocBlock block; + AllocChunk chunk = set->chunks; AssertArg(AllocSetIsValid(set)); @@ -561,83 +341,44 @@ AllocSetReset(MemoryContext context) AllocSetCheck(context); #endif - /* Clear chunk freelists */ - MemSetAligned(set->freelist, 0, sizeof(set->freelist)); - - block = set->blocks; - - /* New blocks list is either empty or just the keeper block */ - set->blocks = set->keeper; - - while (block != NULL) + while (chunk != NULL) { - AllocBlock next = block->next; + AllocChunk next = chunk->next; - if (block == set->keeper) - { - /* Reset the block, but don't return it to malloc */ - char *datastart = ((char *) block) + ALLOC_BLOCKHDRSZ; - -#ifdef CLOBBER_FREED_MEMORY - wipe_mem(datastart, block->freeptr - datastart); -#else - /* wipe_mem() would have done this */ - VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart); +#ifdef MEMORY_CONTEXT_CHECKING + void *pointer = AllocChunkGetPointer(chunk); + VALGRIND_MAKE_MEM_DEFINED(&chunk->requested_size, + sizeof(chunk->requested_size)); + /* Test for someone scribbling on unused space in chunk */ + + if (chunk->requested_size < chunk->size) + if (!sentinel_ok(pointer, chunk->requested_size)) + elog(WARNING, "detected write past chunk end in %s %p (%ld < %ld)", + set->header.name, chunk, chunk->requested_size, chunk->size); #endif - block->freeptr = datastart; - block->next = NULL; - } - else - { - /* Normal case, release the block */ + #ifdef CLOBBER_FREED_MEMORY - wipe_mem(block, block->freeptr - ((char *) block)); + /* clobber everything (including the header) */ + wipe_mem(chunk, chunk->size + ALLOC_CHUNKHDRSZ); #endif - free(block); - } - block = next; + free(chunk); + + chunk = next; } - /* Reset block size allocation sequence, too */ - set->nextBlockSize = set->initBlockSize; + set->chunks = NULL; } /* * AllocSetDelete * Frees all memory which is allocated in the given set, * in preparation for deletion of the set. - * - * Unlike AllocSetReset, this *must* free all resources of the set. - * But note we are not responsible for deleting the context node itself. */ static void AllocSetDelete(MemoryContext context) { - AllocSet set = (AllocSet) context; - AllocBlock block = set->blocks; - - AssertArg(AllocSetIsValid(set)); - -#ifdef MEMORY_CONTEXT_CHECKING - /* Check for corruption and leaks before freeing */ - AllocSetCheck(context); -#endif - - /* Make it look empty, just in case... */ - MemSetAligned(set->freelist, 0, sizeof(set->freelist)); - set->blocks = NULL; - set->keeper = NULL; - - while (block != NULL) - { - AllocBlock next = block->next; - -#ifdef CLOBBER_FREED_MEMORY - wipe_mem(block, block->freeptr - ((char *) block)); -#endif - free(block); - block = next; - } + /* just reset the context ... */ + AllocSetReset(context); } /* @@ -653,269 +394,45 @@ static void * AllocSetAlloc(MemoryContext context, Size size) { AllocSet set = (AllocSet) context; - AllocBlock block; - AllocChunk chunk; - int fidx; - Size chunk_size; - Size blksize; + AllocChunk chunk = NULL; + Size chunk_size = size; AssertArg(AllocSetIsValid(set)); - /* - * If requested size exceeds maximum for chunks, allocate an entire block - * for this request. - */ - if (size > set->allocChunkLimit) - { - chunk_size = MAXALIGN(size); - blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; - block = (AllocBlock) malloc(blksize); - if (block == NULL) - { - MemoryContextStats(TopMemoryContext); - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"), - errdetail("Failed on request of size %zu.", size))); - } - block->aset = set; - block->freeptr = block->endptr = ((char *) block) + blksize; + /* allocate the chunk with a header, add it to the 'chunks' list */ + chunk_size = AllocChunkSize(size); - chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); - chunk->aset = set; - chunk->size = chunk_size; -#ifdef MEMORY_CONTEXT_CHECKING - /* Valgrind: Will be made NOACCESS below. */ - chunk->requested_size = size; - /* set mark to catch clobber of "unused" space */ - if (size < chunk_size) - set_sentinel(AllocChunkGetPointer(chunk), size); -#endif -#ifdef RANDOMIZE_ALLOCATED_MEMORY - /* fill the allocated space with junk */ - randomize_mem((char *) AllocChunkGetPointer(chunk), size); -#endif + chunk = (AllocChunk)malloc(chunk_size + ALLOC_CHUNKHDRSZ); - /* - * Stick the new block underneath the active allocation block, so that - * we don't lose the use of the space remaining therein. - */ - if (set->blocks != NULL) - { - block->next = set->blocks->next; - set->blocks->next = block; - } - else - { - block->next = NULL; - set->blocks = block; - } - - AllocAllocInfo(set, chunk); - - /* - * Chunk header public fields remain DEFINED. The requested - * allocation itself can be NOACCESS or UNDEFINED; our caller will - * soon make it UNDEFINED. Make extra space at the end of the chunk, - * if any, NOACCESS. - */ - VALGRIND_MAKE_MEM_NOACCESS((char *) chunk + ALLOC_CHUNK_PUBLIC, - chunk_size + ALLOC_CHUNKHDRSZ - ALLOC_CHUNK_PUBLIC); - - return AllocChunkGetPointer(chunk); - } - - /* - * Request is small enough to be treated as a chunk. Look in the - * corresponding free list to see if there is a free chunk we could reuse. - * If one is found, remove it from the free list, make it again a member - * of the alloc set and return its data address. - */ - fidx = AllocSetFreeIndex(size); - chunk = set->freelist[fidx]; - if (chunk != NULL) + /* if malloc failed, report the error */ + if (chunk == NULL) { - Assert(chunk->size >= size); - - set->freelist[fidx] = (AllocChunk) chunk->aset; - - chunk->aset = (void *) set; - -#ifdef MEMORY_CONTEXT_CHECKING - /* Valgrind: Free list requested_size should be DEFINED. */ - chunk->requested_size = size; - VALGRIND_MAKE_MEM_NOACCESS(&chunk->requested_size, - sizeof(chunk->requested_size)); - /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) - set_sentinel(AllocChunkGetPointer(chunk), size); -#endif -#ifdef RANDOMIZE_ALLOCATED_MEMORY - /* fill the allocated space with junk */ - randomize_mem((char *) AllocChunkGetPointer(chunk), size); -#endif - - AllocAllocInfo(set, chunk); - return AllocChunkGetPointer(chunk); - } - - /* - * Choose the actual chunk size to allocate. - */ - chunk_size = (1 << ALLOC_MINBITS) << fidx; - Assert(chunk_size >= size); - - /* - * If there is enough room in the active allocation block, we will put the - * chunk into that block. Else must start a new one. - */ - if ((block = set->blocks) != NULL) - { - Size availspace = block->endptr - block->freeptr; - - if (availspace < (chunk_size + ALLOC_CHUNKHDRSZ)) - { - /* - * The existing active (top) block does not have enough room for - * the requested allocation, but it might still have a useful - * amount of space in it. Once we push it down in the block list, - * we'll never try to allocate more space from it. So, before we - * do that, carve up its free space into chunks that we can put on - * the set's freelists. - * - * Because we can only get here when there's less than - * ALLOC_CHUNK_LIMIT left in the block, this loop cannot iterate - * more than ALLOCSET_NUM_FREELISTS-1 times. - */ - while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ)) - { - Size availchunk = availspace - ALLOC_CHUNKHDRSZ; - int a_fidx = AllocSetFreeIndex(availchunk); - - /* - * In most cases, we'll get back the index of the next larger - * freelist than the one we need to put this chunk on. The - * exception is when availchunk is exactly a power of 2. - */ - if (availchunk != ((Size) 1 << (a_fidx + ALLOC_MINBITS))) - { - a_fidx--; - Assert(a_fidx >= 0); - availchunk = ((Size) 1 << (a_fidx + ALLOC_MINBITS)); - } - - chunk = (AllocChunk) (block->freeptr); - - /* Prepare to initialize the chunk header. */ - VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNK_USED); - - block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ); - availspace -= (availchunk + ALLOC_CHUNKHDRSZ); - - chunk->size = availchunk; -#ifdef MEMORY_CONTEXT_CHECKING - chunk->requested_size = 0; /* mark it free */ -#endif - chunk->aset = (void *) set->freelist[a_fidx]; - set->freelist[a_fidx] = chunk; - } - - /* Mark that we need to create a new block */ - block = NULL; - } + MemoryContextStats(TopMemoryContext); + ereport(ERROR, + (errcode(ERRCODE_OUT_OF_MEMORY), + errmsg("out of memory"), + errdetail("Failed on request of size %zu.", size))); } - /* - * Time to create a new regular (multi-chunk) block? - */ - if (block == NULL) - { - Size required_size; - - /* - * The first such block has size initBlockSize, and we double the - * space in each succeeding block, but not more than maxBlockSize. - */ - blksize = set->nextBlockSize; - set->nextBlockSize <<= 1; - if (set->nextBlockSize > set->maxBlockSize) - set->nextBlockSize = set->maxBlockSize; - - /* - * If initBlockSize is less than ALLOC_CHUNK_LIMIT, we could need more - * space... but try to keep it a power of 2. - */ - required_size = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; - while (blksize < required_size) - blksize <<= 1; - - /* Try to allocate it */ - block = (AllocBlock) malloc(blksize); - - /* - * We could be asking for pretty big blocks here, so cope if malloc - * fails. But give up if there's less than a meg or so available... - */ - while (block == NULL && blksize > 1024 * 1024) - { - blksize >>= 1; - if (blksize < required_size) - break; - block = (AllocBlock) malloc(blksize); - } - - if (block == NULL) - { - MemoryContextStats(TopMemoryContext); - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"), - errdetail("Failed on request of size %zu.", size))); - } - - block->aset = set; - block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ; - block->endptr = ((char *) block) + blksize; - - /* - * If this is the first block of the set, make it the "keeper" block. - * Formerly, a keeper block could only be created during context - * creation, but allowing it to happen here lets us have fast reset - * cycling even for contexts created with minContextSize = 0; that way - * we don't have to force space to be allocated in contexts that might - * never need any space. Don't mark an oversize block as a keeper, - * however. - */ - if (set->keeper == NULL && blksize == set->initBlockSize) - set->keeper = block; - - /* Mark unallocated space NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, - blksize - ALLOC_BLOCKHDRSZ); - - block->next = set->blocks; - set->blocks = block; - } + /* set the general chunk info */ + chunk->aset = set; + chunk->size = chunk_size; - /* - * OK, do the allocation - */ - chunk = (AllocChunk) (block->freeptr); + /* add the chunk to the head of the list */ + chunk->prev = NULL; + chunk->next = set->chunks; - /* Prepare to initialize the chunk header. */ - VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNK_USED); + /* update the current head (if there's any) */ + if (set->chunks != NULL) + set->chunks->prev = chunk; - block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ); - Assert(block->freeptr <= block->endptr); + set->chunks = chunk; - chunk->aset = (void *) set; - chunk->size = chunk_size; #ifdef MEMORY_CONTEXT_CHECKING + /* Valgrind: Will be made NOACCESS below. */ chunk->requested_size = size; - VALGRIND_MAKE_MEM_NOACCESS(&chunk->requested_size, - sizeof(chunk->requested_size)); /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) + if (size < chunk_size) set_sentinel(AllocChunkGetPointer(chunk), size); #endif #ifdef RANDOMIZE_ALLOCATED_MEMORY @@ -945,59 +462,45 @@ AllocSetFree(MemoryContext context, void *pointer) /* Test for someone scribbling on unused space in chunk */ if (chunk->requested_size < chunk->size) if (!sentinel_ok(pointer, chunk->requested_size)) - elog(WARNING, "detected write past chunk end in %s %p", - set->header.name, chunk); + elog(WARNING, "detected write past chunk end in %s %p (%ld < %ld)", + set->header.name, chunk, chunk->requested_size, chunk->size); #endif - if (chunk->size > set->allocChunkLimit) + /* FIXME could we remove this if? */ + if (set != NULL) { - /* - * Big chunks are certain to have been allocated as single-chunk - * blocks. Find the containing block and return it to malloc(). - */ - AllocBlock block = set->blocks; - AllocBlock prevblock = NULL; - - while (block != NULL) + if ((chunk->prev == NULL) && (chunk->next == NULL)) + { + /* head and tail at the same time (single chunk -> just reset the 'chunks' list) */ + set->chunks = NULL; + } + else if (chunk->prev == NULL) + { + /* head (but not tail) -> just mark the second element as head */ + set->chunks = chunk->next; + set->chunks->prev = NULL; + } + else if (chunk->next == NULL) { - if (chunk == (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ)) - break; - prevblock = block; - block = block->next; + /* tail (but not head) -> just mark the previous one as a tail */ + chunk->prev->next = NULL; + chunk->prev = NULL; } - if (block == NULL) - elog(ERROR, "could not find block containing chunk %p", chunk); - /* let's just make sure chunk is the only one in the block */ - Assert(block->freeptr == ((char *) block) + - (chunk->size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)); - - /* OK, remove block from aset's list and free it */ - if (prevblock == NULL) - set->blocks = block->next; else - prevblock->next = block->next; -#ifdef CLOBBER_FREED_MEMORY - wipe_mem(block, block->freeptr - ((char *) block)); -#endif - free(block); + { + /* nor a tail, nor a head */ + chunk->next->prev = chunk->prev; + chunk->prev->next = chunk->next; + } } - else - { - /* Normal case, put the chunk into appropriate freelist */ - int fidx = AllocSetFreeIndex(chunk->size); - - chunk->aset = (void *) set->freelist[fidx]; #ifdef CLOBBER_FREED_MEMORY - wipe_mem(pointer, chunk->size); + /* clobber the whole chunk (including the header) */ + wipe_mem(chunk, chunk->size + ALLOC_CHUNKHDRSZ); #endif -#ifdef MEMORY_CONTEXT_CHECKING - /* Reset requested_size to 0 in chunks that are on freelist */ - chunk->requested_size = 0; -#endif - set->freelist[fidx] = chunk; - } + /* free the memory */ + free(chunk); } /* @@ -1014,9 +517,9 @@ AllocSetFree(MemoryContext context, void *pointer) static void * AllocSetRealloc(MemoryContext context, void *pointer, Size size) { - AllocSet set = (AllocSet) context; AllocChunk chunk = AllocPointerGetChunk(pointer); Size oldsize = chunk->size; + void *newPointer; #ifdef MEMORY_CONTEXT_CHECKING VALGRIND_MAKE_MEM_DEFINED(&chunk->requested_size, @@ -1025,7 +528,7 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) if (chunk->requested_size < oldsize) if (!sentinel_ok(pointer, chunk->requested_size)) elog(WARNING, "detected write past chunk end in %s %p", - set->header.name, chunk); + ((AllocSet)context)->header.name, chunk); #endif /* @@ -1076,131 +579,16 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size) return pointer; } + + /* nah, we need a new chunk */ + newPointer = AllocSetAlloc(context, size); - if (oldsize > set->allocChunkLimit) - { - /* - * The chunk must have been allocated as a single-chunk block. Find - * the containing block and use realloc() to make it bigger with - * minimum space wastage. - */ - AllocBlock block = set->blocks; - AllocBlock prevblock = NULL; - Size chksize; - Size blksize; - - while (block != NULL) - { - if (chunk == (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ)) - break; - prevblock = block; - block = block->next; - } - if (block == NULL) - elog(ERROR, "could not find block containing chunk %p", chunk); - /* let's just make sure chunk is the only one in the block */ - Assert(block->freeptr == ((char *) block) + - (chunk->size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ)); - - /* Do the realloc */ - chksize = MAXALIGN(size); - blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ; - block = (AllocBlock) realloc(block, blksize); - if (block == NULL) - { - MemoryContextStats(TopMemoryContext); - ereport(ERROR, - (errcode(ERRCODE_OUT_OF_MEMORY), - errmsg("out of memory"), - errdetail("Failed on request of size %zu.", size))); - } - block->freeptr = block->endptr = ((char *) block) + blksize; - - /* Update pointers since block has likely been moved */ - chunk = (AllocChunk) (((char *) block) + ALLOC_BLOCKHDRSZ); - pointer = AllocChunkGetPointer(chunk); - if (prevblock == NULL) - set->blocks = block; - else - prevblock->next = block; - chunk->size = chksize; - -#ifdef MEMORY_CONTEXT_CHECKING -#ifdef RANDOMIZE_ALLOCATED_MEMORY - /* We can only fill the extra space if we know the prior request */ - randomize_mem((char *) pointer + chunk->requested_size, - size - chunk->requested_size); -#endif - - /* - * realloc() (or randomize_mem()) will have left the newly-allocated - * part UNDEFINED, but we may need to adjust trailing bytes from the - * old allocation. - */ - VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + chunk->requested_size, - oldsize - chunk->requested_size); + /* transfer existing data (certain to fit) */ + memcpy(newPointer, pointer, oldsize); - chunk->requested_size = size; - VALGRIND_MAKE_MEM_NOACCESS(&chunk->requested_size, - sizeof(chunk->requested_size)); + AllocSetFree(context, pointer); - /* set mark to catch clobber of "unused" space */ - if (size < chunk->size) - set_sentinel(AllocChunkGetPointer(chunk), size); -#else /* !MEMORY_CONTEXT_CHECKING */ - - /* - * We don't know how much of the old chunk size was the actual - * allocation; it could have been as small as one byte. We have to be - * conservative and just mark the entire old portion DEFINED. - */ - VALGRIND_MAKE_MEM_DEFINED(pointer, oldsize); -#endif - - /* Make any trailing alignment padding NOACCESS. */ - VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size, chksize - size); - return AllocChunkGetPointer(chunk); - } - else - { - /* - * Small-chunk case. We just do this by brute force, ie, allocate a - * new chunk and copy the data. Since we know the existing data isn't - * huge, this won't involve any great memcpy expense, so it's not - * worth being smarter. (At one time we tried to avoid memcpy when it - * was possible to enlarge the chunk in-place, but that turns out to - * misbehave unpleasantly for repeated cycles of - * palloc/repalloc/pfree: the eventually freed chunks go into the - * wrong freelist for the next initial palloc request, and so we leak - * memory indefinitely. See pgsql-hackers archives for 2007-08-11.) - */ - AllocPointer newPointer; - - /* allocate new chunk */ - newPointer = AllocSetAlloc((MemoryContext) set, size); - - /* - * AllocSetAlloc() just made the region NOACCESS. Change it to - * UNDEFINED for the moment; memcpy() will then transfer definedness - * from the old allocation to the new. If we know the old allocation, - * copy just that much. Otherwise, make the entire old chunk defined - * to avoid errors as we copy the currently-NOACCESS trailing bytes. - */ - VALGRIND_MAKE_MEM_UNDEFINED(newPointer, size); -#ifdef MEMORY_CONTEXT_CHECKING - oldsize = chunk->requested_size; -#else - VALGRIND_MAKE_MEM_DEFINED(pointer, oldsize); -#endif - - /* transfer existing data (certain to fit) */ - memcpy(newPointer, pointer, oldsize); - - /* free old chunk */ - AllocSetFree((MemoryContext) set, pointer); - - return newPointer; - } + return newPointer; } /* @@ -1241,39 +629,27 @@ AllocSetIsEmpty(MemoryContext context) static void AllocSetStats(MemoryContext context, int level) { + int i; AllocSet set = (AllocSet) context; - Size nblocks = 0; Size nchunks = 0; Size totalspace = 0; - Size freespace = 0; - AllocBlock block; + Size headers = 0; AllocChunk chunk; - int fidx; - int i; - for (block = set->blocks; block != NULL; block = block->next) + for (chunk = set->chunks; chunk != NULL; chunk = chunk->next) { - nblocks++; - totalspace += block->endptr - ((char *) block); - freespace += block->endptr - block->freeptr; - } - for (fidx = 0; fidx < ALLOCSET_NUM_FREELISTS; fidx++) - { - for (chunk = set->freelist[fidx]; chunk != NULL; - chunk = (AllocChunk) chunk->aset) - { - nchunks++; - freespace += chunk->size + ALLOC_CHUNKHDRSZ; - } + nchunks++; + totalspace += chunk->size + ALLOC_CHUNKHDRSZ; } + headers = nchunks * ALLOC_CHUNKHDRSZ; + for (i = 0; i < level; i++) fprintf(stderr, " "); fprintf(stderr, - "%s: %zu total in %zd blocks; %zu free (%zd chunks); %zu used\n", - set->header.name, totalspace, nblocks, freespace, nchunks, - totalspace - freespace); + "%s: %zu total in %zd chunks (%zd in headers)\n", + set->header.name, totalspace, nchunks, headers); } @@ -1290,87 +666,7 @@ AllocSetStats(MemoryContext context, int level) static void AllocSetCheck(MemoryContext context) { - AllocSet set = (AllocSet) context; - char *name = set->header.name; - AllocBlock block; - - for (block = set->blocks; block != NULL; block = block->next) - { - char *bpoz = ((char *) block) + ALLOC_BLOCKHDRSZ; - long blk_used = block->freeptr - bpoz; - long blk_data = 0; - long nchunks = 0; - - /* - * Empty block - empty can be keeper-block only - */ - if (!blk_used) - { - if (set->keeper != block) - elog(WARNING, "problem in alloc set %s: empty block %p", - name, block); - } - - /* - * Chunk walker - */ - while (bpoz < block->freeptr) - { - AllocChunk chunk = (AllocChunk) bpoz; - Size chsize, - dsize; - - chsize = chunk->size; /* aligned chunk size */ - VALGRIND_MAKE_MEM_DEFINED(&chunk->requested_size, - sizeof(chunk->requested_size)); - dsize = chunk->requested_size; /* real data */ - if (dsize > 0) /* not on a free list */ - VALGRIND_MAKE_MEM_NOACCESS(&chunk->requested_size, - sizeof(chunk->requested_size)); - - /* - * Check chunk size - */ - if (dsize > chsize) - elog(WARNING, "problem in alloc set %s: req size > alloc size for chunk %p in block %p", - name, chunk, block); - if (chsize < (1 << ALLOC_MINBITS)) - elog(WARNING, "problem in alloc set %s: bad size %zu for chunk %p in block %p", - name, chsize, chunk, block); - - /* single-chunk block? */ - if (chsize > set->allocChunkLimit && - chsize + ALLOC_CHUNKHDRSZ != blk_used) - elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p", - name, chunk, block); - - /* - * If chunk is allocated, check for correct aset pointer. (If it's - * free, the aset is the freelist pointer, which we can't check as - * easily...) - */ - if (dsize > 0 && chunk->aset != (void *) set) - elog(WARNING, "problem in alloc set %s: bogus aset link in block %p, chunk %p", - name, block, chunk); - - /* - * Check for overwrite of "unallocated" space in chunk - */ - if (dsize > 0 && dsize < chsize && - !sentinel_ok(chunk, ALLOC_CHUNKHDRSZ + dsize)) - elog(WARNING, "problem in alloc set %s: detected write past chunk end in block %p, chunk %p", - name, block, chunk); - - blk_data += chsize; - nchunks++; - - bpoz += ALLOC_CHUNKHDRSZ + chsize; - } - if ((blk_data + (nchunks * ALLOC_CHUNKHDRSZ)) != blk_used) - elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p", - name, block); - } } #endif /* MEMORY_CONTEXT_CHECKING */