allocator: Implement support for allocation cache

README.md

@@ -978,6 +978,16 @@ meshopt_setAllocator(malloc, free);
 
 > Note that the library expects the allocation function to either throw in case of out-of-memory (in which case the exception will propagate to the caller) or abort, so technically the use of `malloc` above isn't safe. If you want to handle out-of-memory errors without using C++ exceptions, you can use `setjmp`/`longjmp` instead.
 
+For applications that end up using many allocations, such as when processing many small meshes or individual clusters as part of `clusterlod.h`, it may be beneficial to set up the allocator cache:
+
+```c++
+meshopt_setAllocatorCache(16, 32 << 20); // 16 threads x 32MB per thread
+```
+
+This will allocate a pool of memory that will be used for subsequent temporary allocations; this is particularly helpful for allocators that don't reuse larger allocations, or use global locks to manage heap memory, such as the default Windows heap. After processing is finished, `meshopt_setAllocatorCache(0, 0)` frees the cache; while the cache is active, the allocation callbacks can't be overridden, but if the cache is configured after `meshopt_setAllocator`, the cache itself and any allocations that don't fit will use the callbacks too.
+
+> Note that the combination of allocator cache and `setjmp`/`longjmp` allocation error handling is not supported.
+
 When building meshoptimizer as a shared library, allocations from the templated index wrappers provided in the header (used when index data is not `unsigned int`) will only be redirected to these callbacks if the library is built with `MESHOPTIMIZER_ALLOC_EXPORT` defined.
 
 When building meshoptimizer with MSVC, if a non-default calling convention is used such as `__vectorcall` (via `/Gv`), `MESHOPTIMIZER_ALLOC_CALLCONV` should be defined to match the calling convention of `operator new`/`delete`.
@@ -1001,6 +1011,7 @@ Currently, the following APIs are experimental:
 - `meshopt_SimplifyPreserveFolds` and `meshopt_SimplifyErrorClamped` flags for `meshopt_simplify*` functions
 - `meshopt_generateNormals` function
 - `meshopt_remesh` function and `meshopt_Remesh*` flags
+- `meshopt_setAllocatorCache` function
 
 ## License
 

demo/nanite.cpp

@@ -140,6 +140,11 @@ void nanite(const std::vector<Vertex>& vertices, const std::vector<unsigned int>
 	float znear = 1e-2f;
 	float proj = 1.f / tanf(fovy * 3.1415926f / 180.f * 0.5f);
 
+	// this is optional, but it can noticeably reduce page faults on windows for large meshes
+	// for multi-threaded applications, set the first parameter to the expected concurrency; here we just need one thread
+	// in practice, you'd do this *once* before processing all meshes of course, not per mesh; the call is not thread-safe
+	meshopt_setAllocatorCache(1, 32 << 20);
+
 	clodBuild(config, mesh, [&](clodGroup group, const clodCluster* clusters, size_t cluster_count) -> int { // clang-format!
 		if (stats.size() <= size_t(group.depth))
 			stats.push_back({});
@@ -185,6 +190,8 @@ void nanite(const std::vector<Vertex>& vertices, const std::vector<unsigned int>
 		return int(groups.size() - 1);
 	});
 
+	meshopt_setAllocatorCache(0, 0);
+
 	// for cluster connectivity analysis and boundary statistics, we need a position-only remap that maps vertices with the same position to the same index
 	std::vector<unsigned int> remap(vertices.size());
 	meshopt_generatePositionRemap(&remap[0], &vertices[0].px, vertices.size(), sizeof(Vertex));

demo/tests.cpp

@@ -1761,6 +1761,29 @@ static void customAllocator()
 	allocCount = freeCount = 0;
 }
 
+static void customAllocatorCache()
+{
+	meshopt_setAllocator(customAlloc, customFree);
+
+	// cache memory is allocated using the callbacks
+	meshopt_setAllocatorCache(2, 48);
+	assert(allocCount == 1 && freeCount == 0);
+
+	float vb[] = {1, 0, 0, 0, 1, 0, 0, 0, 1};
+	unsigned int ib[] = {0, 1, 2};
+
+	// in-place vertex copy (36 bytes) fits into the block, remap table (12 bytes) doesn't fit after it
+	meshopt_optimizeVertexFetch(vb, ib, 3, vb, 3, 12);
+	assert(allocCount == 2 && freeCount == 1);
+
+	meshopt_setAllocatorCache(0, 0);
+	assert(allocCount == 2 && freeCount == 2);
+
+	meshopt_setAllocator(operator new, operator delete);
+
+	allocCount = freeCount = 0;
+}
+
 static void emptyMesh()
 {
 	meshopt_optimizeVertexCache(NULL, NULL, 0, 0);
@@ -3652,6 +3675,7 @@ void runTests()
 	remapCustom();
 
 	customAllocator();
+	customAllocatorCache();
 
 	emptyMesh();
 

src/allocator.cpp

@@ -2,6 +2,122 @@
 #include "meshoptimizer.h"
 
 #include <assert.h>
+#include <stdint.h>
+
+// allocator cache requires std::atomic and thread_local for thread safety; if these are not available, meshopt_setAllocatorCache is not supported
+#if !defined(MESHOPTIMIZER_ALLOC_NOCACHE) && __cplusplus < 201103L && !(defined(_MSC_VER) && _MSC_VER >= 1900)
+#define MESHOPTIMIZER_ALLOC_NOCACHE
+#endif
+
+#ifndef MESHOPTIMIZER_ALLOC_NOCACHE
+#include <atomic>
+#endif
+
+#ifndef MESHOPTIMIZER_ALLOC_NOCACHE
+namespace meshopt
+{
+
+struct GlobalCache
+{
+	std::atomic<uint64_t> blocks;
+	char padding[120]; // avoid false sharing between threads
+
+	void* data;
+	size_t data_size;
+	size_t block_size;
+	uint64_t all_blocks;
+	meshopt_Allocator::Storage fallback;
+};
+
+struct LocalCache
+{
+	void* block;
+	size_t offset;
+	uint64_t block_mask;
+};
+
+alignas(128) static GlobalCache gCache = {};
+thread_local static LocalCache gCacheLocal;
+
+static int cacheIndex(uint64_t mask)
+{
+	int index = -1;
+	while (mask)
+		mask >>= 1, index++;
+	return index;
+}
+
+static void* MESHOPTIMIZER_ALLOC_CALLCONV cacheAllocate(size_t size)
+{
+	GlobalCache& global = gCache;
+	LocalCache& local = gCacheLocal;
+
+	// try to grab an available local block
+	if (local.block == NULL && size < global.block_size && global.blocks.load(std::memory_order_relaxed) != 0)
+	{
+		uint64_t blocks = global.blocks.load(std::memory_order_relaxed);
+		uint64_t mask = 0;
+
+		do
+		{
+			// prefer last index for coherency, but settle for lowest bit otherwise
+			mask = (blocks & local.block_mask) ? local.block_mask : blocks & (0 - blocks);
+			// no available block, unlikely to get one soon
+			if (blocks == 0)
+				break;
+			// reloads blocks on failure
+		} while (!global.blocks.compare_exchange_weak(blocks, blocks & ~mask, std::memory_order_acquire, std::memory_order_relaxed));
+
+		if (mask)
+		{
+			// extract block index from mask (must only have one bit set)
+			int index = cacheIndex(mask);
+			assert(mask == (1ull << index));
+
+			local.block = static_cast<char*>(global.data) + index * global.block_size;
+			local.block_mask = mask;
+		}
+	}
+
+	// allocate from local block if any
+	if (local.block && size < global.block_size && local.offset < global.block_size - size)
+	{
+		void* ptr = static_cast<char*>(local.block) + local.offset;
+		local.offset += size ? size : 1;
+		local.offset = (local.offset + 15) & ~size_t(15); // align future allocations to 16b
+		return ptr;
+	}
+
+	return global.fallback.allocate(size);
+}
+
+static void MESHOPTIMIZER_ALLOC_CALLCONV cacheDeallocate(void* ptr)
+{
+	GlobalCache& global = gCache;
+	LocalCache& local = gCacheLocal;
+
+	// has our allocation come from the cache?
+	if (global.data && ptr >= global.data && ptr < static_cast<char*>(global.data) + global.data_size)
+	{
+		// meshopt allocations are guaranteed to be stack ordered and thread local
+		assert(local.block && ptr >= local.block && ptr < static_cast<char*>(local.block) + local.offset);
+		local.offset = static_cast<char*>(ptr) - static_cast<char*>(local.block);
+
+		// return local block to the pool
+		if (local.offset == 0)
+		{
+			assert(local.block_mask);
+			global.blocks.fetch_or(local.block_mask, std::memory_order_release);
+			local.block = NULL;
+			// keep block_mask as an affinity hint for the next allocation
+		}
+	}
+	else
+		global.fallback.deallocate(ptr);
+}
+
+} // namespace meshopt
+#endif
 
 #ifdef MESHOPTIMIZER_ALLOC_EXPORT
 meshopt_Allocator::Storage& meshopt_Allocator::storage()
@@ -15,7 +131,62 @@ void meshopt_setAllocator(void* (MESHOPTIMIZER_ALLOC_CALLCONV* allocate)(size_t)
 {
 	assert(allocate && deallocate);
 
+#ifndef MESHOPTIMIZER_ALLOC_NOCACHE
+	assert(!meshopt::gCache.data); // changing allocation callbacks is prohibited if the cache is already set up
+#endif
+
 	meshopt_Allocator::Storage& s = meshopt_Allocator::storage();
 	s.allocate = allocate;
 	s.deallocate = deallocate;
 }
+
+#ifndef MESHOPTIMIZER_ALLOC_NOCACHE
+void meshopt_setAllocatorCache(size_t block_count, size_t block_size)
+{
+	using namespace meshopt;
+
+	assert(block_count <= 64);
+	assert(block_count == 0 || block_size >= 16);
+
+	meshopt_Allocator::Storage& allocator = meshopt_Allocator::storage();
+
+	// reset prior global state
+	// note: all previously allocated blocks must have been returned at this point; this is guaranteed by the absence of concurrent execution with meshopt_ functions
+	if (gCache.data)
+	{
+		assert(gCache.blocks.load() == gCache.all_blocks);
+		gCache.fallback.deallocate(gCache.data);
+		gCache.data = NULL;
+		gCache.data_size = 0;
+		gCache.block_size = 0;
+		gCache.all_blocks = 0;
+		gCache.blocks.store(0);
+
+		// reset global allocator
+		allocator = gCache.fallback;
+	}
+
+	// individual allocations are aligned to 16 bytes, so we only use a 16-byte-aligned subset
+	block_size &= ~size_t(15);
+
+	// setup the cache if requested
+	if (block_count && block_size)
+	{
+		size_t data_size = block_count > size_t(-1) / block_size ? size_t(-1) : block_count * block_size;
+
+		// allocate a block for each thread and mark each block as available
+		gCache.data = allocator.allocate(data_size);
+		if (!gCache.data)
+			return;
+
+		gCache.data_size = data_size;
+		gCache.block_size = block_size;
+		gCache.all_blocks = (block_count >= 64) ? ~0ull : (1ull << block_count) - 1;
+		gCache.blocks.store(gCache.all_blocks);
+		gCache.fallback = allocator;
+
+		allocator.allocate = cacheAllocate;
+		allocator.deallocate = cacheDeallocate;
+	}
+}
+#endif

src/meshoptimizer.h

@@ -1043,6 +1043,17 @@ MESHOPTIMIZER_API int meshopt_computePositionExponent(const float* minv, const f
  */
 MESHOPTIMIZER_API void meshopt_setAllocator(void* (MESHOPTIMIZER_ALLOC_CALLCONV* allocate)(size_t), void (MESHOPTIMIZER_ALLOC_CALLCONV* deallocate)(void*));
 
+/**
+ * Experimental: Enable allocation cache
+ * Allocates a fixed number of memory blocks to be used for temporary allocations in the library.
+ * Each thread performing temporary allocations can only use one block at a time; block count should be set according to expected concurrency.
+ * Once cache is setup, using meshopt_setAllocator is prohibited; cache needs to be disabled (using block_count=0) before reconfiguring allocation callbacks.
+ * This function is not thread safe and must not be called concurrently with any other meshopt_ function.
+ *
+ * block_count can be at most 64; when set to 0, the allocation cache is disabled.
+ */
+MESHOPTIMIZER_EXPERIMENTAL void meshopt_setAllocatorCache(size_t block_count, size_t block_size);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif