From ea9b8328b499cb692227585882bfa868589d0625 Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 23:00:19 -0500 Subject: [PATCH 1/2] perf: resume cache_skip_pinned where the last scan from the same object stopped --- libCacheSim/cache/cache.c | 63 ++++++++++++++++++++++ libCacheSim/include/libCacheSim/cacheObj.h | 37 ++++++++++--- 2 files changed, 92 insertions(+), 8 deletions(-) diff --git a/libCacheSim/cache/cache.c b/libCacheSim/cache/cache.c index 02664495..e782625d 100644 --- a/libCacheSim/cache/cache.c +++ b/libCacheSim/cache/cache.c @@ -373,9 +373,71 @@ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) { __thread cache_pin_pred_f g_cache_pin_pred = NULL; __thread void *g_cache_pin_ctx = NULL; +/* Where cache_skip_pinned_scan stopped, per starting object: `from` and every + * object from it to `last` (walking towards the head) are pinned. A few + * entries, because a composite policy scans several queues (S3FIFO's small, + * main and ghost queues, SLRU's segments) in turn, each from its own tail. + * + * An entry stays correct while that run of pinned objects stays in place: + * objects are only inserted at a queue's ends, never inside the run, and the + * pin set does not change while a predicate is installed. What takes `from` or + * `last` out of its place drops the entry through cache_pin_scan_forget: the + * list helpers that unlink or move an object (remove_obj_from_list, + * move_obj_to_head, move_obj_to_tail) and cache_remove_obj_base, which every + * eviction and removal that frees an object goes through. An object strictly + * between them may leave freely, since the rest of the run stays linked. The + * policies that unlink or free objects by hand (ARC, CAR, ClockPro, LeCaR, + * MRU, CR_LFU) keep their own lists and never call cache_skip_pinned. */ +#define PIN_SCAN_MEMO_SIZE 8 +static __thread struct { + cache_obj_t *from; + cache_obj_t *last; +} pin_scan_memo[PIN_SCAN_MEMO_SIZE]; +static __thread int pin_scan_memo_next = 0; + void cache_set_pin_predicate(cache_pin_pred_f pred, void *ctx) { g_cache_pin_pred = pred; g_cache_pin_ctx = ctx; + memset(pin_scan_memo, 0, sizeof(pin_scan_memo)); + pin_scan_memo_next = 0; +} + +void cache_pin_scan_forget(const cache_obj_t *obj) { + for (int i = 0; i < PIN_SCAN_MEMO_SIZE; i++) { + if (pin_scan_memo[i].from == obj || pin_scan_memo[i].last == obj) { + pin_scan_memo[i].from = NULL; + pin_scan_memo[i].last = NULL; + } + } +} + +cache_obj_t *cache_skip_pinned_scan(cache_obj_t *candidate) { + if (candidate == NULL || !cache_obj_is_pinned(candidate)) return candidate; + + int slot = -1; + for (int i = 0; i < PIN_SCAN_MEMO_SIZE; i++) { + if (pin_scan_memo[i].from == candidate) { + slot = i; + break; + } + } + cache_obj_t *last = slot >= 0 ? pin_scan_memo[slot].last : candidate; + cache_obj_t *obj = last->queue.prev; + while (obj != NULL && cache_obj_is_pinned(obj)) { + last = obj; + obj = obj->queue.prev; + } + + if (slot < 0) { + slot = pin_scan_memo_next; + pin_scan_memo_next = (pin_scan_memo_next + 1) % PIN_SCAN_MEMO_SIZE; + } + pin_scan_memo[slot].from = candidate; + pin_scan_memo[slot].last = last; + // Everything up to the head is pinned. The entry is kept: a caller that + // falls back to another queue (SLRU) finds this one full again in O(1), and + // one that evicts or moves `candidate` drops the entry while doing so. + return obj != NULL ? obj : candidate; } /** @@ -419,6 +481,7 @@ void cache_evict_base(cache_t *cache, cache_obj_t *obj, void cache_remove_obj_base(cache_t *cache, cache_obj_t *obj, bool remove_from_hashtable) { DEBUG_ASSERT(cache->occupied_byte >= obj->obj_size + cache->obj_md_size); + cache_pin_scan_obj_leaves(obj); cache->occupied_byte -= (obj->obj_size + cache->obj_md_size); cache->n_obj -= 1; if (remove_from_hashtable) { diff --git a/libCacheSim/include/libCacheSim/cacheObj.h b/libCacheSim/include/libCacheSim/cacheObj.h index 95220588..bc625f36 100644 --- a/libCacheSim/include/libCacheSim/cacheObj.h +++ b/libCacheSim/include/libCacheSim/cacheObj.h @@ -273,6 +273,18 @@ static inline cache_obj_t *create_cache_obj_from_request(const request_t *req) { return cache_obj; } +/* Eviction pinning is described below ("eviction pinning"). These are declared + * here so that the list helpers can tell cache_skip_pinned when an object it + * remembers leaves its place in a list. */ +typedef bool (*cache_pin_pred_f)(void *ctx, obj_id_t obj_id); +extern __thread cache_pin_pred_f g_cache_pin_pred; +void cache_pin_scan_forget(const cache_obj_t *obj); + +/** Call before unlinking, moving or freeing an object. */ +static inline void cache_pin_scan_obj_leaves(const cache_obj_t *obj) { + if (unlikely(g_cache_pin_pred != NULL)) cache_pin_scan_forget(obj); +} + /** remove the object from the built-in doubly linked list * * @param head @@ -281,6 +293,7 @@ static inline cache_obj_t *create_cache_obj_from_request(const request_t *req) { */ static inline void remove_obj_from_list(cache_obj_t **head, cache_obj_t **tail, cache_obj_t *cache_obj) { + cache_pin_scan_obj_leaves(cache_obj); if (head != NULL && cache_obj == *head) { *head = cache_obj->queue.next; if (cache_obj->queue.next != NULL) cache_obj->queue.next->queue.prev = NULL; @@ -308,6 +321,7 @@ static inline void remove_obj_from_list(cache_obj_t **head, cache_obj_t **tail, */ static inline void move_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, cache_obj_t *cache_obj) { + cache_pin_scan_obj_leaves(cache_obj); if (*head == *tail) { // the list only has one element DEBUG_ASSERT(cache_obj == *head); @@ -355,6 +369,7 @@ static inline void move_obj_to_tail(cache_obj_t **head, cache_obj_t **tail, static inline void move_obj_to_head(cache_obj_t **head, cache_obj_t **tail, cache_obj_t *cache_obj) { DEBUG_ASSERT(head != NULL); + cache_pin_scan_obj_leaves(cache_obj); if (tail != NULL && *head == *tail) { // the list only has one element @@ -497,14 +512,14 @@ static inline cache_obj_t *prev_obj_in_slist(cache_obj_t *head, * each other's pins. Two caches driven from one thread share it, so set it * only around the eviction it is meant for, as prefixsim does. */ -typedef bool (*cache_pin_pred_f)(void *ctx, obj_id_t obj_id); - -/** Install (pred=NULL to remove) the calling thread's pin predicate. +/** Install (pred=NULL to remove) the calling thread's pin predicate. The + * predicate must not change its answer for any object while it is installed; + * install it again (with the same arguments, if need be) after it does. */ void cache_set_pin_predicate(cache_pin_pred_f pred, void *ctx); -/* Definitions live in cache.c; declared here so the check can inline. */ -extern __thread cache_pin_pred_f g_cache_pin_pred; +/* Definitions live in cache.c; declared here so the check can inline. + * g_cache_pin_pred and cache_pin_pred_f are declared above the list helpers. */ extern __thread void *g_cache_pin_ctx; /** Whether any pinning is in force. Lets callers keep a pin-free fast path. */ @@ -531,12 +546,18 @@ static inline bool cache_obj_is_pinned(const cache_obj_t *obj) { * refused to evict would hang cache_get_base()'s "evict until there is room" * loop. A pinned block evicted because its whole queue was pinned still shows * up in prefixsim's n_self_eviction, so the residue stays measurable. + * + * The scan resumes where the last one from the same `candidate` stopped. A + * queue's victim end does not move while its pinned objects stay put, so the + * evictions that make room for one request would otherwise each walk the same + * pinned objects again, which is quadratic in the request's size. See + * cache_skip_pinned_scan in cache.c for when a remembered position is dropped. */ +cache_obj_t *cache_skip_pinned_scan(cache_obj_t *candidate); + static inline cache_obj_t *cache_skip_pinned(cache_obj_t *candidate) { if (likely(g_cache_pin_pred == NULL)) return candidate; - cache_obj_t *obj = candidate; - while (obj != NULL && cache_obj_is_pinned(obj)) obj = obj->queue.prev; - return obj != NULL ? obj : candidate; + return cache_skip_pinned_scan(candidate); } static inline void free_cache_obj(cache_obj_t *cache_obj) { From 3a8904b74e8a5ea10b59b11ca4b6ef90b0f0d156 Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 23:00:19 -0500 Subject: [PATCH 2/2] test: check resumed pin scans against scans from scratch --- test/CMakeLists.txt | 4 ++ test/test_pinScan.c | 164 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 test/test_pinScan.c diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2379739e..ed9137e6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -58,6 +58,9 @@ add_test_executable(testLcpReader test_lcpReader.cpp) # the prefix tree behind the partial-node policies add_test_executable(testPrefixRadixTree test_prefixRadixTree.cpp) +# cache_skip_pinned's resumed scans against scans from scratch +add_test_executable(testPinScan test_pinScan.c) + # Create C++ test executable (mrcProfiler test) # add_test_executable(testMrcProfiler test_mrcProfiler.cpp) # target_link_libraries(testMrcProfiler mrcProfiler_lib) @@ -77,6 +80,7 @@ add_test(NAME testPrefetchAlgo COMMAND testPrefetchAlgo WORKING_DIRECTORY .) add_test(NAME testDataStructure COMMAND testDataStructure WORKING_DIRECTORY .) add_test(NAME testLcpReader COMMAND testLcpReader WORKING_DIRECTORY .) add_test(NAME testPrefixRadixTree COMMAND testPrefixRadixTree WORKING_DIRECTORY .) +add_test(NAME testPinScan COMMAND testPinScan WORKING_DIRECTORY .) add_test(NAME testUtils COMMAND testUtils WORKING_DIRECTORY .) find_package(Python3 COMPONENTS Interpreter) diff --git a/test/test_pinScan.c b/test/test_pinScan.c new file mode 100644 index 00000000..1b11cdba --- /dev/null +++ b/test/test_pinScan.c @@ -0,0 +1,164 @@ +// cache_skip_pinned resumes each scan where the last one from the same object +// stopped (cache_skip_pinned_scan in cache/cache.c). These checks drive it on +// bare lists, through the list helpers the eviction algorithms use, and compare +// every answer with a scan from scratch while objects are evicted, moved, +// removed and inserted around the remembered positions. + +#include +#include +#include +#include + +#include "libCacheSim.h" + +#define N_LISTS 3 +#define MAX_LIST_LEN 200 +#define MAX_ID 4000000 + +static bool pinned[MAX_ID]; + +static bool is_pinned(void *ctx, obj_id_t id) { + (void)ctx; + return pinned[id]; +} + +typedef struct { + cache_obj_t *head; + cache_obj_t *tail; + int64_t n; +} list_t; + +static list_t lists[N_LISTS]; +static obj_id_t next_id; +static uint64_t rng; +static int pin_pct; + +static uint64_t rnd(void) { // xorshift64 + rng ^= rng << 13; + rng ^= rng >> 7; + rng ^= rng << 17; + return rng; +} + +static cache_obj_t *scan_from_scratch(cache_obj_t *candidate) { + cache_obj_t *obj = candidate; + while (obj != NULL && pinned[obj->obj_id]) obj = obj->queue.prev; + return obj != NULL ? obj : candidate; +} + +static cache_obj_t *new_obj(void) { + cache_obj_t *obj = create_cache_obj_from_request(NULL); + obj->obj_id = next_id++; + pinned[obj->obj_id] = (int)(rnd() % 100) < pin_pct; + return obj; +} + +/** the k-th object from the tail */ +static cache_obj_t *nth(const list_t *l, int64_t k) { + cache_obj_t *obj = l->tail; + while (k-- > 0) obj = obj->queue.prev; + return obj; +} + +static void unlink_and_free(list_t *l, cache_obj_t *obj) { + remove_obj_from_list(&l->head, &l->tail, obj); + free_cache_obj(obj); + l->n -= 1; +} + +/** One random walk of list operations; returns the number of wrong answers. */ +static int64_t run(uint64_t seed, int64_t n_steps) { + rng = seed; + next_id = 1; + pin_pct = 80; + memset(lists, 0, sizeof(lists)); + cache_set_pin_predicate(is_pinned, NULL); + + int64_t n_wrong = 0, n_scans = 0; + for (int64_t step = 0; step < n_steps; step++) { + list_t *l = &lists[rnd() % N_LISTS]; + const int op = (int)(rnd() % 100); + + if (l->n == 0 || (op < 25 && l->n < MAX_LIST_LEN)) { + // insert a new object, mostly at the head as the policies do + cache_obj_t *obj = new_obj(); + if (rnd() % 10 == 0) + append_obj_to_tail(&l->head, &l->tail, obj); + else + prepend_obj_to_head(&l->head, &l->tail, obj); + l->n += 1; + } else if (op < 55) { + // scan from the tail, or from anywhere (a hand) + cache_obj_t *from = rnd() % 5 == 0 ? nth(l, rnd() % l->n) : l->tail; + n_scans += 1; + if (cache_skip_pinned(from) != scan_from_scratch(from)) n_wrong += 1; + } else if (op < 75 || l->n >= MAX_LIST_LEN) { + // evict what the scan returns, a pinned object if the list is all pinned + cache_obj_t *victim = cache_skip_pinned(l->tail); + n_scans += 1; + if (victim != scan_from_scratch(l->tail)) n_wrong += 1; + unlink_and_free(l, victim); + } else if (op < 83) { + move_obj_to_head(&l->head, &l->tail, nth(l, rnd() % l->n)); + } else if (op < 87) { + move_obj_to_tail(&l->head, &l->tail, nth(l, rnd() % l->n)); + } else if (op < 97) { + unlink_and_free(l, nth(l, rnd() % l->n)); + } else { + // change the pin set; the predicate must be installed again after that + pin_pct = (int)(rnd() % 101); + for (int i = 0; i < N_LISTS; i++) { + for (cache_obj_t *obj = lists[i].head; obj != NULL; + obj = obj->queue.next) { + if (rnd() % 4 == 0) pinned[obj->obj_id] = !pinned[obj->obj_id]; + } + } + cache_set_pin_predicate(is_pinned, NULL); + } + if (next_id >= MAX_ID) break; + } + + for (int i = 0; i < N_LISTS; i++) { + while (lists[i].tail != NULL) unlink_and_free(&lists[i], lists[i].tail); + } + cache_set_pin_predicate(NULL, NULL); + printf(" seed %llu: %lld scans, %lld wrong\n", (unsigned long long)seed, + (long long)n_scans, (long long)n_wrong); + return n_wrong; +} + +/** Without a predicate nothing is skipped. */ +static bool no_predicate_skips_nothing(void) { + list_t l = {NULL, NULL, 0}; + next_id = 1; + pin_pct = 100; + for (int i = 0; i < 4; i++) { + prepend_obj_to_head(&l.head, &l.tail, new_obj()); + l.n += 1; + } + const bool ok = cache_skip_pinned(l.tail) == l.tail; + while (l.tail != NULL) unlink_and_free(&l, l.tail); + return ok; +} + +int main(void) { + bool ok = true; + printf("no predicate, nothing skipped\n"); + if (!no_predicate_skips_nothing()) { + printf(" FAIL: an object was skipped with no predicate installed\n"); + ok = false; + } + + printf("resumed scans agree with scans from scratch\n"); + const uint64_t seeds[] = {1, 2, 3, 42, 20261109}; + for (size_t i = 0; i < sizeof(seeds) / sizeof(seeds[0]); i++) { + if (run(seeds[i], 500000) != 0) ok = false; + } + + if (!ok) { + printf("pin scan check FAILED\n"); + return 1; + } + printf("all pin scan checks passed\n"); + return 0; +}