perf: resume cache_skip_pinned where the last scan from the same object stopped

libCacheSim/cache/cache.c

@@ -373,9 +373,71 @@ cache_obj_t *cache_insert_base(cache_t *cache, const request_t *req) {
 __thread cache_pin_pred_f g_cache_pin_pred = NULL;
 __thread void *g_cache_pin_ctx = NULL;
 
+/* Where cache_skip_pinned_scan stopped, per starting object: `from` and every
+ * object from it to `last` (walking towards the head) are pinned. A few
+ * entries, because a composite policy scans several queues (S3FIFO's small,
+ * main and ghost queues, SLRU's segments) in turn, each from its own tail.
+ *
+ * An entry stays correct while that run of pinned objects stays in place:
+ * objects are only inserted at a queue's ends, never inside the run, and the
+ * pin set does not change while a predicate is installed. What takes `from` or
+ * `last` out of its place drops the entry through cache_pin_scan_forget: the
+ * list helpers that unlink or move an object (remove_obj_from_list,
+ * move_obj_to_head, move_obj_to_tail) and cache_remove_obj_base, which every
+ * eviction and removal that frees an object goes through. An object strictly
+ * between them may leave freely, since the rest of the run stays linked. The
+ * policies that unlink or free objects by hand (ARC, CAR, ClockPro, LeCaR,
+ * MRU, CR_LFU) keep their own lists and never call cache_skip_pinned. */
+#define PIN_SCAN_MEMO_SIZE 8
+static __thread struct {
+  cache_obj_t *from;
+  cache_obj_t *last;
+} pin_scan_memo[PIN_SCAN_MEMO_SIZE];
+static __thread int pin_scan_memo_next = 0;
+
 void cache_set_pin_predicate(cache_pin_pred_f pred, void *ctx) {
   g_cache_pin_pred = pred;
   g_cache_pin_ctx = ctx;
+  memset(pin_scan_memo, 0, sizeof(pin_scan_memo));
+  pin_scan_memo_next = 0;
+}
+
+void cache_pin_scan_forget(const cache_obj_t *obj) {
+  for (int i = 0; i < PIN_SCAN_MEMO_SIZE; i++) {
+    if (pin_scan_memo[i].from == obj || pin_scan_memo[i].last == obj) {
+      pin_scan_memo[i].from = NULL;
+      pin_scan_memo[i].last = NULL;
+    }
+  }
+}
+
+cache_obj_t *cache_skip_pinned_scan(cache_obj_t *candidate) {
+  if (candidate == NULL || !cache_obj_is_pinned(candidate)) return candidate;
+
+  int slot = -1;
+  for (int i = 0; i < PIN_SCAN_MEMO_SIZE; i++) {
+    if (pin_scan_memo[i].from == candidate) {
+      slot = i;
+      break;
+    }
+  }
+  cache_obj_t *last = slot >= 0 ? pin_scan_memo[slot].last : candidate;
+  cache_obj_t *obj = last->queue.prev;
+  while (obj != NULL && cache_obj_is_pinned(obj)) {
+    last = obj;
+    obj = obj->queue.prev;
+  }
+
+  if (slot < 0) {
+    slot = pin_scan_memo_next;
+    pin_scan_memo_next = (pin_scan_memo_next + 1) % PIN_SCAN_MEMO_SIZE;
+  }
+  pin_scan_memo[slot].from = candidate;
+  pin_scan_memo[slot].last = last;
+  // Everything up to the head is pinned. The entry is kept: a caller that
+  // falls back to another queue (SLRU) finds this one full again in O(1), and
+  // one that evicts or moves `candidate` drops the entry while doing so.
+  return obj != NULL ? obj : candidate;
 }
 
 /**
@@ -419,6 +481,7 @@ void cache_evict_base(cache_t *cache, cache_obj_t *obj,
 void cache_remove_obj_base(cache_t *cache, cache_obj_t *obj,
                            bool remove_from_hashtable) {
   DEBUG_ASSERT(cache->occupied_byte >= obj->obj_size + cache->obj_md_size);
+  cache_pin_scan_obj_leaves(obj);
   cache->occupied_byte -= (obj->obj_size + cache->obj_md_size);
   cache->n_obj -= 1;
   if (remove_from_hashtable) {

libCacheSim/include/libCacheSim/cacheObj.h

@@ -273,6 +273,18 @@ static inline cache_obj_t *create_cache_obj_from_request(const request_t *req) {
   return cache_obj;
 }
 
+/* Eviction pinning is described below ("eviction pinning"). These are declared
+ * here so that the list helpers can tell cache_skip_pinned when an object it
+ * remembers leaves its place in a list. */
+typedef bool (*cache_pin_pred_f)(void *ctx, obj_id_t obj_id);
+extern __thread cache_pin_pred_f g_cache_pin_pred;
+void cache_pin_scan_forget(const cache_obj_t *obj);
+
+/** Call before unlinking, moving or freeing an object. */
+static inline void cache_pin_scan_obj_leaves(const cache_obj_t *obj) {
+  if (unlikely(g_cache_pin_pred != NULL)) cache_pin_scan_forget(obj);
+}
+
 /** remove the object from the built-in doubly linked list
  *
  * @param head
@@ -281,6 +293,7 @@ static inline cache_obj_t *create_cache_obj_from_request(const request_t *req) {
  */
 static inline void remove_obj_from_list(cache_obj_t **head, cache_obj_t **tail,
                                         cache_obj_t *cache_obj) {
+  cache_pin_scan_obj_leaves(cache_obj);
   if (head != NULL && cache_obj == *head) {
     *head = cache_obj->queue.next;
     if (cache_obj->queue.next != NULL) cache_obj->queue.next->queue.prev = NULL;
@@ -308,6 +321,7 @@ static inline void remove_obj_from_list(cache_obj_t **head, cache_obj_t **tail,
  */
 static inline void move_obj_to_tail(cache_obj_t **head, cache_obj_t **tail,
                                     cache_obj_t *cache_obj) {
+  cache_pin_scan_obj_leaves(cache_obj);
   if (*head == *tail) {
     // the list only has one element
     DEBUG_ASSERT(cache_obj == *head);
@@ -355,6 +369,7 @@ static inline void move_obj_to_tail(cache_obj_t **head, cache_obj_t **tail,
 static inline void move_obj_to_head(cache_obj_t **head, cache_obj_t **tail,
                                     cache_obj_t *cache_obj) {
   DEBUG_ASSERT(head != NULL);
+  cache_pin_scan_obj_leaves(cache_obj);
 
   if (tail != NULL && *head == *tail) {
     // the list only has one element
@@ -497,14 +512,14 @@ static inline cache_obj_t *prev_obj_in_slist(cache_obj_t *head,
  * each other's pins. Two caches driven from one thread share it, so set it
  * only around the eviction it is meant for, as prefixsim does.
  */
-typedef bool (*cache_pin_pred_f)(void *ctx, obj_id_t obj_id);
-
-/** Install (pred=NULL to remove) the calling thread's pin predicate.
+/** Install (pred=NULL to remove) the calling thread's pin predicate. The
+ * predicate must not change its answer for any object while it is installed;
+ * install it again (with the same arguments, if need be) after it does.
  */
 void cache_set_pin_predicate(cache_pin_pred_f pred, void *ctx);
 
-/* Definitions live in cache.c; declared here so the check can inline. */
-extern __thread cache_pin_pred_f g_cache_pin_pred;
+/* Definitions live in cache.c; declared here so the check can inline.
+ * g_cache_pin_pred and cache_pin_pred_f are declared above the list helpers. */
 extern __thread void *g_cache_pin_ctx;
 
 /** Whether any pinning is in force. Lets callers keep a pin-free fast path. */
@@ -531,12 +546,18 @@ static inline bool cache_obj_is_pinned(const cache_obj_t *obj) {
  * refused to evict would hang cache_get_base()'s "evict until there is room"
  * loop. A pinned block evicted because its whole queue was pinned still shows
  * up in prefixsim's n_self_eviction, so the residue stays measurable.
+ *
+ * The scan resumes where the last one from the same `candidate` stopped. A
+ * queue's victim end does not move while its pinned objects stay put, so the
+ * evictions that make room for one request would otherwise each walk the same
+ * pinned objects again, which is quadratic in the request's size. See
+ * cache_skip_pinned_scan in cache.c for when a remembered position is dropped.
  */
+cache_obj_t *cache_skip_pinned_scan(cache_obj_t *candidate);
+
 static inline cache_obj_t *cache_skip_pinned(cache_obj_t *candidate) {
   if (likely(g_cache_pin_pred == NULL)) return candidate;
-  cache_obj_t *obj = candidate;
-  while (obj != NULL && cache_obj_is_pinned(obj)) obj = obj->queue.prev;
-  return obj != NULL ? obj : candidate;
+  return cache_skip_pinned_scan(candidate);
 }
 
 static inline void free_cache_obj(cache_obj_t *cache_obj) {

test/CMakeLists.txt

@@ -58,6 +58,9 @@ add_test_executable(testLcpReader test_lcpReader.cpp)
 # the prefix tree behind the partial-node policies
 add_test_executable(testPrefixRadixTree test_prefixRadixTree.cpp)
 
+# cache_skip_pinned's resumed scans against scans from scratch
+add_test_executable(testPinScan test_pinScan.c)
+
 # Create C++ test executable (mrcProfiler test)
 # add_test_executable(testMrcProfiler test_mrcProfiler.cpp)
 # target_link_libraries(testMrcProfiler mrcProfiler_lib)
@@ -77,6 +80,7 @@ add_test(NAME testPrefetchAlgo COMMAND testPrefetchAlgo WORKING_DIRECTORY .)
 add_test(NAME testDataStructure COMMAND testDataStructure WORKING_DIRECTORY .)
 add_test(NAME testLcpReader COMMAND testLcpReader WORKING_DIRECTORY .)
 add_test(NAME testPrefixRadixTree COMMAND testPrefixRadixTree WORKING_DIRECTORY .)
+add_test(NAME testPinScan COMMAND testPinScan WORKING_DIRECTORY .)
 add_test(NAME testUtils COMMAND testUtils WORKING_DIRECTORY .)
 
 find_package(Python3 COMPONENTS Interpreter)

test/test_pinScan.c

@@ -0,0 +1,164 @@
+// cache_skip_pinned resumes each scan where the last one from the same object
+// stopped (cache_skip_pinned_scan in cache/cache.c). These checks drive it on
+// bare lists, through the list helpers the eviction algorithms use, and compare
+// every answer with a scan from scratch while objects are evicted, moved,
+// removed and inserted around the remembered positions.
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "libCacheSim.h"
+
+#define N_LISTS 3
+#define MAX_LIST_LEN 200
+#define MAX_ID 4000000
+
+static bool pinned[MAX_ID];
+
+static bool is_pinned(void *ctx, obj_id_t id) {
+  (void)ctx;
+  return pinned[id];
+}
+
+typedef struct {
+  cache_obj_t *head;
+  cache_obj_t *tail;
+  int64_t n;
+} list_t;
+
+static list_t lists[N_LISTS];
+static obj_id_t next_id;
+static uint64_t rng;
+static int pin_pct;
+
+static uint64_t rnd(void) {  // xorshift64
+  rng ^= rng << 13;
+  rng ^= rng >> 7;
+  rng ^= rng << 17;
+  return rng;
+}
+
+static cache_obj_t *scan_from_scratch(cache_obj_t *candidate) {
+  cache_obj_t *obj = candidate;
+  while (obj != NULL && pinned[obj->obj_id]) obj = obj->queue.prev;
+  return obj != NULL ? obj : candidate;
+}
+
+static cache_obj_t *new_obj(void) {
+  cache_obj_t *obj = create_cache_obj_from_request(NULL);
+  obj->obj_id = next_id++;
+  pinned[obj->obj_id] = (int)(rnd() % 100) < pin_pct;
+  return obj;
+}
+
+/** the k-th object from the tail */
+static cache_obj_t *nth(const list_t *l, int64_t k) {
+  cache_obj_t *obj = l->tail;
+  while (k-- > 0) obj = obj->queue.prev;
+  return obj;
+}
+
+static void unlink_and_free(list_t *l, cache_obj_t *obj) {
+  remove_obj_from_list(&l->head, &l->tail, obj);
+  free_cache_obj(obj);
+  l->n -= 1;
+}
+
+/** One random walk of list operations; returns the number of wrong answers. */
+static int64_t run(uint64_t seed, int64_t n_steps) {
+  rng = seed;
+  next_id = 1;
+  pin_pct = 80;
+  memset(lists, 0, sizeof(lists));
+  cache_set_pin_predicate(is_pinned, NULL);
+
+  int64_t n_wrong = 0, n_scans = 0;
+  for (int64_t step = 0; step < n_steps; step++) {
+    list_t *l = &lists[rnd() % N_LISTS];
+    const int op = (int)(rnd() % 100);
+
+    if (l->n == 0 || (op < 25 && l->n < MAX_LIST_LEN)) {
+      // insert a new object, mostly at the head as the policies do
+      cache_obj_t *obj = new_obj();
+      if (rnd() % 10 == 0)
+        append_obj_to_tail(&l->head, &l->tail, obj);
+      else
+        prepend_obj_to_head(&l->head, &l->tail, obj);
+      l->n += 1;
+    } else if (op < 55) {
+      // scan from the tail, or from anywhere (a hand)
+      cache_obj_t *from = rnd() % 5 == 0 ? nth(l, rnd() % l->n) : l->tail;
+      n_scans += 1;
+      if (cache_skip_pinned(from) != scan_from_scratch(from)) n_wrong += 1;
+    } else if (op < 75 || l->n >= MAX_LIST_LEN) {
+      // evict what the scan returns, a pinned object if the list is all pinned
+      cache_obj_t *victim = cache_skip_pinned(l->tail);
+      n_scans += 1;
+      if (victim != scan_from_scratch(l->tail)) n_wrong += 1;
+      unlink_and_free(l, victim);
+    } else if (op < 83) {
+      move_obj_to_head(&l->head, &l->tail, nth(l, rnd() % l->n));
+    } else if (op < 87) {
+      move_obj_to_tail(&l->head, &l->tail, nth(l, rnd() % l->n));
+    } else if (op < 97) {
+      unlink_and_free(l, nth(l, rnd() % l->n));
+    } else {
+      // change the pin set; the predicate must be installed again after that
+      pin_pct = (int)(rnd() % 101);
+      for (int i = 0; i < N_LISTS; i++) {
+        for (cache_obj_t *obj = lists[i].head; obj != NULL;
+             obj = obj->queue.next) {
+          if (rnd() % 4 == 0) pinned[obj->obj_id] = !pinned[obj->obj_id];
+        }
+      }
+      cache_set_pin_predicate(is_pinned, NULL);
+    }
+    if (next_id >= MAX_ID) break;
+  }
+
+  for (int i = 0; i < N_LISTS; i++) {
+    while (lists[i].tail != NULL) unlink_and_free(&lists[i], lists[i].tail);
+  }
+  cache_set_pin_predicate(NULL, NULL);
+  printf("  seed %llu: %lld scans, %lld wrong\n", (unsigned long long)seed,
+         (long long)n_scans, (long long)n_wrong);
+  return n_wrong;
+}
+
+/** Without a predicate nothing is skipped. */
+static bool no_predicate_skips_nothing(void) {
+  list_t l = {NULL, NULL, 0};
+  next_id = 1;
+  pin_pct = 100;
+  for (int i = 0; i < 4; i++) {
+    prepend_obj_to_head(&l.head, &l.tail, new_obj());
+    l.n += 1;
+  }
+  const bool ok = cache_skip_pinned(l.tail) == l.tail;
+  while (l.tail != NULL) unlink_and_free(&l, l.tail);
+  return ok;
+}
+
+int main(void) {
+  bool ok = true;
+  printf("no predicate, nothing skipped\n");
+  if (!no_predicate_skips_nothing()) {
+    printf("  FAIL: an object was skipped with no predicate installed\n");
+    ok = false;
+  }
+
+  printf("resumed scans agree with scans from scratch\n");
+  const uint64_t seeds[] = {1, 2, 3, 42, 20261109};
+  for (size_t i = 0; i < sizeof(seeds) / sizeof(seeds[0]); i++) {
+    if (run(seeds[i], 500000) != 0) ok = false;
+  }
+
+  if (!ok) {
+    printf("pin scan check FAILED\n");
+    return 1;
+  }
+  printf("all pin scan checks passed\n");
+  return 0;
+}