add support for might-exist lookups (#660)

* add support for might-exist lookups.  Such lookups are faster---they check only the memtables and filters---but may have false positives.
* fix suspiciously old-looking memory leak

---------

Signed-off-by: Rob Johnson <rob@robjohnson.io>

Author: rtjohnso

.gitignore

@@ -14,3 +14,7 @@ tags
 unit_tests_db
 compile_flags.txt
 build
+\#*\#
+compile_commands.json
+*~
+.cache/

examples/splinterdb_intro_example.c

@@ -72,7 +72,8 @@ main()
 
    // Retrieve a key-value pair.
    splinterdb_lookup_result result;
-   splinterdb_lookup_result_init(spl_handle, &result, 0, NULL);
+   splinterdb_lookup_result_init(
+      spl_handle, &result, SPLINTERDB_LOOKUP_VALUE, 0, NULL);
 
    fruit = "Orange";
    key   = slice_create((size_t)strlen(fruit), fruit);

examples/splinterdb_wide_values_example.c

@@ -75,7 +75,8 @@ main()
    char outbuf[USER_AVG_VALUE_SIZE];
 
    splinterdb_lookup_result result;
-   splinterdb_lookup_result_init(spl_handle, &result, sizeof(outbuf), outbuf);
+   splinterdb_lookup_result_init(
+      spl_handle, &result, SPLINTERDB_LOOKUP_VALUE, sizeof(outbuf), outbuf);
 
    printf("Retrieve values of different lengths using output buffer of"
           " fixed size=%lu bytes:\n",

include/splinterdb/splinterdb.h

@@ -174,24 +174,28 @@ splinterdb_open(const splinterdb_config *cfg, splinterdb **kvs);
 void
 splinterdb_close(splinterdb **kvs);
 
-// Insert a key and value.
-// Relies on data_config->encode_message
+// Insert a key and value.  Overwrites any previous value associated with the
+// key.
 int
 splinterdb_insert(splinterdb *kvsb, slice key, slice value);
 
 // Delete a given key and any associated value / messages
 int
 splinterdb_delete(splinterdb *kvsb, slice key);
 
-// Insert a key and value.
-// Relies on data_config->encode_message
+// Update the value associated with key.
 int
 splinterdb_update(splinterdb *kvsb, slice key, slice delta);
 
 // Lookups
 
+typedef uint64 splinterdb_lookup_flags;
+
+#define SPLINTERDB_LOOKUP_VALUE       ((splinterdb_lookup_flags)0)
+#define SPLINTERDB_LOOKUP_MIGHT_EXIST ((splinterdb_lookup_flags)1 << 0)
+
 // Size of opaque data required to hold a lookup result
-#define SPLINTERDB_LOOKUP_BUFSIZE (18 * sizeof(void *))
+#define SPLINTERDB_LOOKUP_BUFSIZE (20 * sizeof(void *))
 
 // A lookup result is stored and parsed from here
 //
@@ -203,6 +207,12 @@ typedef struct {
 
 // Initialize a lookup result object.
 //
+// flags may be SPLINTERDB_LOOKUP_VALUE or SPLINTERDB_LOOKUP_MIGHT_EXIST.
+//
+// When SPLINTERDB_LOOKUP_MIGHT_EXIST is set, splinterdb_lookup_found() reports
+// true if the key might exist, and false if the key definitely does not exist.
+// splinterdb_lookup_result_value() will return EINVAL for such results.
+//
 // If buffer is NULL, then the library will allocate and manage memory.
 //
 // If the caller provides a buffer, that will be used, unless a lookup
@@ -212,15 +222,16 @@ typedef struct {
 // After this function returns, the caller must ensure that
 // 1. *result is only used in conjunction with the kvs
 //    Attempting to use one lookup_result with multiple instances of splinterdb
-//    may cause problems in future versions of splinterdb
+//    will cause problems
 // 2. The lifetime of *result must not exceed the lifetime of kvs
 //    The result should be deinit'ed before calling splinterdb_close on kvs
 //
-// While the current version of SplinterDB does not rely on these rules, future
+// While the current version of SplinterDB does not rely on Rule 2, future
 // versions may store pointers to Splinter's own memory in the lookup_result.
 void
 splinterdb_lookup_result_init(const splinterdb         *kvs,        // IN
                               splinterdb_lookup_result *result,     // IN/OUT
+                              splinterdb_lookup_flags   flags,      // IN
                               uint64                    buffer_len, // IN
                               char                     *buffer      // IN
 );
@@ -231,7 +242,10 @@ splinterdb_lookup_result_init(const splinterdb         *kvs,        // IN
 void
 splinterdb_lookup_result_deinit(splinterdb_lookup_result *result); // IN
 
-// Returns true if the result was found
+// Returns true if the result was found.
+//
+// For existence-only lookups, this returns true when the key exists or might
+// exist.
 _Bool
 splinterdb_lookup_found(const splinterdb_lookup_result *result); // IN
 
@@ -243,11 +257,6 @@ splinterdb_lookup_result_value(const splinterdb_lookup_result *result, // IN
                                slice                          *value   // OUT
 );
 
-int
-splinterdb_lookup_result_key(const splinterdb_lookup_result *result, // IN
-                             slice                          *key     // OUT
-);
-
 // Lookup the message for a given key
 //
 // result must have first been initialized using splinterdb_lookup_result_init

src/btree.c

@@ -2079,7 +2079,6 @@ btree_lookup_node(cache              *cc,             // IN
  *
  * LOCAL Variables:
  * - state->h: the height of the current node
- * - state->found: whether the target was found
  * - state->child_node: the child node
  */
 static inline async_status
@@ -2114,11 +2113,12 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth)
         state->h > state->stop_at_height;
         state->h--)
    {
-      int64 child_idx =
+      bool32 found;
+      int64  child_idx =
          key_is_positive_infinity(state->target)
-            ? btree_num_entries(state->node.hdr) - 1
-            : btree_find_pivot(
-                 state->cfg, state->node.hdr, state->target, &state->found);
+             ? btree_num_entries(state->node.hdr) - 1
+             : btree_find_pivot(
+                 state->cfg, state->node.hdr, state->target, &found);
       if (child_idx < 0) {
          child_idx = 0;
       }
@@ -2165,7 +2165,7 @@ btree_lookup_node_async(btree_lookup_async_state *state, uint64 depth)
  *
  * OUT Parameters:
  * - state->node: the node found
- * - state->found: whether the target was found in the leaf
+ * - state->found_key: the key found in the leaf, or NULL_KEY otherwise.
  * - state->msg: the message of the target
  *
  * LOCAL Variables:
@@ -2183,9 +2183,11 @@ btree_lookup_with_ref_async(btree_lookup_async_state *state, uint64 depth)
    state->stats          = NULL;
    async_await_subroutine(state, btree_lookup_node_async);
 
-   int64 idx = btree_find_tuple(
-      state->cfg, state->node.hdr, state->target, &state->found);
-   if (state->found) {
+   state->found_key = NULL_KEY;
+   bool32 found;
+   int64  idx =
+      btree_find_tuple(state->cfg, state->node.hdr, state->target, &found);
+   if (found) {
       leaf_entry *entry =
          btree_get_leaf_entry(state->cfg, state->node.hdr, idx);
       state->found_key = leaf_entry_key(entry);
@@ -2205,13 +2207,15 @@ btree_lookup_with_ref(cache              *cc,        // IN
                       key                 target,    // IN
                       btree_node         *node,      // OUT
                       key                *found_key, // OUT
-                      message            *msg,       // OUT
-                      bool32             *found)                 // OUT
+                      message            *msg)                  // OUT
 {
-   platform_status rc = STATUS_OK;
+   platform_status rc    = STATUS_OK;
+   bool32          found = FALSE;
+
+   *found_key = NULL_KEY;
    btree_lookup_node(cc, cfg, root_addr, target, 0, type, node, NULL);
-   int64 idx = btree_find_tuple(cfg, node->hdr, target, found);
-   if (*found) {
+   int64 idx = btree_find_tuple(cfg, node->hdr, target, &found);
+   if (found) {
       leaf_entry *entry = btree_get_leaf_entry(cfg, node->hdr, idx);
       *found_key        = leaf_entry_key(entry);
       *msg              = leaf_entry_message(entry);
@@ -2221,57 +2225,17 @@ btree_lookup_with_ref(cache              *cc,        // IN
    return rc;
 }
 
-async_status
-btree_lookup_async(btree_lookup_async_state *state)
-{
-   async_begin(state, 0);
-
-   async_await_subroutine(state, btree_lookup_with_ref_async);
-
-   platform_status rc = STATUS_OK;
-   if (state->found) {
-      if (state->keybuf != NULL) {
-         rc = key_buffer_copy_key(state->keybuf, state->found_key);
-      }
-      bool32 success =
-         merge_accumulator_copy_message(state->result, state->msg);
-      if (!success) {
-         rc = STATUS_NO_MEMORY;
-      }
-      btree_node_unget(state->cc, state->cfg, &state->node);
-   }
-   async_return(state, rc);
-}
-
-
 platform_status
-btree_lookup(cache             *cc,        // IN
-             btree_config      *cfg,       // IN
-             uint64             root_addr, // IN
-             page_type          type,      // IN
-             key                target,    // IN
-             key_buffer        *keybuf,    // OUT
-             merge_accumulator *result)    // OUT
+btree_lookup(cache         *cc,        // IN
+             btree_config  *cfg,       // IN
+             uint64         root_addr, // IN
+             page_type      type,      // IN
+             key            target,    // IN
+             lookup_result *result)    // OUT
 {
-   btree_node      node;
-   key             found_key;
-   message         data;
-   platform_status rc = STATUS_OK;
-   bool32          local_found;
-
-   btree_lookup_with_ref(
-      cc, cfg, root_addr, type, target, &node, &found_key, &data, &local_found);
-   if (local_found) {
-      if (keybuf != NULL) {
-         rc = key_buffer_copy_key(keybuf, found_key);
-      }
-      bool32 success = merge_accumulator_copy_message(result, data);
-      if (!success) {
-         rc = STATUS_NO_MEMORY;
-      }
-      btree_node_unget(cc, cfg, &node);
-   }
-   return rc;
+   lookup_result_reset(result);
+   return btree_lookup_and_merge(
+      cc, cfg, root_addr, type, target, result, NULL);
 }
 
 platform_status
@@ -2280,8 +2244,7 @@ btree_lookup_and_merge(cache              *cc,        // IN
                        uint64              root_addr, // IN
                        page_type           type,      // IN
                        key                 target,    // IN
-                       key_buffer         *keybuf,    // OUT
-                       merge_accumulator  *data,      // OUT
+                       lookup_result      *result,    // IN/OUT
                        bool32             *local_found)           // OUT
 {
    btree_node      node;
@@ -2291,27 +2254,17 @@ btree_lookup_and_merge(cache              *cc,        // IN
 
    log_trace_key(target, "btree_lookup");
 
-   btree_lookup_with_ref(cc,
-                         cfg,
-                         root_addr,
-                         type,
-                         target,
-                         &node,
-                         &found_key,
-                         &local_data,
-                         local_found);
-   if (*local_found) {
-      if (keybuf != NULL) {
-         rc = key_buffer_copy_key(keybuf, found_key);
-      }
-      if (merge_accumulator_is_null(data)) {
-         bool32 success = merge_accumulator_copy_message(data, local_data);
-         if (!success) {
-            rc = STATUS_NO_MEMORY;
-         }
-      } else if (btree_merge_tuples(cfg, target, local_data, data)) {
-         rc = STATUS_NO_MEMORY;
+   if (local_found != NULL) {
+      *local_found = FALSE;
+   }
+
+   btree_lookup_with_ref(
+      cc, cfg, root_addr, type, target, &node, &found_key, &local_data);
+   if (!key_is_null(found_key)) {
+      if (local_found != NULL) {
+         *local_found = TRUE;
       }
+      rc = lookup_result_update(result, found_key, local_data);
       btree_node_unget(cc, cfg, &node);
    }
    return rc;
@@ -2325,11 +2278,8 @@ btree_lookup_and_merge(cache              *cc,        // IN
  * - state->type: the type of the root node
  * - state->target: the key to look up
  *
- * IN/OUT Parameters:
- * - state->result: the result of the lookup
- *
  * OUT Parameters:
- * - state->found: whether the target was found in the leaf
+ * - state->found_key: the key found in the leaf, or NULL_KEY otherwise.
  *
  * LOCAL Variables:
  * - state->node: the node found
@@ -2347,26 +2297,20 @@ btree_lookup_and_merge_async(btree_lookup_async_state *state)
    async_await_subroutine(state, btree_lookup_with_ref_async);
 
    platform_status rc = STATUS_OK;
-   if (state->found) {
-      if (state->keybuf != NULL) {
-         rc = key_buffer_copy_key(state->keybuf, state->found_key);
-      }
-      if (merge_accumulator_is_null(state->result)) {
-         bool32 success =
-            merge_accumulator_copy_message(state->result, state->msg);
-         if (!success) {
-            rc = STATUS_NO_MEMORY;
-         }
-      } else if (btree_merge_tuples(
-                    state->cfg, state->target, state->msg, state->result))
-      {
-         rc = STATUS_NO_MEMORY;
-      }
+   if (!key_is_null(state->found_key)) {
+      rc = lookup_result_update(state->result, state->found_key, state->msg);
       btree_node_unget(state->cc, state->cfg, &state->node);
    }
    async_return(state, rc);
 }
 
+async_status
+btree_lookup_async(btree_lookup_async_state *state)
+{
+   lookup_result_reset(state->result);
+   return btree_lookup_and_merge_async(state);
+}
+
 /*
  *-----------------------------------------------------------------------------
  * btree_iterator_init     --