Better inlining

amalgamation.sh

@@ -37,8 +37,25 @@ DEMOCPP="amalgamation_demo.cpp"
 ALL_PUBLIC_H="
 $SCRIPTPATH/include/roaring/roaring_version.h
 $SCRIPTPATH/include/roaring/portability.h
+$SCRIPTPATH/include/roaring/isadetection.h
 $SCRIPTPATH/include/roaring/roaring_types.h
 $SCRIPTPATH/include/roaring/bitset/bitset.h
+$SCRIPTPATH/include/roaring/containers/container_defs.h
+$SCRIPTPATH/include/roaring/array_util.h
+$SCRIPTPATH/include/roaring/bitset_util.h
+$SCRIPTPATH/include/roaring/containers/array.h
+$SCRIPTPATH/include/roaring/containers/bitset.h
+$SCRIPTPATH/include/roaring/containers/run.h
+$SCRIPTPATH/include/roaring/containers/convert.h
+$SCRIPTPATH/include/roaring/containers/mixed_equal.h
+$SCRIPTPATH/include/roaring/containers/mixed_subset.h
+$SCRIPTPATH/include/roaring/containers/mixed_andnot.h
+$SCRIPTPATH/include/roaring/containers/mixed_intersection.h
+$SCRIPTPATH/include/roaring/containers/mixed_negation.h
+$SCRIPTPATH/include/roaring/containers/mixed_union.h
+$SCRIPTPATH/include/roaring/containers/mixed_xor.h
+$SCRIPTPATH/include/roaring/containers/containers.h
+$SCRIPTPATH/include/roaring/roaring_array.h
 $SCRIPTPATH/include/roaring/roaring.h
 $SCRIPTPATH/include/roaring/memory.h
 $SCRIPTPATH/include/roaring/roaring64.h
@@ -56,25 +73,8 @@ $SCRIPTPATH/cpp/roaring/roaring64map.hh
 # need to be in this order.
 #
 ALL_PRIVATE_H="
-$SCRIPTPATH/include/roaring/isadetection.h
 $SCRIPTPATH/include/roaring/containers/perfparameters.h
-$SCRIPTPATH/include/roaring/containers/container_defs.h
-$SCRIPTPATH/include/roaring/array_util.h
 $SCRIPTPATH/include/roaring/utilasm.h
-$SCRIPTPATH/include/roaring/bitset_util.h
-$SCRIPTPATH/include/roaring/containers/array.h
-$SCRIPTPATH/include/roaring/containers/bitset.h
-$SCRIPTPATH/include/roaring/containers/run.h
-$SCRIPTPATH/include/roaring/containers/convert.h
-$SCRIPTPATH/include/roaring/containers/mixed_equal.h
-$SCRIPTPATH/include/roaring/containers/mixed_subset.h
-$SCRIPTPATH/include/roaring/containers/mixed_andnot.h
-$SCRIPTPATH/include/roaring/containers/mixed_intersection.h
-$SCRIPTPATH/include/roaring/containers/mixed_negation.h
-$SCRIPTPATH/include/roaring/containers/mixed_union.h
-$SCRIPTPATH/include/roaring/containers/mixed_xor.h
-$SCRIPTPATH/include/roaring/containers/containers.h
-$SCRIPTPATH/include/roaring/roaring_array.h
 $SCRIPTPATH/include/roaring/art/art.h
 "
 
@@ -89,7 +89,7 @@ $SCRIPTPATH/include/roaring/art/art.h
 ALL_PRIVATE_C=$( ( ( \
     [ -d $SCRIPTPATH/.git ] \
         && ( type git >/dev/null 2>&1 ) \
-        && ( git -C $SCRIPTPATH ls-files 'src/*.c' ) \
+        && ( git -C $SCRIPTPATH ls-files 'src/*.c') \
     ) || ( find $SCRIPTPATH/src -name '*.c' ) ) | sort )
 # Verify up-front that all the files exist
 #
@@ -251,7 +251,6 @@ echo "Creating ${DEMOCPP}..."
     cat <<< '
 #include <iostream>
 #include "roaring.hh"
-//#include "roaring.c"
 
 int main() {
   roaring::Roaring r1;

include/roaring/containers/container_defs.h

@@ -32,7 +32,6 @@ namespace internal {  // No extern "C" (contains template)
  * Then undefine the awkward macro so it's not used any more than it has to be.
  */
 typedef ROARING_CONTAINER_T container_t;
-#undef ROARING_CONTAINER_T
 
 /*
  * See ROARING_CONTAINER_T for notes on using container_t as a base class.

include/roaring/containers/containers.h

@@ -2434,15 +2434,131 @@ roaring_container_iterator_t container_init_iterator_last(const container_t *c,
  * Moves the iterator to the next entry. Returns true and sets `value` if a
  * value is present.
  */
-bool container_iterator_next(const container_t *c, uint8_t typecode,
-                             roaring_container_iterator_t *it, uint16_t *value);
+inline bool container_iterator_next(const container_t *c, uint8_t typecode,
+                                    roaring_container_iterator_t *it,
+                                    uint16_t *value) {
+    switch (typecode) {
+        case BITSET_CONTAINER_TYPE: {
+            const bitset_container_t *bc = const_CAST_bitset(c);
+            it->index++;
+
+            uint32_t wordindex = it->index / 64;
+            if (wordindex >= BITSET_CONTAINER_SIZE_IN_WORDS) {
+                return false;
+            }
+
+            uint64_t word =
+                bc->words[wordindex] & (UINT64_MAX << (it->index % 64));
+            // next part could be optimized/simplified
+            while (word == 0 &&
+                   (wordindex + 1 < BITSET_CONTAINER_SIZE_IN_WORDS)) {
+                wordindex++;
+                word = bc->words[wordindex];
+            }
+            if (word != 0) {
+                it->index = wordindex * 64 + roaring_trailing_zeroes(word);
+                *value = it->index;
+                return true;
+            }
+            return false;
+        }
+        case ARRAY_CONTAINER_TYPE: {
+            const array_container_t *ac = const_CAST_array(c);
+            it->index++;
+            if (it->index < ac->cardinality) {
+                *value = ac->array[it->index];
+                return true;
+            }
+            return false;
+        }
+        case RUN_CONTAINER_TYPE: {
+            if (*value == UINT16_MAX) {  // Avoid overflow to zero
+                return false;
+            }
+
+            const run_container_t *rc = const_CAST_run(c);
+            uint32_t limit =
+                rc->runs[it->index].value + rc->runs[it->index].length;
+            if (*value < limit) {
+                (*value)++;
+                return true;
+            }
+
+            it->index++;
+            if (it->index < rc->n_runs) {
+                *value = rc->runs[it->index].value;
+                return true;
+            }
+            return false;
+        }
+        default:
+            assert(false);
+            roaring_unreachable;
+            return false;
+    }
+}
 
 /**
  * Moves the iterator to the previous entry. Returns true and sets `value` if a
  * value is present.
  */
-bool container_iterator_prev(const container_t *c, uint8_t typecode,
-                             roaring_container_iterator_t *it, uint16_t *value);
+inline bool container_iterator_prev(const container_t *c, uint8_t typecode,
+                                    roaring_container_iterator_t *it,
+                                    uint16_t *value) {
+    switch (typecode) {
+        case BITSET_CONTAINER_TYPE: {
+            if (--it->index < 0) {
+                return false;
+            }
+
+            const bitset_container_t *bc = const_CAST_bitset(c);
+            int32_t wordindex = it->index / 64;
+            uint64_t word =
+                bc->words[wordindex] & (UINT64_MAX >> (63 - (it->index % 64)));
+
+            while (word == 0 && --wordindex >= 0) {
+                word = bc->words[wordindex];
+            }
+            if (word == 0) {
+                return false;
+            }
+
+            it->index = (wordindex * 64) + (63 - roaring_leading_zeroes(word));
+            *value = it->index;
+            return true;
+        }
+        case ARRAY_CONTAINER_TYPE: {
+            if (--it->index < 0) {
+                return false;
+            }
+            const array_container_t *ac = const_CAST_array(c);
+            *value = ac->array[it->index];
+            return true;
+        }
+        case RUN_CONTAINER_TYPE: {
+            if (*value == 0) {
+                return false;
+            }
+
+            const run_container_t *rc = const_CAST_run(c);
+            (*value)--;
+            if (*value >= rc->runs[it->index].value) {
+                return true;
+            }
+
+            if (--it->index < 0) {
+                return false;
+            }
+
+            *value = rc->runs[it->index].value + rc->runs[it->index].length;
+            return true;
+        }
+        default:
+            assert(false);
+            roaring_unreachable;
+            return false;
+    }
+}
 
 /**
  * Moves the iterator to the smallest entry that is greater than or equal to

include/roaring/roaring.h

@@ -13,8 +13,10 @@
 
 // Include other headers after roaring_types.h
 #include <roaring/bitset/bitset.h>
+#include <roaring/containers/containers.h>
 #include <roaring/memory.h>
 #include <roaring/portability.h>
+#include <roaring/roaring_array.h>
 #include <roaring/roaring_version.h>
 
 #ifdef __cplusplus
@@ -462,7 +464,27 @@ bool roaring_bitmap_remove_checked(roaring_bitmap_t *r, uint32_t x);
 /**
  * Check if value is present
  */
-bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val);
+inline bool roaring_bitmap_contains(const roaring_bitmap_t *r, uint32_t val) {
+    // For performance reasons, this function is inline and uses internal
+    // functions directly.
+#ifdef __cplusplus
+    using namespace ::roaring::internal;
+#endif
+    const uint16_t hb = val >> 16;
+    /*
+     * the next function call involves a binary search and lots of branching.
+     */
+    int32_t i = ra_get_index(&r->high_low_container, hb);
+    if (i < 0) return false;
+
+    uint8_t typecode;
+    // next call ought to be cheap
+    container_t *container = ra_get_container_at_index(&r->high_low_container,
+                                                       (uint16_t)i, &typecode);
+    // rest might be a tad expensive, possibly involving another round of binary
+    // search
+    return container_contains(container, val & 0xFFFF, typecode);
+}
 
 /**
  * Check whether a range of values from range_start (included)

src/CMakeLists.txt

@@ -62,6 +62,8 @@ endif(ROARING_DISABLE_NEON)
 target_link_libraries(roaring PUBLIC "$<BUILD_INTERFACE:roaring-headers>")
 target_link_libraries(roaring PUBLIC "$<BUILD_INTERFACE:roaring-headers-cpp>")
 
+target_include_directories(roaring PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
 #
 #install(TARGETS roaring DESTINATION lib)
 #