From e227c0768d096235dbcec7a3a8a9f810a4ce724f Mon Sep 17 00:00:00 2001 From: mpope9 Date: Sat, 20 Nov 2021 13:36:06 -0800 Subject: [PATCH] Merging upstream xorfilter.h changes, bumping version --- c_src/xorfilter.h | 79 +++++++++++++++++++---------------------- src/exor_filter.app.src | 2 +- 2 files changed, 38 insertions(+), 43 deletions(-) diff --git a/c_src/xorfilter.h b/c_src/xorfilter.h index 0599344..96eb45e 100644 --- a/c_src/xorfilter.h +++ b/c_src/xorfilter.h @@ -24,7 +24,7 @@ #ifndef XOR_MAX_ITERATIONS #define XOR_MAX_ITERATIONS 100 // probabillity of success should always be > 0.5 so 100 iterations is highly unlikely -#endif +#endif /** * We assume that you have a large set of 64-bit integers @@ -136,10 +136,9 @@ static inline bool xor8_allocate(uint32_t size, xor8_t *filter) { } // allocate enough capacity for a set containing up to 'size' elements -// caller is responsible to call xor8_free(filter) +// caller is responsible to call xor16_free(filter) static inline bool xor16_allocate(uint32_t size, xor16_t *filter) { size_t capacity = 32 + 1.23 * size; - filter->blockLength = capacity / 3; capacity = capacity / 3 * 3; filter->fingerprints = (uint16_t *)malloc(capacity * sizeof(uint16_t)); if (filter->fingerprints != NULL) { @@ -212,18 +211,6 @@ struct xor_h0h1h2_s { typedef struct xor_h0h1h2_s xor_h0h1h2_t; -static inline xor_h0h1h2_t xor8_get_just_h0_h1_h2(uint64_t hash, - const xor8_t *filter) { - xor_h0h1h2_t answer; - uint32_t r0 = (uint32_t)hash; - uint32_t r1 = (uint32_t)xor_rotl64(hash, 21); - uint32_t r2 = (uint32_t)xor_rotl64(hash, 42); - - answer.h0 = xor_reduce(r0, filter->blockLength); - answer.h1 = xor_reduce(r1, filter->blockLength); - answer.h2 = xor_reduce(r2, filter->blockLength); - return answer; -} static inline uint32_t xor8_get_h0(uint64_t hash, const xor8_t *filter) { uint32_t r0 = (uint32_t)hash; return xor_reduce(r0, filter->blockLength); @@ -345,7 +332,7 @@ static inline void xor_make_buffer_current(xor_setbuffer_t *buffer, } *Qsize = qsize; buffer->counts[slot] = 0; - } + } } @@ -458,6 +445,7 @@ static inline uint32_t xor_flushone_decrement_buffer(xor_setbuffer_t *buffer, // a return value of false is provided. // bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) { + if(size == 0) { return false; } uint64_t rng_counter = 1; filter->seed = xor_rng_splitmix64(&rng_counter); size_t arrayLength = filter->blockLength * 3; // size of the backing array @@ -476,14 +464,9 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) xor_xorset_t *sets = (xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t)); xor_xorset_t *sets0 = sets; - xor_xorset_t *sets1 = sets + blockLength; - xor_xorset_t *sets2 = sets + 2 * blockLength; xor_keyindex_t *Q = (xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t)); - xor_keyindex_t *Q0 = Q; - xor_keyindex_t *Q1 = Q + blockLength; - xor_keyindex_t *Q2 = Q + 2 * blockLength; xor_keyindex_t *stack = (xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t)); @@ -497,6 +480,12 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) free(stack); return false; } + xor_xorset_t *sets1 = sets + blockLength; + xor_xorset_t *sets2 = sets + 2 * blockLength; + xor_keyindex_t *Q0 = Q; + xor_keyindex_t *Q1 = Q + blockLength; + xor_keyindex_t *Q2 = Q + 2 * blockLength; + int iterations = 0; while (true) { @@ -532,7 +521,7 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) Q0[Q0size].index = i; Q0[Q0size].hash = sets0[i].xormask; Q0size++; - } + } } for (size_t i = 0; i < filter->blockLength; i++) { @@ -668,6 +657,7 @@ bool xor8_buffered_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) // a return value of false is provided. // bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) { + if(size == 0) { return false; } uint64_t rng_counter = 1; filter->seed = xor_rng_splitmix64(&rng_counter); size_t arrayLength = filter->blockLength * 3; // size of the backing array @@ -675,15 +665,9 @@ bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) { xor_xorset_t *sets = (xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t)); - xor_xorset_t *sets0 = sets; - xor_xorset_t *sets1 = sets + blockLength; - xor_xorset_t *sets2 = sets + 2 * blockLength; xor_keyindex_t *Q = (xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t)); - xor_keyindex_t *Q0 = Q; - xor_keyindex_t *Q1 = Q + blockLength; - xor_keyindex_t *Q2 = Q + 2 * blockLength; xor_keyindex_t *stack = (xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t)); @@ -694,6 +678,13 @@ bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) { free(stack); return false; } + xor_xorset_t *sets0 = sets; + xor_xorset_t *sets1 = sets + blockLength; + xor_xorset_t *sets2 = sets + 2 * blockLength; + xor_keyindex_t *Q0 = Q; + xor_keyindex_t *Q1 = Q + blockLength; + xor_keyindex_t *Q2 = Q + 2 * blockLength; + int iterations = 0; while (true) { @@ -874,12 +865,13 @@ bool xor8_populate(const uint64_t *keys, uint32_t size, xor8_t *filter) { // a return value of false is provided. // bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) { + if(size == 0) { return false; } uint64_t rng_counter = 1; filter->seed = xor_rng_splitmix64(&rng_counter); size_t arrayLength = filter->blockLength * 3; // size of the backing array xor_setbuffer_t buffer0, buffer1, buffer2; size_t blockLength = filter->blockLength; - bool ok0 = xor_init_buffer(&buffer0, blockLength); + bool ok0 = xor_init_buffer(&buffer0, blockLength); bool ok1 = xor_init_buffer(&buffer1, blockLength); bool ok2 = xor_init_buffer(&buffer2, blockLength); if (!ok0 || !ok1 || !ok2) { @@ -891,15 +883,9 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte xor_xorset_t *sets = (xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t)); - xor_xorset_t *sets0 = sets; - xor_xorset_t *sets1 = sets + blockLength; - xor_xorset_t *sets2 = sets + 2 * blockLength; xor_keyindex_t *Q = (xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t)); - xor_keyindex_t *Q0 = Q; - xor_keyindex_t *Q1 = Q + blockLength; - xor_keyindex_t *Q2 = Q + 2 * blockLength; xor_keyindex_t *stack = (xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t)); @@ -913,6 +899,13 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte free(stack); return false; } + xor_xorset_t *sets0 = sets; + xor_xorset_t *sets1 = sets + blockLength; + xor_xorset_t *sets2 = sets + 2 * blockLength; + xor_keyindex_t *Q0 = Q; + xor_keyindex_t *Q1 = Q + blockLength; + xor_keyindex_t *Q2 = Q + 2 * blockLength; + int iterations = 0; while (true) { @@ -1087,6 +1080,7 @@ bool xor16_buffered_populate(const uint64_t *keys, uint32_t size, xor16_t *filte // a return value of false is provided. // bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) { + if(size == 0) { return false; } uint64_t rng_counter = 1; filter->seed = xor_rng_splitmix64(&rng_counter); size_t arrayLength = filter->blockLength * 3; // size of the backing array @@ -1094,15 +1088,9 @@ bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) { xor_xorset_t *sets = (xor_xorset_t *)malloc(arrayLength * sizeof(xor_xorset_t)); - xor_xorset_t *sets0 = sets; - xor_xorset_t *sets1 = sets + blockLength; - xor_xorset_t *sets2 = sets + 2 * blockLength; xor_keyindex_t *Q = (xor_keyindex_t *)malloc(arrayLength * sizeof(xor_keyindex_t)); - xor_keyindex_t *Q0 = Q; - xor_keyindex_t *Q1 = Q + blockLength; - xor_keyindex_t *Q2 = Q + 2 * blockLength; xor_keyindex_t *stack = (xor_keyindex_t *)malloc(size * sizeof(xor_keyindex_t)); @@ -1113,6 +1101,14 @@ bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) { free(stack); return false; } + xor_xorset_t *sets0 = sets; + xor_xorset_t *sets1 = sets + blockLength; + xor_xorset_t *sets2 = sets + 2 * blockLength; + + xor_keyindex_t *Q0 = Q; + xor_keyindex_t *Q1 = Q + blockLength; + xor_keyindex_t *Q2 = Q + 2 * blockLength; + int iterations = 0; while (true) { @@ -1284,4 +1280,3 @@ bool xor16_populate(const uint64_t *keys, uint32_t size, xor16_t *filter) { #endif - diff --git a/src/exor_filter.app.src b/src/exor_filter.app.src index ff1aeab..99e4e27 100644 --- a/src/exor_filter.app.src +++ b/src/exor_filter.app.src @@ -1,6 +1,6 @@ {application, exor_filter, [{description, "Nif wrapper for xor_filters."}, - {vsn, "0.8.0"}, + {vsn, "0.8.1"}, {registered, []}, {applications, [kernel,