path: root/third_party/xxhash/tests/collisions/main.c

/*
 * Brute force collision tester for 64-bit hashes
 * Part of the xxHash project
 * Copyright (C) 2019-2020 Yann Collet
 *
 * GPL v2 License
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 *
 * You can contact the author at:
 *   - xxHash homepage: https://www.xxhash.com
 *   - xxHash source repository: https://github.com/Cyan4973/xxHash
 */

/*
 * The collision tester will generate 24 billion hashes (by default),
 * and count how many collisions were produced by the 64-bit hash algorithm.
 * The optimal amount of collisions for 64-bit is ~18 collisions.
 * A good hash should be close to this figure.
 *
 * This program requires a lot of memory:
 * - Either store hash values directly => 192 GB
 * - Or use a filter:
 *   -  32 GB (by default) for the filter itself
 *   -  + ~14 GB for the list of hashes (depending on the filter's outcome)
 * Due to these memory constraints, it requires a 64-bit system.
 */


 /* ===  Dependencies  === */

#include <stdint.h>   /* uint64_t */
#include <stdlib.h>   /* malloc, free, qsort, exit */
#include <string.h>   /* memset */
#include <stdio.h>    /* printf, fflush */

#undef NDEBUG   /* ensure assert is _not_ disabled */
#include <assert.h>

#include "hashes.h"   /* UniHash, hashfn, hashfnTable */

#include "sort.hh"    /* sort64 */



typedef enum { ht32, ht64, ht128 } Htype_e;

/* ===  Debug  === */

#define EXIT(...) { printf(__VA_ARGS__); printf("\n"); exit(1); }

static void hexRaw(const void* buffer, size_t size)
{
    const unsigned char* p = (const unsigned char*)buffer;
    for (size_t i=0; i<size; i++) {
        printf("%02X", p[i]);
    }
}

void printHash(const void* table, size_t n, Htype_e htype)
{
    if ((htype == ht64) || (htype == ht32)){
        uint64_t const h64 = ((const uint64_t*)table)[n];
        hexRaw(&h64, sizeof(h64));
    } else {
        assert(htype == ht128);
        XXH128_hash_t const h128 = ((const XXH128_hash_t*)table)[n];
        hexRaw(&h128, sizeof(h128));
    }
}

/* ===  Generate Random unique Samples to hash  === */

/*
 * These functions will generate and update a sample to hash.
 * initSample() will fill a buffer with random bytes,
 * updateSample() will modify one slab in the input buffer.
 * updateSample() guarantees it will produce unique samples,
 * but it needs to know the total number of samples.
 */


static const uint64_t prime64_1 = 11400714785074694791ULL;   /* 0b1001111000110111011110011011000110000101111010111100101010000111 */
static const uint64_t prime64_2 = 14029467366897019727ULL;   /* 0b1100001010110010101011100011110100100111110101001110101101001111 */
static const uint64_t prime64_3 =  1609587929392839161ULL;   /* 0b0001011001010110011001111011000110011110001101110111100111111001 */

static uint64_t avalanche64(uint64_t h64)
{
    h64 ^= h64 >> 33;
    h64 *= prime64_2;
    h64 ^= h64 >> 29;
    h64 *= prime64_3;
    h64 ^= h64 >> 32;
    return h64;
}

static unsigned char randomByte(size_t n)
{
    uint64_t n64 = avalanche64(n+1);
    n64 *= prime64_1;
    return (unsigned char)(n64 >> 56);
}

typedef enum { sf_slab5, sf_sparse } sf_genMode;


#ifdef SLAB5

/*
 * Slab5 sample generation.
 * This algorithm generates unique inputs flipping on average 16 bits per candidate.
 * It is generally much more friendly for most hash algorithms, especially
 * weaker ones, as it shuffles more the input.
 * The algorithm also avoids overfitting the per4 or per8 ingestion patterns.
 */

#define SLAB_SIZE 5

typedef struct {
    void* buffer;
    size_t size;
    sf_genMode mode;
    size_t prngSeed;
    uint64_t hnb;
} sampleFactory;

static void init_sampleFactory(sampleFactory* sf, uint64_t htotal)
{
    uint64_t const minNbSlabs = ((htotal-1) >> 32) + 1;
    uint64_t const minSize = minNbSlabs * SLAB_SIZE;
    if (sf->size < minSize)
        EXIT("sample size must be >= %i bytes for this amount of hashes",
            (int)minSize);

    unsigned char* const p = (unsigned char*)sf->buffer;
    for (size_t n=0; n < sf->size; n++)
        p[n] = randomByte(n);
    sf->hnb = 0;
}

static sampleFactory*
create_sampleFactory(size_t size, uint64_t htotal, uint64_t seed)
{
    sampleFactory* const sf = malloc(sizeof(sampleFactory));
    if (!sf) EXIT("not enough memory");
    void* const buffer = malloc(size);
    if (!buffer) EXIT("not enough memory");
    sf->buffer = buffer;
    sf->size = size;
    sf->mode = sf_slab5;
    sf->prngSeed = seed;
    init_sampleFactory(sf, htotal);
    return sf;
}

static void free_sampleFactory(sampleFactory* sf)
{
    if (!sf) return;
    free(sf->buffer);
    free(sf);
}

static inline void update_sampleFactory(sampleFactory* sf)
{
    size_t const nbSlabs = sf->size / SLAB_SIZE;
    size_t const SlabNb = sf->hnb % nbSlabs;
    sf->hnb++;

    char* const ptr = (char*)sf->buffer;
    size_t const start = (SlabNb * SLAB_SIZE) + 1;
    uint32_t val32;
    memcpy(&val32, ptr+start, sizeof(val32));
    static const uint32_t prime32_5 = 374761393U;
    val32 += prime32_5;
    memcpy(ptr+start, &val32, sizeof(val32));
}

#else

/*
 * Sparse sample generation.
 * This is the default pattern generator.
 * It only flips one bit at a time (mostly).
 * Low hamming distance scenario is more difficult for weak hash algorithms.
 * Note that CRC is immune to this scenario, since they are specifically
 * designed to detect low hamming distances.
 * Prefer the Slab5 pattern generator for collisions on CRC algorithms.
 */

#define SPARSE_LEVEL_MAX 15

/* Nb of combinations of m bits in a register of n bits */
static double Cnm(int n, int m)
{
    assert(n > 0);
    assert(m > 0);
    assert(m <= m);
    double acc = 1;
    for (int i=0; i<m; i++) {
        acc *= n - i;
        acc /= 1 + i;
    }
    return acc;
}

static int enoughCombos(size_t size, uint64_t htotal)
{
    if (size < 2) return 0;   /* ensure no multiplication by negative */
    uint64_t acc = 0;
    uint64_t const srcBits = size * 8; assert(srcBits < INT_MAX);
    int nbBitsSet = 0;
    while (acc < htotal) {
        nbBitsSet++;
        if (nbBitsSet >= SPARSE_LEVEL_MAX) return 0;
        acc += (uint64_t)Cnm((int)srcBits, nbBitsSet);
    }
    return 1;
}

typedef struct {
    void* buffer;
    size_t size;
    sf_genMode mode;
    /* sparse */
    size_t bitIdx[SPARSE_LEVEL_MAX];
    int level;
    size_t maxBitIdx;
    /* slab5 */
    size_t nbSlabs;
    size_t current;
    size_t prngSeed;
} sampleFactory;

static void init_sampleFactory(sampleFactory* sf, uint64_t htotal)
{
    if (!enoughCombos(sf->size, htotal)) {
        EXIT("sample size must be larger for this amount of hashes");
    }

    memset(sf->bitIdx, 0, sizeof(sf->bitIdx));
    sf->level = 0;

    unsigned char* const p = (unsigned char*)sf->buffer;
    for (size_t n=0; n<sf->size; n++)
        p[n] = randomByte(sf->prngSeed + n);
}

static sampleFactory*
create_sampleFactory(size_t size, uint64_t htotal, uint64_t seed)
{
    sampleFactory* const sf = malloc(sizeof(sampleFactory));
    if (!sf) EXIT("not enough memory");
    void* const buffer = malloc(size);
    if (!buffer) EXIT("not enough memory");
    sf->buffer = buffer;
    sf->size = size;
    sf->mode = sf_sparse;
    sf->maxBitIdx = size * 8;
    sf->prngSeed = seed;
    init_sampleFactory(sf, htotal);
    return sf;
}

static void free_sampleFactory(sampleFactory* sf)
{
    if (!sf) return;
    free(sf->buffer);
    free(sf);
}

static void flipbit(void* buffer, uint64_t bitID)
{
    size_t const pos = bitID >> 3;
    unsigned char const mask = (unsigned char)(1 << (bitID & 7));
    unsigned char* const p = (unsigned char*)buffer;
    p[pos] ^= mask;
}

static int updateBit(void* buffer, size_t* bitIdx, int level, size_t max)
{
    if (level==0) return 0;   /* can't progress further */

    flipbit(buffer, bitIdx[level]); /* erase previous bits */

    if (bitIdx[level] < max-1) { /* simple case: go to next bit */
        bitIdx[level]++;
        flipbit(buffer, bitIdx[level]); /* set new bit */
        return 1;
    }

    /* reached last bit: need to update a bit from lower level */
    if (!updateBit(buffer, bitIdx, level-1, max-1)) return 0;
    bitIdx[level] = bitIdx[level-1] + 1;
    flipbit(buffer, bitIdx[level]); /* set new bit */
    return 1;
}

static inline void update_sampleFactory(sampleFactory* sf)
{
    if (!updateBit(sf->buffer, sf->bitIdx, sf->level, sf->maxBitIdx)) {
        /* no more room => move to next level */
        sf->level++;
        assert(sf->level < SPARSE_LEVEL_MAX);

        /* set new bits */
        for (int i=1; i <= sf->level; i++) {
            sf->bitIdx[i] = (size_t)(i-1);
            flipbit(sf->buffer, sf->bitIdx[i]);
        }
    }
}

#endif /* pattern generator selection */


/* ===  Candidate Filter  === */

typedef unsigned char Filter;

Filter* create_Filter(int bflog)
{
    assert(bflog < 64 && bflog > 1);
    size_t bfsize = (size_t)1 << bflog;
    Filter* bf = malloc(bfsize);
    assert(((void)"Filter creation failed", bf));
    memset(bf, 0, bfsize);
    return bf;
}

void free_Filter(Filter* bf)
{
    free(bf);
}

#ifdef FILTER_1_PROBE

/*
 * Attach hash to a slot
 * return: Nb of potential collision candidates detected
 *          0: position not yet occupied
 *          2: position previously occupied by a single candidate
 *          1: position already occupied by multiple candidates
 */
inline int Filter_insert(Filter* bf, int bflog, uint64_t hash)
{
    int const slotNb = hash & 3;
    int const shift = slotNb * 2 ;

    size_t const bfmask = ((size_t)1 << bflog) - 1;
    size_t const pos = (hash >> 2) & bfmask;

    int const existingCandidates = ((((unsigned char*)bf)[pos]) >> shift) & 3;

    static const int addCandidates[4] = { 0, 2, 1, 1 };
    static const int nextValue[4] = { 1, 2, 3, 3 };

    ((unsigned char*)bf)[pos] |= (unsigned char)(nextValue[existingCandidates] << shift);
    return addCandidates[existingCandidates];
}

/*
 * Check if provided 64-bit hash is a collision candidate
 * Requires the slot to be occupied by at least 2 candidates.
 * return >0 if hash is a collision candidate
 *         0 otherwise (slot unoccupied, or only one candidate)
 * note: unoccupied slots should not happen in this algorithm,
 *       since all hashes are supposed to have been inserted at least once.
 */
inline int Filter_check(const Filter* bf, int bflog, uint64_t hash)
{
    int const slotNb = hash & 3;
    int const shift = slotNb * 2;

    size_t const bfmask = ((size_t)1 << bflog) - 1;
    size_t const pos = (hash >> 2) & bfmask;

    return (((const unsigned char*)bf)[pos]) >> (shift+1) & 1;
}

#else

/*
 * 2-probes strategy,
 * more efficient at filtering candidates,
 * requires filter size to be > nb of hashes
 */

#define MIN(a,b)   ((a) < (b) ? (a) : (b))
#define MAX(a,b)   ((a) > (b) ? (a) : (b))

/*
 * Attach hash to 2 slots
 * return: Nb of potential candidates detected
 *          0: position not yet occupied
 *          2: position previously occupied by a single candidate (at most)
 *          1: position already occupied by multiple candidates
 */
static inline int Filter_insert(Filter* bf, int bflog, uint64_t hash)
 {
     hash = avalanche64(hash);
     unsigned const slot1 = hash & 255;
     hash >>= 8;
     unsigned const slot2 = hash & 255;
     hash >>= 8;

     size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
     size_t const cacheLineNb = hash & fclmask;

     size_t const pos1 = (cacheLineNb << 6) + (slot1 >> 2);
     unsigned const shift1 = (slot1 & 3) * 2;
     unsigned const ex1 = (bf[pos1] >> shift1) & 3;

     size_t const pos2 = (cacheLineNb << 6) + (slot2 >> 2);
     unsigned const shift2 = (slot2 & 3) * 2;
     unsigned const ex2 = (bf[pos2] >> shift2) & 3;

     unsigned const existing = MIN(ex1, ex2);

     static const int addCandidates[4] = { 0, 2, 1, 1 };
     static const unsigned nextValue[4] = { 1, 2, 3, 3 };

     bf[pos1] &= (Filter)(~(3 << shift1)); /* erase previous value */
     bf[pos1] |= (Filter)(MAX(ex1, nextValue[existing]) << shift1);
     bf[pos2] |= (Filter)(MAX(ex2, nextValue[existing]) << shift2);

     return addCandidates[existing];
 }


/*
 * Check if provided 64-bit hash is a collision candidate
 * Requires the slot to be occupied by at least 2 candidates.
 * return >0 if hash is a collision candidate
 *         0 otherwise (slot unoccupied, or only one candidate)
 * note: unoccupied slots should not happen in this algorithm,
 *       since all hashes are supposed to have been inserted at least once.
 */
static inline int Filter_check(const Filter* bf, int bflog, uint64_t hash)
 {
     hash = avalanche64(hash);
     unsigned const slot1 = hash & 255;
     hash >>= 8;
     unsigned const slot2 = hash & 255;
     hash >>= 8;

     size_t const fclmask = ((size_t)1 << (bflog-6)) - 1;
     size_t const cacheLineNb = hash & fclmask;

     size_t const pos1 = (cacheLineNb << 6) + (slot1 >> 2);
     unsigned const shift1 = (slot1 & 3) * 2;
     unsigned const ex1 = (bf[pos1] >> shift1) & 3;

     size_t const pos2 = (cacheLineNb << 6) + (slot2 >> 2);
     unsigned const shift2 = (slot2 & 3) * 2;
     unsigned const ex2 = (bf[pos2] >> shift2) & 3;

     return (ex1 >= 2) && (ex2 >= 2);
 }

#endif // FILTER_1_PROBE


/* ===  Display  === */

#include <time.h>   /* clock_t, clock, time_t, time, difftime */

void update_indicator(uint64_t v, uint64_t total)
{
    static clock_t start = 0;
    if (start==0) start = clock();
    clock_t const updateRate = CLOCKS_PER_SEC / 2;

    clock_t const clockSpan = (clock_t)(clock() - start);
    if (clockSpan > updateRate) {
        start = clock();
        assert(v <= total);
        assert(total > 0);
        double share = ((double)v / (double)total) * 100;
        printf("%6.2f%% (%llu)  \r", share, (unsigned long long)v);
        fflush(NULL);
    }
}

/* note: not thread safe */
const char* displayDelay(double delay_s)
{
    static char delayString[50];
    memset(delayString, 0, sizeof(delayString));

    int const mn = ((int)delay_s / 60) % 60;
    int const h = (int)delay_s / 3600;
    int const sec = (int)delay_s % 60;

    char* p = delayString;
    if (h) sprintf(p, "%i h ", h);
    if (mn || h) {
        p = delayString + strlen(delayString);
        sprintf(p, "%i mn ", mn);
    }
    p = delayString + strlen(delayString);
    sprintf(p, "%is ", sec);

    return delayString;
}


/* ===  Math  === */

static double power(uint64_t base, int p)
{
    double value = 1;
    assert(p>=0);
    for (int i=0; i<p; i++) {
        value *= (double)base;
    }
    return value;
}

static double estimateNbCollisions(uint64_t nbH, int nbBits)
{
    return ((double)nbH * (double)(nbH-1)) / power(2, nbBits+1);
}

static int highestBitSet(uint64_t v)
{
    assert(v!=0);
    int bitId = 0;
    while (v >>= 1) bitId++;
    return bitId;
}


/* ===  Filter and search collisions  === */

#undef NDEBUG   /* ensure assert is not disabled */
#include <assert.h>

/* will recommend 24 billion samples for 64-bit hashes,
 * expecting 18 collisions for a good 64-bit hash */
#define NB_BITS_MAX 64   /* can't store nor analyze hash wider than 64-bits for the time being */
uint64_t select_nbh(int nbBits)
{
    assert(nbBits > 0);
    if (nbBits > NB_BITS_MAX) nbBits = NB_BITS_MAX;
    double targetColls = (double)((128 + 17) - (nbBits * 2));
    uint64_t nbH = 24;
    while (estimateNbCollisions(nbH, nbBits) < targetColls) nbH *= 2;
    return nbH;
}


typedef struct {
    uint64_t nbCollisions;
} searchCollisions_results;

typedef struct {
    uint64_t nbH;
    uint64_t mask;
    uint64_t maskSelector;
    size_t sampleSize;
    uint64_t prngSeed;
    int filterLog;      /* <0 = disable filter;  0 = auto-size; */
    int hashID;
    int display;
    int nbThreads;
    searchCollisions_results* resultPtr;
} searchCollisions_parameters;

#define DISPLAY(...) { if (display) printf(__VA_ARGS__); }

static int isEqual(void* hTablePtr, size_t index1, size_t index2, Htype_e htype)
{
    if ((htype == ht64) || (htype == ht32)) {
        uint64_t const h1 = ((const uint64_t*)hTablePtr)[index1];
        uint64_t const h2 = ((const uint64_t*)hTablePtr)[index2];
        return (h1 == h2);
    } else {
        assert(htype == ht128);
        XXH128_hash_t const h1 = ((const XXH128_hash_t*)hTablePtr)[index1];
        XXH128_hash_t const h2 = ((const XXH128_hash_t*)hTablePtr)[index2];
        return XXH128_isEqual(h1, h2);
    }
}

static int isHighEqual(void* hTablePtr, size_t index1, size_t index2, Htype_e htype, int rShift)
{
    uint64_t h1, h2;
    if ((htype == ht64) || (htype == ht32)) {
        h1 = ((const uint64_t*)hTablePtr)[index1];
        h2 = ((const uint64_t*)hTablePtr)[index2];
    } else {
        assert(htype == ht128);
        h1 = ((const XXH128_hash_t*)hTablePtr)[index1].high64;
        h2 = ((const XXH128_hash_t*)hTablePtr)[index2].high64;
        assert(rShift >= 64);
        rShift -= 64;
    }
    assert(0 <= rShift && rShift < 64);
    return (h1 >> rShift) == (h2 >> rShift);
}

/* assumption: (htype*)hTablePtr[index] is valid */
static void addHashCandidate(void* hTablePtr, UniHash h, Htype_e htype, size_t index)
{
    if ((htype == ht64) || (htype == ht32)) {
        ((uint64_t*)hTablePtr)[index] = h.h64;
    } else {
        assert(htype == ht128);
        ((XXH128_hash_t*)hTablePtr)[index] = h.h128;
    }
}

static int getNbBits_fromHtype(Htype_e htype) {
    switch(htype) {
        case ht32: return 32;
        case ht64: return 64;
        case ht128:return 128;
        default: EXIT("hash size not supported");
    }
}

static Htype_e getHtype_fromHbits(int nbBits) {
    switch(nbBits) {
        case 32 : return ht32;
        case 64 : return ht64;
        case 128: return ht128;
        default: EXIT("hash size not supported");
    }
}

static size_t search_collisions(
    searchCollisions_parameters param)
{
    uint64_t totalH = param.nbH;
    const uint64_t hMask = param.mask;
    const uint64_t hSelector = param.maskSelector;
    int bflog = param.filterLog;
    const int filter = (param.filterLog >= 0);
    const size_t sampleSize = param.sampleSize;
    const int hashID = param.hashID;
    const Htype_e htype = getHtype_fromHbits(hashfnTable[hashID].bits);
    const int display = param.display;
    /* init */
    sampleFactory* const sf = create_sampleFactory(sampleSize, totalH, param.prngSeed);
    if (!sf) EXIT("not enough memory");

    //const char* const hname = hashfnTable[hashID].name;
    hashfn const hfunction = hashfnTable[hashID].fn;
    int const hwidth = hashfnTable[hashID].bits;
    if (totalH == 0) totalH = select_nbh(hwidth);
    if (bflog == 0) bflog = highestBitSet(totalH) + 1;   /* auto-size filter */
    uint64_t const bfsize = (1ULL << bflog);


    /* ===  filter hashes (optional)  === */

    Filter* bf = NULL;
    uint64_t nbPresents = totalH;

    if (filter) {
        time_t const filterTBegin = time(NULL);
        DISPLAY(" Creating filter (%i GB) \n", (int)(bfsize >> 30));
        bf = create_Filter(bflog);
        if (!bf) EXIT("not enough memory for filter");


        DISPLAY(" Generate %llu hashes from samples of %u bytes \n",
                (unsigned long long)totalH, (unsigned)sampleSize);
        nbPresents = 0;

        for (uint64_t n=0; n < totalH; n++) {
            if (display && ((n&0xFFFFF) == 1) )
                update_indicator(n, totalH);
            update_sampleFactory(sf);

            UniHash const h = hfunction(sf->buffer, sampleSize);
            if ((h.h64 & hMask) != hSelector) continue;

            nbPresents += (uint64_t)Filter_insert(bf, bflog, h.h64);
        }

        if (nbPresents==0) {
            DISPLAY(" Analysis completed: No collision detected \n");
            if (param.resultPtr) param.resultPtr->nbCollisions = 0;
            free_Filter(bf);
            free_sampleFactory(sf);
            return 0;
        }

        {   double const filterDelay = difftime(time(NULL), filterTBegin);
            DISPLAY(" Generation and filter completed in %s, detected up to %llu candidates \n",
                    displayDelay(filterDelay), (unsigned long long) nbPresents);
    }   }


    /* === store hash candidates: duplicates will be present here === */

    time_t const storeTBegin = time(NULL);
    size_t const hashByteSize = (htype == ht128) ? 16 : 8;
    size_t const tableSize = (nbPresents+1) * hashByteSize;
    assert(tableSize > nbPresents);  /* check tableSize calculation overflow */
    DISPLAY(" Storing hash candidates (%i MB) \n", (int)(tableSize >> 20));

    /* Generate and store hashes */
    void* const hashCandidates = malloc(tableSize);
    if (!hashCandidates) EXIT("not enough memory to store candidates");
    init_sampleFactory(sf, totalH);
    size_t nbCandidates = 0;
    for (uint64_t n=0; n < totalH; n++) {
        if (display && ((n&0xFFFFF) == 1) ) update_indicator(n, totalH);
        update_sampleFactory(sf);

        UniHash const h = hfunction(sf->buffer, sampleSize);
        if ((h.h64 & hMask) != hSelector) continue;

        if (filter) {
            if (Filter_check(bf, bflog, h.h64)) {
                assert(nbCandidates < nbPresents);
                addHashCandidate(hashCandidates, h, htype, nbCandidates++);
            }
        } else {
            assert(nbCandidates < nbPresents);
            addHashCandidate(hashCandidates, h, htype, nbCandidates++);
        }
    }
    if (nbCandidates < nbPresents) {
        /* Try to mitigate gnuc_quicksort behavior, by reducing allocated memory,
         * since gnuc_quicksort uses a lot of additional memory for mergesort */
        void* const checkPtr = realloc(hashCandidates, nbCandidates * hashByteSize);
        assert(checkPtr != NULL);
        assert(checkPtr == hashCandidates);  /* simplification: since we are reducing the size,
                                              * we hope to keep the same ptr position.
                                              * Otherwise, hashCandidates must be mutable. */
        DISPLAY(" List of hashes reduced to %u MB from %u MB (saved %u MB) \n",
                (unsigned)((nbCandidates * hashByteSize) >> 20),
                (unsigned)(tableSize >> 20),
                (unsigned)((tableSize - (nbCandidates * hashByteSize)) >> 20) );
    }
    double const storeTDelay = difftime(time(NULL), storeTBegin);
    DISPLAY(" Stored %llu hash candidates in %s \n",
            (unsigned long long) nbCandidates, displayDelay(storeTDelay));
    free_Filter(bf);
    free_sampleFactory(sf);


    /* === step 3: look for duplicates === */
    time_t const sortTBegin = time(NULL);
    DISPLAY(" Sorting candidates... ");
    fflush(NULL);
    if ((htype == ht64) || (htype == ht32)) {
        /*
         * Use C++'s std::sort, as it's faster than C stdlib's qsort, and
         * doesn't suffer from gnuc_libsort's memory expansion
         */
        sort64(hashCandidates, nbCandidates);
    } else {
        assert(htype == ht128);
        sort128(hashCandidates, nbCandidates); /* sort with custom comparator */
    }
    double const sortTDelay = difftime(time(NULL), sortTBegin);
    DISPLAY(" Completed in %s \n", displayDelay(sortTDelay));

    /* scan and count duplicates */
    time_t const countBegin = time(NULL);
    DISPLAY(" Looking for duplicates: ");
    fflush(NULL);
    size_t collisions = 0;
    for (size_t n=1; n<nbCandidates; n++) {
        if (isEqual(hashCandidates, n, n-1, htype)) {
#if defined(COL_DISPLAY_DUPLICATES)
            printf("collision: ");
            printHash(hashCandidates, n, htype);
            printf(" / ");
            printHash(hashCandidates, n-1, htype);
            printf(" \n");
#endif
            collisions++;
    }   }

    if (!filter /* all candidates */ && display /*single thead*/ ) {
        /* check partial bitfields (high bits) */
        DISPLAY(" \n");
        int const hashBits = getNbBits_fromHtype(htype);
        double worstRatio = 0.;
        int worstNbHBits = 0;
        for (int nbHBits = 1; nbHBits < hashBits; nbHBits++) {
            uint64_t const nbSlots = (uint64_t)1 << nbHBits;
            double const expectedCollisions = estimateNbCollisions(nbCandidates, nbHBits);
            if ( (nbSlots > nbCandidates * 100)  /* within range for meaningfull collision analysis results */
              && (expectedCollisions > 18.0) ) {
                int const rShift = hashBits - nbHBits;
                size_t HBits_collisions = 0;
                for (size_t n=1; n<nbCandidates; n++) {
                    if (isHighEqual(hashCandidates, n, n-1, htype, rShift)) {
                        HBits_collisions++;
                }   }
                double const collisionRatio = (double)HBits_collisions / expectedCollisions;
                if (collisionRatio > 2.0) DISPLAY("WARNING !!!  ===> ");
                DISPLAY(" high %i bits: %zu collision (%.1f expected): x%.2f \n",
                        nbHBits, HBits_collisions, expectedCollisions, collisionRatio);
                if (collisionRatio > worstRatio) {
                    worstNbHBits = nbHBits;
                    worstRatio = collisionRatio;
        }   }   }
        DISPLAY("Worst collision ratio at %i high bits: x%.2f \n",
                worstNbHBits, worstRatio);
    }
    double const countDelay = difftime(time(NULL), countBegin);
    DISPLAY(" Completed in %s \n", displayDelay(countDelay));

    /* clean and exit */
    free (hashCandidates);

#if 0  /* debug */
    for (size_t n=0; n<nbCandidates; n++)
        printf("0x%016llx \n", (unsigned long long)hashCandidates[n]);
#endif

    if (param.resultPtr) param.resultPtr->nbCollisions = collisions;
    return collisions;
}



#if defined(__MACH__) || defined(__linux__)
#include <sys/resource.h>
static size_t getProcessMemUsage(int children)
{
    struct rusage stats;
    if (getrusage(children ? RUSAGE_CHILDREN : RUSAGE_SELF, &stats) == 0)
      return (size_t)stats.ru_maxrss;
    return 0;
}
#else
static size_t getProcessMemUsage(int ignore) { return 0; }
#endif

void time_collisions(searchCollisions_parameters param)
{
    uint64_t totalH = param.nbH;
    int hashID = param.hashID;
    int display = param.display;

    /* init */
    assert(0 <= hashID && hashID < HASH_FN_TOTAL);
    //const char* const hname = hashfnTable[hashID].name;
    int const hwidth = hashfnTable[hashID].bits;
    if (totalH == 0) totalH = select_nbh(hwidth);
    double const targetColls = estimateNbCollisions(totalH, hwidth);

    /* Start the timer to measure start/end of hashing + collision detection. */
    time_t const programTBegin = time(NULL);

    /* Generate hashes, and count collisions */
    size_t const collisions = search_collisions(param);

    /* display results */
    double const programTDelay = difftime(time(NULL), programTBegin);
    size_t const programBytesSelf = getProcessMemUsage(0);
    size_t const programBytesChildren = getProcessMemUsage(1);
    DISPLAY("\n\n");
    DISPLAY("===>   Found %llu collisions (x%.2f, %.1f expected) in %s\n",
            (unsigned long long)collisions,
            (double)collisions / targetColls,
            targetColls,
            displayDelay(programTDelay));
    if (programBytesSelf)
      DISPLAY("===>   MaxRSS(self) %zuMB, MaxRSS(children) %zuMB\n",
              programBytesSelf>>20,
              programBytesChildren>>20);
    DISPLAY("------------------------------------------ \n");
}

// wrapper for pthread interface
void MT_searchCollisions(void* payload)
{
    search_collisions(*(searchCollisions_parameters*)payload);
}

/* ===  Command Line  === */

/*!
 * readU64FromChar():
 * Allows and interprets K, KB, KiB, M, MB and MiB suffix.
 * Will also modify `*stringPtr`, advancing it to the position where it stopped reading.
 */
static uint64_t readU64FromChar(const char** stringPtr)
{
    static uint64_t const max = (((uint64_t)(-1)) / 10) - 1;
    uint64_t result = 0;
    while ((**stringPtr >='0') && (**stringPtr <='9')) {
        assert(result < max);
        result *= 10;
        result += (unsigned)(**stringPtr - '0');
        (*stringPtr)++ ;
    }
    if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
        uint64_t const maxK = ((uint64_t)(-1)) >> 10;
        assert(result < maxK);
        result <<= 10;
        if ((**stringPtr=='M') || (**stringPtr=='G')) {
            assert(result < maxK);
            result <<= 10;
            if (**stringPtr=='G') {
                assert(result < maxK);
                result <<= 10;
            }
        }
        (*stringPtr)++;  /* skip `K` or `M` */
        if (**stringPtr=='i') (*stringPtr)++;
        if (**stringPtr=='B') (*stringPtr)++;
    }
    return result;
}


/**
 * longCommandWArg():
 * Checks if *stringPtr is the same as longCommand.
 * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
 * @return 0 and doesn't modify *stringPtr otherwise.
 */
static int longCommandWArg(const char** stringPtr, const char* longCommand)
{
    assert(longCommand); assert(stringPtr); assert(*stringPtr);
    size_t const comSize = strlen(longCommand);
    int const result = !strncmp(*stringPtr, longCommand, comSize);
    if (result) *stringPtr += comSize;
    return result;
}


#include "pool.h"

/*
 * As some hashes use different algorithms depending on input size,
 * it can be necessary to test multiple input sizes
 * to paint an accurate picture of collision performance
 */
#define SAMPLE_SIZE_DEFAULT 256
#define HASHFN_ID_DEFAULT 0

void help(const char* exeName)
{
    printf("usage: %s [hashName] [opt] \n\n", exeName);
    printf("list of hashNames:");
    printf("%s ", hashfnTable[0].name);
    for (int i=1; i < HASH_FN_TOTAL; i++) {
        printf(", %s ", hashfnTable[i].name);
    }
    printf(" \n");
    printf("Default hashName is %s\n", hashfnTable[HASHFN_ID_DEFAULT].name);

    printf(" \n");
    printf("Optional parameters: \n");
    printf("  --nbh=NB       Select nb of hashes to generate (%llu by default) \n", (unsigned long long)select_nbh(64));
    printf("  --filter       Activates the filter. Slower, but reduces memory usage for the same nb of hashes.\n");
    printf("  --threadlog=NB Use 2^NB threads.\n");
    printf("  --len=MB       Set length of the input (%i bytes by default) \n", SAMPLE_SIZE_DEFAULT);
}

int bad_argument(const char* exeName)
{
    printf("incorrect command: \n");
    help(exeName);
    return 1;
}


int main(int argc, const char** argv)
{
    if (sizeof(size_t) < 8) return 1;  // cannot work on systems without ability to allocate objects >= 4 GB

    assert(argc > 0);
    const char* const exeName = argv[0];
    uint64_t totalH = 0;  /* auto, based on nbBits */
    int bflog = 0;    /* auto */
    int filter = 0;   /* disabled */
    size_t sampleSize = SAMPLE_SIZE_DEFAULT;
    int hashID = HASHFN_ID_DEFAULT;
    int threadlog = 0;
    uint64_t prngSeed = 0;

    int arg_nb;
    for (arg_nb = 1; arg_nb < argc; arg_nb++) {
        const char** arg = argv + arg_nb;

        if (!strcmp(*arg, "-h")) { help(exeName); return 0; }
        if (longCommandWArg(arg, "-T")) { threadlog = (int)readU64FromChar(arg); continue; }

        if (!strcmp(*arg, "--filter"))    { filter=1; continue; }
        if (!strcmp(*arg, "--no-filter")) { filter=0; continue; }

        if (longCommandWArg(arg, "--seed")) { prngSeed = readU64FromChar(arg); continue; }
        if (longCommandWArg(arg, "--nbh=")) { totalH = readU64FromChar(arg); continue; }
        if (longCommandWArg(arg, "--filter=")) { filter=1; bflog = (int)readU64FromChar(arg); assert(bflog < 64); continue; }
        if (longCommandWArg(arg, "--filterlog=")) { filter=1; bflog = (int)readU64FromChar(arg); assert(bflog < 64); continue; }
        if (longCommandWArg(arg, "--size=")) { sampleSize = (size_t)readU64FromChar(arg); continue; }
        if (longCommandWArg(arg, "--len=")) { sampleSize = (size_t)readU64FromChar(arg); continue; }
        if (longCommandWArg(arg, "--threadlog=")) { threadlog = (int)readU64FromChar(arg); continue; }

        /* argument understood as hash name (must be correct) */
        int hnb;
        for (hnb=0; hnb < HASH_FN_TOTAL; hnb++) {
            if (!strcmp(*arg, hashfnTable[hnb].name)) { hashID = hnb; break; }
        }
        if (hnb == HASH_FN_TOTAL) return bad_argument(exeName);
    }

    /* init */
    const char* const hname = hashfnTable[hashID].name;
    int const hwidth = hashfnTable[hashID].bits;
    if (totalH == 0) totalH = select_nbh(hwidth);
    double const targetColls = estimateNbCollisions(totalH, hwidth);
    if (bflog == 0) bflog = highestBitSet(totalH) + 1;   /* auto-size filter */
    if (!filter) bflog = -1; // disable filter

    if (sizeof(size_t) < 8)
      EXIT("This program has not been validated on architectures other than "
           "64bit \n");

    printf(" *** Collision tester for 64+ bit hashes ***  \n\n");
    printf("Testing %s algorithm (%i-bit) \n", hname, hwidth);
    printf("This program will allocate a lot of memory,\n");
    printf("generate %llu %i-bit hashes from samples of %u bytes, \n",
            (unsigned long long)totalH, hwidth, (unsigned)sampleSize);
    printf("and attempt to produce %.0f collisions. \n\n", targetColls);

    int const nbThreads = 1 << threadlog;
    if (nbThreads <= 0) EXIT("Invalid --threadlog value.");

    if (nbThreads == 1) {

        searchCollisions_parameters params;
        params.nbH = totalH;
        params.mask = 0;
        params.maskSelector = 0;
        params.sampleSize = sampleSize;
        params.filterLog = bflog;
        params.hashID = hashID;
        params.display = 1;
        params.resultPtr = NULL;
        params.prngSeed = prngSeed;
        params.nbThreads = 1;
        time_collisions(params);

    } else { /*  nbThreads > 1 */

        /* use multithreading */
        if (threadlog >= 30) EXIT("too many threads requested");
        if ((uint64_t)nbThreads > (totalH >> 16))
            EXIT("too many threads requested");
        if (bflog > 0 && threadlog > (bflog-10))
            EXIT("too many threads requested");
        printf("using %i threads ... \n", nbThreads);

        /* allocation */
        time_t const programTBegin = time(NULL);
        POOL_ctx* const pt = POOL_create((size_t)nbThreads, 1);
        if (!pt) EXIT("not enough memory for threads");
        searchCollisions_results* const MTresults = calloc (sizeof(searchCollisions_results), (size_t)nbThreads);
        if (!MTresults) EXIT("not enough memory");
        searchCollisions_parameters* const MTparams = calloc (sizeof(searchCollisions_parameters), (size_t)nbThreads);
        if (!MTparams) EXIT("not enough memory");

        /* distribute jobs */
        for (int tnb=0; tnb<nbThreads; tnb++) {
            MTparams[tnb].nbH = totalH;
            MTparams[tnb].mask = (uint64_t)nbThreads - 1;
            MTparams[tnb].sampleSize = sampleSize;
            MTparams[tnb].filterLog = bflog ? bflog - threadlog : 0;
            MTparams[tnb].hashID = hashID;
            MTparams[tnb].display = 0;
            MTparams[tnb].resultPtr = MTresults+tnb;
            MTparams[tnb].prngSeed = prngSeed;
            MTparams[tnb].maskSelector = (uint64_t)tnb;
            POOL_add(pt, MT_searchCollisions, MTparams + tnb);
        }
        POOL_free(pt);  /* actually joins and free */

        /* Gather results */
        uint64_t nbCollisions=0;
        for (int tnb=0; tnb<nbThreads; tnb++) {
            nbCollisions += MTresults[tnb].nbCollisions;
        }

        double const programTDelay = difftime(time(NULL), programTBegin);
        size_t const programBytesSelf = getProcessMemUsage(0);
        size_t const programBytesChildren = getProcessMemUsage(1);
        printf("\n\n");
        printf("===>   Found %llu collisions (x%.2f, %.1f expected) in %s\n",
                (unsigned long long)nbCollisions,
                (double)nbCollisions / targetColls,
                targetColls,
                displayDelay(programTDelay));
        if (programBytesSelf)
          printf("===>   MaxRSS(self) %zuMB, MaxRSS(children) %zuMB\n",
                 programBytesSelf>>20,
                 programBytesChildren>>20);
        printf("------------------------------------------ \n");

        /* Clean up */
        free(MTparams);
        free(MTresults);
    }

    return 0;
}