popufare/.legacy/busunit/passdb_slim/passdb_slim.c

/*
 * Copyright (c) 2019 Clementine Computing LLC.
 *
 * This file is part of PopuFare.
 *
 * PopuFare is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * PopuFare is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with PopuFare.  If not, see <https://www.gnu.org/licenses/>.
 *
 */

#include <sys/user.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <errno.h>

#include <time.h>
#include <sys/time.h>

#include "../common/common_defs.h"
#include "passdb_slim.h"

int read_config( passdb_slim_config *cfg, char *config_fn );
int passdb_slim_manage_rider_banks(passdb_slim_context *ctx);


static void free_rider_node_list(rider_node *head)
{
    rider_node *p = head;
    rider_node *q;

    while(p)
    {
        q = p;
        p = p->next;

        free(q);
    }
}

#define FIND_IDX_IN_BUCKET(b, idx, p, q)\
    {					\
        p = b;				\
        q = NULL;			\
                                        \
        while(p)			\
        {				\
            if(p->idx == idx)		\
            {				\
                break;			\
            }				\
                                        \
            q = p;			\
            p = p->next;		\
        }				\
    }					\
//---------------------------

#define ADD_TO_BUCKET(b, idx, p, q)	\
    {					\
        p = (rider_node *) malloc( sizeof(rider_node) );\
                                        \
        if(p == NULL) return FAIL_MEM;	\
                                        \
        p->next = NULL;			\
        p->idx = idx;			\
                                        \
        if(q)				\
        {				\
            q->next = p;		\
        }				\
        else				\
        {				\
            b = p;			\
        }				\
    }					\
//---------------------------

#define DEL_FROM_BUCKET(b, p, q)	\
    {					\
        if(q)				\
        {				\
            q->next = p->next;		\
        }				\
        else				\
        {				\
            b = p->next;		\
        }				\
                                        \
        free(p);			\
    }    				\
//---------------------------


//#define INDEX_MIDPOINT 500000

int make_rider_record_from_rider_one_cred( passdb_slim_context *ctx, rider_record *rr, rider_record_slim_one_cred *rr1 )
{
  long long unsigned llu;
  long unsigned a, b;
  int retval;

  memset(rr, 0, sizeof(rider_record));
  if (rr1->id == ID_INVALID)
    return WARN_NOTFOUND;

  rr->seq = rr1->seq;
  rr->id = rr1->id;

  if (rr1->code < 4)
  {
    rr->rfid_value[0] = '\0';
    snprintf(rr->magstripe_value, CREDENTIAL_LEN, "%i:%llu", rr1->code, rr1->credential );
  }
  else
  {
    llu = 0xffffffffL;
    b = (long unsigned)(rr1->credential & llu);
    llu = llu << 32;
    a = (long unsigned)( (rr1->credential & llu) >> 32 );

    rr->magstripe_value[0] = '\0';
    snprintf(rr->rfid_value, CREDENTIAL_LEN, "%i:%lu:%lu", rr1->code, a, b );
  }


  retval = ruleparam_db_get( rr->rule_name, rr->rule_param, ctx->ruleparam_db, rr1->rule_param_bucket_id );
  if (retval < 0)
  {
    fprintf(stderr, "ERROR: make_rider_record_from_rider_one_cred: no mapping for rr1->rule_param_bucket_id (%i), got %i\n",
        rr1->rule_param_bucket_id, retval);

    //DEBUG
    fprintf(stderr, "ERROR: mrrfroc: ruleid %i, got %i rr1( seq:%llu, id:%llu, %u:%llu, %i)\n",
        rr1->rule_param_bucket_id, retval,
        rr1->seq, rr1->id,
        (unsigned int)rr1->code, rr1->credential,
        (unsigned int)rr1->rule_param_bucket_id
        );

    return retval;

  }

  return 0;

}

//--

int make_rider_record_from_rider_two_cred( passdb_slim_context *ctx, rider_record *rr, rider_record_slim_two_cred *rr2 )
{
  int retval;

  memset(rr, 0, sizeof(rider_record));
  if (rr2->id == ID_INVALID)
    return WARN_NOTFOUND;

  rr->seq = rr2->seq;
  rr->id = rr2->id;

  snprintf(rr->magstripe_value, 
           CREDENTIAL_LEN, 
           "%i:%llu",    
           rr2->magstripe_code, rr2->magstripe );

  snprintf(rr->rfid_value,
           CREDENTIAL_LEN, 
           "%i:%lu:%lu", 
           rr2->rfid_code, rr2->rfid_site, rr2->rfid_val );

  retval = ruleparam_db_get( rr->rule_name, rr->rule_param, ctx->ruleparam_db, rr2->rule_param_bucket_id );
  if (retval < 0)
  {
    fprintf(stderr, "ERROR: make_rider_record_from_rider_two_cred: no mapping for rr2->rule_param_bucket_id (%i), got %i\n",
        rr2->rule_param_bucket_id, retval);

    //DEBUG
    fprintf(stderr, "ERROR: mrrfrtc: ruleid %i, got %i rr2( seq:%llu, id:%llu, %u:%lu:%lu %u:%llu, %i)\n",
        rr2->rule_param_bucket_id, retval,
        rr2->seq, rr2->id,
        (unsigned int)rr2->rfid_code, rr2->rfid_site, rr2->rfid_val,
        (unsigned int)rr2->magstripe_code, rr2->magstripe,
        (unsigned int)rr2->rule_param_bucket_id
        );


    return retval;

  }

  return 0;

}

//--

// UNDER DEVELOPMENT
//
//int make_rider_record_from_rider_spillover( passdb_slim_context *ctx, rider_record *rr, rider_record_slim_spillover *rr2 )
int make_rider_record_from_rider_spillover( passdb_slim_context *ctx, rider_record *rr, rider_record *rr_spillover )
{
  int bucket_id;

  memcpy( rr, rr_spillover, sizeof(rider_record) );

  bucket_id = ruleparam_db_find( ctx->ruleparam_db, rr->rule_name, rr->rule_param );
  if (bucket_id == RULEPARAM_DB_NOT_FOUND )
  {
    fprintf(stderr, "ERROR: make_rider_record_from_rider_spillover: no mapping for rule/param (%s,%s), got %i\n",
        rr->rule_name,
        rr->rule_param,
        bucket_id);
    return -1;

  }

  return 0;

}


//--



void populate_one_cred_rider_record( 
    rider_record_slim_one_cred *rr_one, 
    int pos, 
    void *rider_p )
{
  void *p;

  memset(rr_one, 0, sizeof(rider_record_slim_one_cred));

  p = rider_p + (pos*RIDER_ONE_CRED_SIZE);

  //rr_one->seq = (seq_t)( *((seq_t *)p) );
  rr_one->seq = _ulli(p);
  p += sizeof(seq_t);

  //rr_one->id = (logical_card_id_t)( *((logical_card_id_t *)p) );
  rr_one->id = _ulli(p);
  p += sizeof(logical_card_id_t);

  rr_one->code = (unsigned char)( *((unsigned char *)p) );
  p += sizeof(unsigned char);

  //rr_one->credential = (unsigned long long)( *((unsigned long long *)p) );
  rr_one->credential = _ulli(p);
  p += sizeof(unsigned long long);

  //rr_one->rule_param_bucket_id = (unsigned short int)( *((unsigned short int *)p) );
  rr_one->rule_param_bucket_id = _usi(p);
  //p += sizeof(unsigned short int);

}

void populate_two_cred_rider_record( rider_record_slim_two_cred *rr_two, int pos, void *rider_p)
{
  void *p;

  memset(rr_two, 0, sizeof(rider_record_slim_two_cred));

  p = rider_p + (pos*RIDER_TWO_CRED_SIZE);

  //rr_two->seq = (seq_t)( *((seq_t *)p) );
  rr_two->seq = _ulli(p);
  p += sizeof(seq_t);

  //rr_two->id = (logical_card_id_t)( *((logical_card_id_t *)p) );
  rr_two->id = _ulli(p);
  p += sizeof(logical_card_id_t);

  rr_two->magstripe_code = (unsigned char)( *((unsigned char *)p) );
  p += sizeof(unsigned char);

  //rr_two->magstripe = (unsigned long long)( *((unsigned long long *)p) );
  rr_two->magstripe = _ulli(p);
  p += sizeof(unsigned long long);

  rr_two->rfid_code = (unsigned char)( *((unsigned char *)p) );
  p += sizeof(unsigned char);

  //rr_two->rfid_site = (unsigned long)( *((unsigned long *)p) );
  rr_two->rfid_site = _uli(p);
  p += sizeof(unsigned long);

  //rr_two->rfid_val = (unsigned long)( *((unsigned long *)p) );
  rr_two->rfid_val = _uli(p);
  p += sizeof(unsigned long);

  //rr_two->rule_param_bucket_id = (unsigned short int)( *((unsigned short int *)p) );
  rr_two->rule_param_bucket_id = _usi(p);
  //p += sizeof(unsigned short int);

}

// ---


void populate_spillover_rider_record( rider_record *rr, int pos, void *rider_p)
{
  memcpy(rr, rider_p + (pos*RIDER_SPILLOVER_SIZE), sizeof(rider_record) );
}


// ---


void *passdb_slim_get_record_address( passdb_slim_context *ctx, int idx )
{
  int q,r;

  if (idx < INDEX_MIDPOINT)
  {
    q = idx / ctx->n_one_cred_bank_size;
    r = idx - (q * ctx->n_one_cred_bank_size);
      
    if ( (q<0) || (q >= ctx->n_one_cred_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_record_address, bank out of range (%i<0 or %i>%i) (%i,%i) from idx %i\n", q, q, ctx->n_one_cred_bank, q, r, idx);
      return NULL;
    }

    return ctx->rider_one_cred_bank[q] + (r * RIDER_ONE_CRED_SIZE);

  }
  else if ( idx < (2*INDEX_MIDPOINT) )
  {
    q = (idx - INDEX_MIDPOINT) / ctx->n_two_cred_bank_size;
    r = (idx - INDEX_MIDPOINT) - (q * ctx->n_two_cred_bank_size);

    if ( (q<0) || (q >= ctx->n_two_cred_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_record_address, bank out of range (%i<0 or %i>%i) (%i,%i) from idx %i\n", q,q, ctx->n_two_cred_bank, q, r, idx);
      return NULL;
    }

    return ctx->rider_two_cred_bank[q] + (r * RIDER_TWO_CRED_SIZE);

  }
  else
  {
    q = (idx - (2*INDEX_MIDPOINT)) / ctx->n_spillover_bank_size;
    r = (idx - (2*INDEX_MIDPOINT)) - (q * ctx->n_spillover_bank_size);

    if ( (q<0) || (q >= ctx->n_spillover_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_record_address, bank out of range (%i<0 or %i>%i) (%i,%i) from idx %i\n", q,q, ctx->n_spillover_bank, q, r, idx);
      return NULL;
    }

    return ctx->rider_spillover_bank[q] + (r * RIDER_SPILLOVER_SIZE);

  }

  return NULL;
}

// Helper function that stores the bank and position of the index.
// That is, find the quotent and remainder of idx (suitably repositioned)
// with the number of entries in the appropriate bank.
// For example, if we have 251 entries per bank, with 3 banks and an index
// of 255, this will store 1 in bank and 4 in pos.
// You must multiply by the value of the record size and offset by the bank
// start poition to get the mmap'd memory location.
//
// If all you want is the memory record location, use the function
// passdb_slim_get_record_address.
//
// Return -1 if the bank number is invalid (less than 0 or greater than 
// the maximum bank number)
//
int passdb_slim_get_cred_bank_and_pos( passdb_slim_context *ctx, int *bank, int *pos, int idx )
{
  int q, r;


  if (idx < INDEX_MIDPOINT)
  {
    q = idx / ctx->n_one_cred_bank_size ;
    r = idx - (q * ctx->n_one_cred_bank_size);


    if ( (q < 0) || (q >= ctx->n_one_cred_max_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_cred_bank_and_pos, bank out of range (%i,%i) from idx %i\n", q, r, idx);
      return -1;
    }

  }
  else if (idx < (2*INDEX_MIDPOINT))
  {
    q = (idx - INDEX_MIDPOINT) / ctx->n_two_cred_bank_size ;
    r = (idx - INDEX_MIDPOINT) - (q * ctx->n_two_cred_bank_size);

    if ( (q < 0) || (q >= ctx->n_two_cred_max_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_cred_bank_and_pos, bank out of range (%i,%i) from idx %i\n", q, r, idx);
      return -1;
    }

  }
  else 
  {
    q = (idx - (2*INDEX_MIDPOINT)) / ctx->n_spillover_bank_size ;
    r = (idx - (2*INDEX_MIDPOINT)) - (q * ctx->n_spillover_bank_size);

    if ( (q < 0) || (q >= ctx->n_spillover_max_bank) )
    {
      fprintf(stderr, "ERROR: passdb_slim_get_cred_bank_and_pos, bank out of range (%i,%i) from idx %i\n", q, r, idx);
      return -1;
    }

  }

  *bank = q;
  *pos = r;

  return 0;
}

//--------------------------------

void make_rider_record( passdb_slim_context *ctx, rider_record *rr, int idx )
{
  int r;
  void *p;
  rider_record_slim_one_cred rr1 = {0};
  rider_record_slim_two_cred rr2 = {0};

  p = passdb_slim_get_record_address( ctx, idx );

  if (p == NULL)
  {
    fprintf(stderr, "make_rider_record: passdb_slim_get_record_address returned NULL! (idx %i)\n", idx);
    return;
  }

  if (idx < INDEX_MIDPOINT )
  {
    populate_one_cred_rider_record( &rr1, 0, p );
    r = make_rider_record_from_rider_one_cred( ctx, rr, &rr1);
  }
  else if (idx < (2*INDEX_MIDPOINT))
  {
    populate_two_cred_rider_record( &rr2, 0, p );
    r = make_rider_record_from_rider_two_cred( ctx, rr, &rr2);
  }
  else
  {
    memcpy( rr, p , sizeof(rider_record) );
  }

}


//----


int find_id_in_hash(passdb_slim_context *ctx, logical_card_id_t id)
{
    rider_record rr = {0};
    rider_node *p;

    if(id <= 0)
        return FAIL_PARAM;

    p = ctx->logical_card_id_hash[id % ctx->hash_modulus];
    while(p)
    {

        make_rider_record( ctx, &rr, p->idx );
        if (rr.id == id)
           return p->idx;

        p = p->next;  
    }

    return WARN_NOTFOUND;
}

int add_to_id_hash(passdb_slim_context *ctx, int idx)
{
    rider_record rr = {0};

    rider_node *p, *q;
    unsigned int bucket;

    if(idx < 0) 
    {
        return FAIL_PARAM;
    }

    make_rider_record( ctx, &rr, idx );

    if (rr.id == ID_INVALID) 
    {
        return FAIL_PARAM;
    }

    bucket = rr.id % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->logical_card_id_hash[bucket], idx, p, q )

    if(p)
    {
        return FAIL_DUPKEY;	//already exists!
    }

    ADD_TO_BUCKET( ctx->logical_card_id_hash[bucket], idx, p, q );
    return 0;
}


int delete_from_id_hash(passdb_slim_context *ctx, int idx)
{
    rider_record rr = {0};

    rider_node *p, *q;
    unsigned int bucket;

    if(idx < 0) return FAIL_PARAM;

    make_rider_record( ctx, &rr, idx );

    bucket = rr.id % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->logical_card_id_hash[bucket], idx, p, q)

    if(p)
    {
        DEL_FROM_BUCKET( ctx->logical_card_id_hash[bucket], p, q )
        return 0;
    }
    else
    {
        return WARN_NOTFOUND;
    }
}

//##

int find_mag_in_hash(passdb_slim_context *ctx, char *mag)
{
    rider_record rr = {0};

    rider_node *p;

    if(mag[0] == '\0')	return FAIL_PARAM;

    p = ctx->rider_mag_hash[stringhash(mag) % ctx->hash_modulus];

    while(p)
    {

        make_rider_record( ctx, &rr, p->idx );

        if(!strncmp(rr.magstripe_value, mag, CREDENTIAL_LEN))
            return p->idx;

        p = p->next;  
    }

    return WARN_NOTFOUND;
}

int add_to_mag_hash(passdb_slim_context *ctx, int idx)
{
    rider_record rr = {0};

    rider_node *p, *q;
    unsigned int bucket;

    if(idx < 0) return 0;	//if we have a non index or a non-value, return silently


    make_rider_record( ctx, &rr, idx );

    if (rr.magstripe_value[0] == '\0') return 0;

    bucket = stringhash( rr.magstripe_value) % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->rider_mag_hash[bucket], idx, p, q )

    #ifndef ALLOW_CREDENTIAL_COLLISIONS
    							//On allowing hash collisions among credentials see comment tagged **STUPID** later in this file.
    if(p)
    {
        return FAIL_DUPKEY;	//already exists!
    }
    else	
    #endif
    {
        ADD_TO_BUCKET( ctx->rider_mag_hash[bucket], idx, p, q );
        return 0;
    }

}

int delete_from_mag_hash(passdb_slim_context *ctx,  int idx)
{
    rider_record rr = {0};

    rider_node *p, *q;
    unsigned int bucket;

    if(idx < 0) return FAIL_PARAM;

    make_rider_record( ctx, &rr, idx );

    bucket = stringhash( rr.magstripe_value ) % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->rider_mag_hash[bucket], idx, p, q)

    if(p)
    {
        DEL_FROM_BUCKET( ctx->rider_mag_hash[bucket], p, q )
        return 0;
    }
    else
    {
        return WARN_NOTFOUND;
    }
}



//##
int find_rf_in_hash(passdb_slim_context *ctx, char *rfid)
{
    rider_record rr = {0} ;

    rider_node *p;

    if(rfid[0] == '\0')	return WARN_NOTFOUND;

    p = ctx->rider_rf_hash[stringhash(rfid)% ctx->hash_modulus];

    while(p)
    {

        make_rider_record( ctx, &rr, p->idx );
        if(!strncmp( rr.rfid_value, rfid, CREDENTIAL_LEN))
            return p->idx;

        p = p->next;  
    }

    return WARN_NOTFOUND;
}

int add_to_rf_hash(passdb_slim_context *ctx, int idx)
{
    rider_record rr = {0};

    rider_node *p, *q;
    unsigned int bucket;

    if(idx < 0) return 0;	//if we have a non index or a non-value, return silently

    make_rider_record( ctx, &rr, idx );

    if(rr.rfid_value[0] == '\0') return 0;

    bucket = stringhash(rr.rfid_value) % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->rider_rf_hash[bucket], idx, p, q )

    #ifndef ALLOW_CREDENTIAL_COLLISIONS
    							//On allowing hash collisions among credentials see comment tagged **STUPID** later in this file.
    if(p)
    {
        return FAIL_DUPKEY;	//already exists!
    }
    else
    #endif
    {
        ADD_TO_BUCKET( ctx->rider_rf_hash[bucket], idx, p, q );
        return 0;
    }
}


int delete_from_rf_hash(passdb_slim_context *ctx, int idx)
{
    rider_record rr = {0} ;

    rider_node *p, *q;
    unsigned int bucket; 

    if(idx < 0) return FAIL_PARAM;

    make_rider_record( ctx, &rr, idx );

    bucket = stringhash(rr.rfid_value) % ctx->hash_modulus;

    FIND_IDX_IN_BUCKET( ctx->rider_rf_hash[bucket], idx, p, q)

    if(p)
    {
        DEL_FROM_BUCKET( ctx->rider_rf_hash[bucket], p, q )
        return 0;
    }
    else
    {
        return WARN_NOTFOUND;
    }
}

//##

int build_hashes(passdb_slim_context *ctx)
{
    rider_record rr = {0};
    rider_node *p = ctx->activelist;
    int retval;

    while(p)
    {
        make_rider_record( ctx, &rr, p->idx );

        ctx->ruleparam_db->seq = ctx->seq;
        ruleparam_db_update( ctx->ruleparam_db, rr.rule_name, rr.rule_param , 1 );

 
        retval = add_to_id_hash(ctx, p->idx);
        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) indexing rider ID %llu at index %d!\n", retval, rr.id, p->idx);
            return retval;
        }

        retval = add_to_mag_hash(ctx, p->idx);
        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) indexing magstripe %s at index %d!\n", retval, rr.magstripe_value, p->idx);
            return -1;
        }

        retval = add_to_rf_hash(ctx, p->idx);
        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) indexing RFID %s at index %d!\n", retval, rr.rfid_value, p->idx);
            return -1;
        }

        p = p->next;
    }

    // Write reference counts for debugging.
    // Rules and params should have been recorded by 
    // ruleparam_db_update, but we'd like, at least for debugging
    // purposes, to write out the reference counts.  Do this
    // here.
    //

    ruleparam_db_clean( ctx->ruleparam_db );

#ifdef PASSDB_CONSISTENCY_CHECK
    retval = ruleparam_db_consistency_check( ctx->ruleparam_db );
    if (retval < 0)
    {
      fprintf(stderr, "CONSISTENCY CHECK failed in build_hashes: got %i\n", retval);
    }
#endif

    ctx->ruleparam_db->seq = ctx->seq;
    //ruleparam_db_save( ctx->ruleparam_db, ctx->ruleparam_db_fn );
    ruleparam_db_save( ctx->ruleparam_db );

    return 0;
}

// memory page size 4096

int format_new_passdb( char *fn, int sz, long pagesize  )
{
    int i,n;
    int fd;
    char *blank;

    blank = calloc( pagesize, sizeof(char) );
    if (!blank)
      return FAIL_MEM;

    fd = creat(fn, S_IRUSR | S_IWUSR);

    if( fd < 0 )
    {
        fprintf(stderr, "Cannot create pass file %s!\n", fn);
        free(blank);
        return FAIL_DATABASE;
    }

    n = sz / pagesize;
    if ( (sz % pagesize ) > 0 )
    {
      n++;
    }

    for(i = 0; i < n; i++)
    {
        if( write(fd, blank, pagesize) != pagesize )
        {
            fprintf(stderr, "Cannot write blank data to passes file %s!\n", fn);
            free(blank);
            close(fd);
            return FAIL_DATABASE;
        }
    }

    free(blank);
    close(fd);
    return 0;
}

int format_new_passdbs()
{
    char fn[64];
    int bank;
    int r;

    long pagesize;
    passdb_slim_config cfg;
    int sz;
    int quotient, remainder;

    ruleparam_db_ctx *ruleparam_ctx;

    pagesize = sysconf(_SC_PAGE_SIZE);

    r = read_config(&cfg, PASSDB_SLIM_CONFIG_FILE);
    if (r != 0) 
    {
      printf("WARNING: %s was not read, formatting default config file\n", PASSDB_SLIM_CONFIG_FILE);

      make_default_config( PASSDB_SLIM_CONFIG_FILE );
      r = read_config(&cfg, PASSDB_SLIM_CONFIG_FILE);
      if (r!=0)
        return r;
    }


    // Try to load the ruleparam.  If it doesn't exist, try to create it.
    // Else, just deallocate the dummy load.
    //
    
    r = ruleparam_db_load( &ruleparam_ctx , RULEPARAM_DB_FILE );
    if (r < 0)
    {
      printf("WARNING: ruleparam database %s was not read, creating a blank new one\n", RULEPARAM_DB_FILE );

      r = format_new_ruleparamdb( RULEPARAM_DB_FILE );
      if (r < 0)
      {
        perror(RULEPARAM_DB_FILE);
        return r;
      }
    }
    else
    {
      ruleparam_db_free( ruleparam_ctx );
    }


    // For creation of banks
    //

    for (bank=0; bank<cfg.n_one_cred_bank; bank++)
    {
      snprintf(fn, 64, "%s%i%s", cfg.one_cred_db_base_fn, bank, cfg.db_fn_suffix );

      quotient = cfg.n_one_cred_bank_size * RIDER_ONE_CRED_SIZE / pagesize;
      remainder = (cfg.n_one_cred_bank_size * RIDER_ONE_CRED_SIZE ) - (pagesize * quotient) ;

      sz = quotient * pagesize;
      if (remainder > 0)
        sz += pagesize;

      cfg.rider_one_file_bank_size = sz;

      r = format_new_passdb( fn, sz, pagesize );
      if (r != 0) return r;
    }

    //printf("cfg.n_two_cred_bank %i\n", cfg.n_two_cred_bank );

    for (bank=0; bank<cfg.n_two_cred_bank; bank++)
    {

      snprintf(fn, 64, "%s%i%s", cfg.two_cred_db_base_fn, bank, cfg.db_fn_suffix );

      quotient = cfg.n_two_cred_bank_size * RIDER_TWO_CRED_SIZE / pagesize;
      remainder = (cfg.n_two_cred_bank_size * RIDER_TWO_CRED_SIZE ) - (pagesize * quotient) ;

      sz = quotient * pagesize;
      if (remainder > 0)
        sz += pagesize;
        
      cfg.rider_two_file_bank_size = sz;

      r = format_new_passdb( fn, sz, pagesize );
      if (r!=0) return r;
    }



    for (bank=0; bank<cfg.n_spillover_bank; bank++)
    {

      snprintf(fn, 64, "%s%i%s", cfg.spillover_db_base_fn, bank, cfg.db_fn_suffix );

      quotient = cfg.n_spillover_bank_size * RIDER_SPILLOVER_SIZE / pagesize;
      remainder = (cfg.n_spillover_bank_size * RIDER_SPILLOVER_SIZE ) - (pagesize * quotient) ;

      sz = quotient * pagesize;
      if (remainder > 0)
        sz += pagesize;
        
      cfg.rider_spillover_file_bank_size = sz;

      r = format_new_passdb( fn, sz, pagesize );
      if (r!=0) return r;
    }


    free( cfg.one_cred_db_base_fn );
    free( cfg.two_cred_db_base_fn );
    free( cfg.spillover_db_base_fn );
    free( cfg.db_fn_suffix );
    free( cfg.ruleparam_db_fn );

    return 0;

}


int detach_from_passdb(passdb_slim_context *ctx)
{
    int i;
    int bank;

    if(!ctx)
      return FAIL_PARAM;

    if (ctx->ruleparam_db)
    {
      ctx->ruleparam_db->seq = ctx->seq;
      //ruleparam_db_save( ctx->ruleparam_db, ctx->ruleparam_db_fn );
      ruleparam_db_save( ctx->ruleparam_db );
    }

    free_rider_node_list(ctx->freelist_one_cred);
    free_rider_node_list(ctx->freelist_two_cred);
    free_rider_node_list(ctx->freelist_spillover);

    free_rider_node_list(ctx->activelist);

    for(i=0; i < ctx->hash_modulus; i++)
    {
        free_rider_node_list(ctx->logical_card_id_hash[i]);  
        free_rider_node_list(ctx->rider_mag_hash[i]);  
        free_rider_node_list(ctx->rider_rf_hash[i]);  
    }


    for (bank=0; bank<ctx->n_one_cred_bank; bank++)
      munmap( ctx->rider_one_cred_bank[bank], ctx->rider_one_file_bank_size );

    for (bank=0; bank<ctx->n_two_cred_bank; bank++)
      munmap( ctx->rider_two_cred_bank[bank], ctx->rider_two_file_bank_size );

    for (bank=0; bank<ctx->n_spillover_bank; bank++)
      munmap( ctx->rider_spillover_bank[bank], ctx->rider_two_file_bank_size );


    free( ctx->logical_card_id_hash );
    free( ctx->rider_mag_hash );
    free( ctx->rider_rf_hash );

    if (ctx->one_cred_db_bank_fn) 
    {
      for (i=0; i<ctx->n_one_cred_max_bank; i++)
      {
        if (ctx->one_cred_db_bank_fn[i])
        {
          free(ctx->one_cred_db_bank_fn[i]);
        }
      }
      free(ctx->one_cred_db_bank_fn);
    }

    if (ctx->two_cred_db_bank_fn) 
    {
      for (i=0; i<ctx->n_two_cred_max_bank; i++)
      {
        if (ctx->two_cred_db_bank_fn[i])
        {
          free(ctx->two_cred_db_bank_fn[i]);
        }
      }
      free(ctx->two_cred_db_bank_fn);
    }

    if (ctx->spillover_db_bank_fn) 
    {
      for (i=0; i<ctx->n_spillover_max_bank; i++)
      {
        if (ctx->spillover_db_bank_fn[i])
        {
          free(ctx->spillover_db_bank_fn[i]);
        }
      }
      free(ctx->spillover_db_bank_fn);
    }

    if (ctx->one_cred_db_base_fn)
      free( ctx->one_cred_db_base_fn);

    if (ctx->two_cred_db_base_fn)
      free( ctx->two_cred_db_base_fn);

    if (ctx->spillover_db_base_fn)
      free( ctx->spillover_db_base_fn);


    if (ctx->db_fn_suffix)
      free( ctx->db_fn_suffix );

    if (ctx->ruleparam_db_fn)
      free( ctx->ruleparam_db_fn );



    if (ctx->rider_one_cred_bank)
      free( ctx->rider_one_cred_bank );

    if (ctx->rider_two_cred_bank)
      free( ctx->rider_two_cred_bank );

    if (ctx->rider_spillover_bank)
      free( ctx->rider_spillover_bank );



    for (bank=0; bank<ctx->n_one_cred_bank; bank++)
      close(ctx->passes_one_cred_bank_fd[bank]);

    for (bank=0; bank<ctx->n_two_cred_bank; bank++)
      close(ctx->passes_two_cred_bank_fd[bank]);

    for (bank=0; bank<ctx->n_spillover_bank; bank++)
      close(ctx->passes_spillover_bank_fd[bank]);



    if (ctx->passes_one_cred_bank_fd)
      free(ctx->passes_one_cred_bank_fd);

    if (ctx->passes_two_cred_bank_fd)
      free(ctx->passes_two_cred_bank_fd);

    if (ctx->passes_spillover_bank_fd)
      free(ctx->passes_spillover_bank_fd);


    if (ctx->ruleparam_db)
      ruleparam_db_free( ctx->ruleparam_db );

    memset(ctx, 0, sizeof(passdb_slim_context));

    return 0;
}


int init_mmap_passfile( int *fd, void **rider_p, size_t *file_size, size_t rider_size, char *fn )
{
    struct stat st;
    void *t_rider_p;
    int retval;
    int t_fd;

    //int i, k;

    retval = stat(fn, &st);
    if(retval)
    {
        fprintf(stderr, "Cannot find one credential passes file %s!\n", fn);
        return FAIL_DATABASE;
    }

    t_fd = open( fn, O_RDWR | O_SYNC);
    if(t_fd < 0)
    {
        fprintf(stderr, "Cannot open one credential passes file %s!\n", fn);
        return FAIL_DATABASE;
    }

    t_rider_p = (void *) mmap( NULL, st.st_size , PROT_READ | PROT_WRITE, MAP_PRIVATE, t_fd, 0 );

    /*
    for (i=0; i<st.st_size; i++)
    {
      char ch = (unsigned char)(*((unsigned char *)t_rider_p));
      if (ch > 3)
      {
        k = 1;
      }
      else
        k=0;
    }
    */

    if ( (t_rider_p == NULL) || (t_rider_p == MAP_FAILED) )
    {
      close(t_fd);
      fprintf( stderr, 
          "Cannot mmap passes file! Try checking sysctl settings kernel.shmall and kernel.shmmax (return == %p errno == %d)\n",
          t_rider_p, errno);
      return FAIL_MEM;
    }
    else
    {
      //printf("operating in braindead file IO mode...\n");
    }

    *fd = t_fd;
    *rider_p = t_rider_p;

    *file_size = st.st_size ;

    return 0;

}


int load_one_cred(passdb_slim_context *ctx)
{
    int bank ;
    int i, n;
    rider_record_slim_one_cred rr_one;
    seq_t maxseq = 0;

    rider_node *freehead=NULL, *acthead=NULL, *q;
    int numfree = 0, numact = 0;

    n = ctx->n_one_cred_bank * ctx->n_one_cred_bank_size;

    if (ctx->activelist) acthead = ctx->activelist;

    if (ctx->freelist_one_cred) freehead = ctx->freelist_one_cred;

    maxseq = ctx->seq;


    //For all records in our flat file
    //
    for(i=0; i < n; i++)
    {
        bank = i / ctx->n_one_cred_bank_size;
        populate_one_cred_rider_record( &rr_one, i - (bank * ctx->n_one_cred_bank_size), ctx->rider_one_cred_bank[bank] );

        // Check the sequence number and update our latest tally if it is newer.
        //
        if(rr_one.seq > maxseq)
        {
            maxseq = rr_one.seq;
        }

        // If the record is not in use        
        //
        if(rr_one.id == ID_INVALID)
        {
            // Add it to the freelist
            //
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_one_cred_bank; bank++)
                {
                  munmap( ctx->rider_one_cred_bank[bank], ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE );
                }

                fprintf(stderr, "FAIL_MEM: load_one_cred (1)\n");
                return FAIL_MEM;
            }
            else
            {

                numfree++;

                q->next = freehead;
                q->idx = i;

                freehead = q;
            }
        }

        // Else it's an active record
        //
        else
        {
            // Add it to the active list
            //
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_one_cred_bank; bank++)
                {
                  munmap( ctx->rider_one_cred_bank[bank], ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE );
                }

                fprintf(stderr, "FAIL_MEM: load_one_cred (2)\n");
                return FAIL_MEM;
            }
            else
            {
                numact++;

                q->next = acthead;
                q->idx = i;

                acthead = q;

            }
        }  

    }  // for bank

    ctx->freelist_one_cred = freehead;

    ctx->activelist = acthead;

    ctx->seq = maxseq;
    ctx->num_active += numact;
    ctx->num_free += numfree;
    ctx->n_one_cred = numact;

    return 0;

}


int load_two_cred(passdb_slim_context *ctx)
{
    int bank ;
    int i, n;
    rider_record_slim_two_cred rr_two;
    seq_t maxseq = 0;

    rider_node *freehead=NULL, *acthead=NULL, *q;
    int numfree = 0, numact = 0;

    n = ctx->n_two_cred_bank * ctx->n_two_cred_bank_size ;

    if (ctx->freelist_two_cred) freehead = ctx->freelist_two_cred;

    if (ctx->activelist) acthead = ctx->activelist;

    maxseq = ctx->seq;

    for(i=0; i < n; i++)
    {

        bank = i / ctx->n_two_cred_bank_size;
        populate_two_cred_rider_record( &rr_two, i - (bank*ctx->n_two_cred_bank_size), ctx->rider_two_cred_bank[bank] );

        //check the sequence number and update our "latest" tally if it is newer.
        if(rr_two.seq > maxseq)
        {
            maxseq = rr_two.seq;
        }

        //if the record is not in use        
        if(rr_two.id == ID_INVALID)
        {
            //add it to the freelist
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_two_cred_bank; bank++)
                {
                  munmap( ctx->rider_two_cred_bank[bank], ctx->n_two_cred_bank_size * RIDER_ONE_CRED_SIZE );
                }

                fprintf(stderr, "FAIL_MEM: load_two_cred (1)\n");
                return FAIL_MEM;

            }
            else
            {

                numfree++;

                q->next = freehead;
                q->idx = i + INDEX_MIDPOINT;

                freehead = q;
            }
        }
        else
        {
            //add it to the active list
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_two_cred_bank; bank++)
                {
                  munmap( ctx->rider_two_cred_bank[bank], ctx->n_two_cred_bank_size * RIDER_ONE_CRED_SIZE );
                }


                fprintf(stderr, "FAIL_MEM: load_two_cred (2)\n");
                return FAIL_MEM;
            }
            else
            {

                numact++;

                q->next = acthead;
                q->idx = i + INDEX_MIDPOINT;

                acthead = q;
            }
        }  
    }    

    ctx->activelist = acthead;
    ctx->freelist_two_cred = freehead;

    ctx->seq = maxseq;
    ctx->num_active += numact;
    ctx->num_free += numfree;
    ctx->n_two_cred = numact;

    return 0;
}

// UNDER DELEVEOPMENT
//
int load_spillover(passdb_slim_context *ctx)
{
    int bank ;
    int i, n;
    rider_record rr_spillover;
    seq_t maxseq = 0;

    rider_node *freehead=NULL, *acthead=NULL, *q;
    int numfree = 0, numact = 0;

    n = ctx->n_spillover_bank * ctx->n_spillover_bank_size ;

    if (ctx->freelist_spillover) freehead = ctx->freelist_spillover;

    if (ctx->activelist) acthead = ctx->activelist;

    maxseq = ctx->seq;

    for(i=0; i < n; i++)
    {

        bank = i / ctx->n_spillover_bank_size;
        populate_spillover_rider_record( &rr_spillover, i - (bank*ctx->n_spillover_bank_size), ctx->rider_spillover_bank[bank] );

        //check the sequence number and update our "latest" tally if it is newer.
        if(rr_spillover.seq > maxseq)
        {
            maxseq = rr_spillover.seq;
        }

        //if the record is not in use        
        if(rr_spillover.id == ID_INVALID)
        {
            //add it to the freelist
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_spillover_bank; bank++)
                {
                  munmap( ctx->rider_spillover_bank[bank], ctx->n_spillover_bank_size * RIDER_ONE_CRED_SIZE );
                }

                fprintf(stderr, "FAIL_MEM: load_spillover (1)\n");
                return FAIL_MEM;
            }
            else
            {

                numfree++;

                q->next = freehead;
                q->idx = i + (2*INDEX_MIDPOINT);

                freehead = q;
            }
        }
        else
        {
            //add it to the active list
            q = (rider_node *) malloc( sizeof(rider_node) );

            if(!q)
            {
                free_rider_node_list(freehead);
                free_rider_node_list(acthead);
                fprintf(stderr, "Malloc returned NULL loading riders!\n");

                for (bank=0; bank<ctx->n_spillover_bank; bank++)
                {
                  munmap( ctx->rider_spillover_bank[bank], ctx->n_spillover_bank_size * RIDER_ONE_CRED_SIZE );
                }


                fprintf(stderr, "FAIL_MEM: load_spillover (2)\n");
                return FAIL_MEM;
            }
            else
            {

                numact++;

                q->next = acthead;
                q->idx = i + (2*INDEX_MIDPOINT);

                acthead = q;
            }
        }  
    }    

    ctx->activelist = acthead;
    ctx->freelist_spillover = freehead;

    ctx->seq = maxseq;
    ctx->num_active += numact;
    ctx->num_free += numfree;
    ctx->n_spillover = numact;

    return 0;
}

// Slow but sure way of deleting entries.  Restarting from the beginning will
// be slower than if we exploited knowledge of the data structures, but makes
// this routine much more robust and simple to code.  By restarting after
// every delete, we don't need any in depth knowledge about how to search
// or delete entries other than walking through the 'activelist' and using
// 'delete_rider' to delete the entry.
//
void passdb_slim_delete_seq_above( passdb_slim_context *ctx, seq_t cur_seq )
{
  int dirty = 0;
  rider_node *nod;
  rider_record rr;

  do {
    dirty = 0;

    nod = ctx->activelist;
    while (nod)
    {

      make_rider_record( ctx, &rr, nod->idx );

      if (rr.seq > cur_seq)
      {
        delete_rider( ctx, &rr, 1 );
        dirty = 1;
        break;
      }

      nod = nod->next;

    }
  } while (dirty);



}


int attach_to_passdb(passdb_slim_context *ctx)
{
    int retval;

    int numfree = 0, numact = 0;
    int read_n_one_cred; 
    int read_n_two_cred;
    int read_n_spillover;

    int i;
    int n;

    seq_t ruleparam_seq = 0;

    //--------

    if(!ctx)		//fail if we get passed a null pointer
        return FAIL_PARAM;

    //We also want to fail if we get passed a pointer to an active/in-use context...
    //
    if( ctx->rider_one_cred_bank || 
        ctx->rider_two_cred_bank || 
        ctx->rider_spillover_bank || 
        ctx->activelist ||
        ctx->freelist_one_cred || ctx->freelist_two_cred )
    {
        return FAIL_PARAM;
    }

    ctx->rider_one_cred_bank = NULL;
    ctx->rider_two_cred_bank = NULL;
    ctx->rider_spillover_bank = NULL;

    ctx->activelist = NULL;
    ctx->freelist_one_cred = NULL;
    ctx->freelist_two_cred = NULL;
    ctx->freelist_spillover = NULL;

    ctx->num_active = 0;
    ctx->num_free = 0;
    ctx->seq = 0;

    ctx->n_one_cred = 0;
    ctx->n_two_cred = 0;
    ctx->n_spillover = 0;

    ctx->one_cred_db_bank_fn = NULL;
    ctx->two_cred_db_bank_fn = NULL;
    ctx->spillover_db_bank_fn = NULL;

    ctx->ruleparam_db_fn = NULL;

    ctx->one_cred_db_base_fn = NULL;
    ctx->two_cred_db_base_fn = NULL;
    ctx->spillover_db_base_fn = NULL;
    ctx->db_fn_suffix = NULL;

    retval = 
      init_context_from_config( ctx, PASSDB_SLIM_CONFIG_FILE );

    if (retval != 0)
    {
      fprintf(stderr, "ERROR: init_context_from_config (%s) failed, can't proceed\n", PASSDB_SLIM_CONFIG_FILE );
      return retval;
    }


    retval = ruleparam_db_load( &(ctx->ruleparam_db), ctx->ruleparam_db_fn );
    if ( !ctx->ruleparam_db )
    {
        fprintf(stderr, "ruleparam_db_load failed (retval %i).\n", retval);
        detach_from_passdb(ctx);
        return retval;
    }

    ruleparam_seq = ctx->ruleparam_db->seq;

    for (i=0; i<ctx->n_one_cred_bank; i++)
    {
      retval = 
        init_mmap_passfile( &(ctx->passes_one_cred_bank_fd[i]),
                            &(ctx->rider_one_cred_bank[i]),
                            &(ctx->rider_one_file_size),
                            RIDER_ONE_CRED_SIZE,
                            ctx->one_cred_db_bank_fn[i] );

      if (retval != 0)
      {
        fprintf(stderr, "ERROR: attach_to_passdb one credential: %s\n", ctx->one_cred_db_bank_fn[i]);
        return retval;
      }

      read_n_one_cred = ctx->rider_one_file_size / RIDER_ONE_CRED_SIZE ;
      if (read_n_one_cred != ctx->n_one_cred_bank_size )
      {
        fprintf(stderr, "WARNING: n_one_cred in config file (%i) does not match n_one_cred (%i) in db file (%s).  Using db file as truth\n",
            ctx->n_one_cred_bank_size, read_n_one_cred, ctx->one_cred_db_bank_fn[i] );
      }

    }


    for (i=0; i<ctx->n_two_cred_bank; i++)
    {
      retval = 
        init_mmap_passfile( &(ctx->passes_two_cred_bank_fd[i]),
                            &(ctx->rider_two_cred_bank[i]),
                            &(ctx->rider_two_file_size),
                            RIDER_TWO_CRED_SIZE,
                            ctx->two_cred_db_bank_fn[i] );

      if (retval != 0)
      {
        fprintf(stderr, "ERROR: attach_to_passdb two credential: %s\n", ctx->two_cred_db_bank_fn[i]);
        return retval;
      }

      read_n_two_cred = ctx->rider_two_file_size / RIDER_TWO_CRED_SIZE ;
      if (read_n_two_cred != ctx->n_two_cred_bank_size )
      {
        fprintf(stderr, "WARNING: n_two_cred in config file (%i) does not match n_two_cred (%i) in db file (%s).  Using db file as truth\n",
            ctx->n_two_cred_bank_size, read_n_two_cred, ctx->two_cred_db_bank_fn[i] );
      }

    }


    for (i=0; i<ctx->n_spillover_bank; i++)
    {
      retval = 
        init_mmap_passfile( &(ctx->passes_spillover_bank_fd[i]),
                            &(ctx->rider_spillover_bank[i]),
                            &(ctx->rider_spillover_file_size),
                            RIDER_SPILLOVER_SIZE,
                            ctx->spillover_db_bank_fn[i] );

      if (retval != 0)
      {
        fprintf(stderr, "ERROR: attach_to_passdb spillover: %s\n", ctx->spillover_db_bank_fn[i]);
        return retval;
      }

      read_n_spillover = ctx->rider_spillover_file_size / RIDER_SPILLOVER_SIZE ;
      if (read_n_spillover != ctx->n_spillover_bank_size )
      {
        fprintf(stderr, "WARNING: n_spillover in config file (%i) does not match n_spillover (%i) in db file (%s).  Using db file as truth\n",
            ctx->n_spillover_bank_size, read_n_spillover, ctx->spillover_db_bank_fn[i] );
      }

    }


    //-------------

    retval = load_one_cred(ctx);
    if (retval != 0)
    {
        fprintf(stderr, "load_one_cred failed.\n");
        detach_from_passdb(ctx);
        return retval;
    }

    retval = load_two_cred(ctx);
    if (retval != 0)
    {
        fprintf(stderr, "load_two_cred failed.\n");
        detach_from_passdb(ctx);
        return retval;
    }

    retval = load_spillover(ctx);
    if (retval != 0)
    {
        fprintf(stderr, "load_spillover failed.\n");
        detach_from_passdb(ctx);
        return retval;
    }

    retval = build_hashes(ctx);

    if( DB_FAIL(retval) )
    {
        fprintf(stderr, "Building hashes failed.\n");
        detach_from_passdb(ctx);
        return retval;
    }


    if (ruleparam_seq > 0)
    {

      // ruleparam.db didn't get a chance to flush
      // after a pass db update.  We set the current sequence
      // number to be the one we recorded from the old
      // ruleparma.db so that we pick up updates from the
      // lost caedential and we go through and delete 
      // the old entries with sequence numbers older than
      // ruleparam_db->seq.
      //
      // To be safe, we ake the minimum of the two sequence numbers
      // as truth.
      //
      if (ruleparam_seq < ctx->seq)
      {
        //DEBUG
        fprintf(stderr, "WARNING: passdb.attach_to_passdb discr. in ruleparam_seq (%llu) and mem seq (%llu): removing newer seq numbers\n", ruleparam_seq, ctx->seq);

        ctx->seq = ruleparam_seq;
        passdb_slim_delete_seq_above( ctx, ctx->seq );

      }

    }

    ctx->mmap_broken = 1;

    numfree = ctx->num_free;
    numact  = ctx->num_active;

    n = numfree + numact;

    // DEBUG
    //
    printf("Loaded and indexed %d records (%d used, %d free); Newest seq = %llu\n", n, numact, numfree, ctx->seq);

    return n;
} 



static int convert_magstripe( unsigned char *magstripe_code, unsigned long long *magstripe, char *magstripe_value )
{
  int a;
  unsigned long long b;
  int cpos=0;


  for ( cpos=0; 
        (cpos < CREDENTIAL_LEN) && ( magstripe_value[cpos] != ':' ); 
        cpos++);
  if (cpos == CREDENTIAL_LEN) 
  {
    return -1;
  }

  magstripe_value[cpos] = '\0';

  a = atoi(magstripe_value);
  b = (unsigned long long)atoll( magstripe_value + cpos + 1 );
  magstripe_value[cpos] = ':';

  if ((a < 0) || (a > 255))
  {
    return -1;
  }

  *magstripe_code = (unsigned char)a;
  *magstripe = b;

  return 0;
}

static int convert_rfid( unsigned char *rfid_code, 
                         unsigned long *rfid_site, 
                         unsigned long *rfid_val, 
                         char *rfid_value )
{
  int a;
  unsigned long b, c;
  int cpos0, cpos1;

  for ( cpos0=0;
        (cpos0 < CREDENTIAL_LEN) && (rfid_value[cpos0] != ':');
        cpos0++);
  if (cpos0 == CREDENTIAL_LEN)
    return -1;

  for ( cpos1=cpos0+1;
        (cpos1 < CREDENTIAL_LEN) && (rfid_value[cpos1] != ':');
        cpos1++);
  if (cpos1 == CREDENTIAL_LEN)
    return -1;

  rfid_value[cpos0] = '\0';
  rfid_value[cpos1] = '\0';


  a = atoi(rfid_value);
  b = (unsigned long )atol(rfid_value + cpos0 + 1);
  c = (unsigned long )atol(rfid_value + cpos1 + 1);

  rfid_value[cpos0] = ':';
  rfid_value[cpos1] = ':';

  if ((a < 0) || (a > 255))
    return -1;

  *rfid_code = (unsigned char)a;
  *rfid_site = (unsigned long)b;
  *rfid_val = (unsigned long)c;

  return 0;
}


static int convert_single_credential( rider_record_slim_one_cred *rr1, rider_record *rec )
{
  unsigned char magstripe_code, rfid_code;
  unsigned long long magstripe;
  unsigned long rfid_site, rfid_val;
  unsigned long long tsite, tval;

  int r;

  r = convert_magstripe( &magstripe_code, &magstripe, rec->magstripe_value );
  if ( (r == 0) && 
      ((magstripe_code != 0) || (magstripe != 0)) )  
  {
    rr1->code = magstripe_code;
    rr1->credential = magstripe;
    return 0;
  }

  r = convert_rfid( &rfid_code, &rfid_site, &rfid_val, rec->rfid_value );
  if ( (r == 0) && 
       ((rfid_code != 0) || (rfid_site != 0) || (rfid_val != 0)) )
  {
    rr1->code = rfid_code;
    tsite = rfid_site;
    tval  = rfid_val;
    rr1->credential = (tsite << 32) | tval;
    return 0;
  }

  return -1;
}

static int copy_rider_one_cred( passdb_slim_context *ctx, int idx, rider_record_slim_one_cred *rr1 )
{
  void *p;

  p = passdb_slim_get_record_address( ctx, idx );
  if (!p)
  {
    fprintf(stderr, "ERROR: copy_rider_one_cred, got NULL address, bad index? (idx %i)\n", idx);
    return -1;
  }

  //*((seq_t *)p) = rr1->seq;
  _ulliw(p, rr1->seq);
  p += sizeof(seq_t);

  //*((logical_card_id_t *)p) = rr1->id;
  _ulliw(p, rr1->id);
  p += sizeof(logical_card_id_t);

  *((unsigned char *)p) = rr1->code;
  p += sizeof(unsigned char);

  //*((unsigned long long *)p) = rr1->credential;
  _ulliw(p, rr1->credential);
  p += sizeof(unsigned long long);

  //*((unsigned short int *)p) = rr1->rule_param_bucket_id;;
  _usiw(p, rr1->rule_param_bucket_id);
  //p += sizeof(unsigned short int );


  return 0;
}

static int copy_rider_two_cred( passdb_slim_context *ctx, int idx, rider_record_slim_two_cred *rr2 )
{
  void *p;

  p = passdb_slim_get_record_address( ctx, idx );
  if (!p)
  {
    fprintf(stderr, "ERROR: copy_rider_two_cred, got NULL address, bad index? (idx %i)\n", idx);
    return -1;
  }

  //*((seq_t *)p) = rr2->seq;
  _ulliw(p, rr2->seq);
  p += sizeof(seq_t);

  //*((logical_card_id_t *)p) = rr2->id;
  _ulliw(p, rr2->id);
  p += sizeof(logical_card_id_t);

  *((unsigned char *)p) = rr2->magstripe_code;
  p += sizeof(unsigned char);

  //*((unsigned long long *)p) = rr2->magstripe;
  _ulliw(p, rr2->magstripe);
  p += sizeof(unsigned long long);

  *((unsigned char *)p) = rr2->rfid_code;
  p += sizeof(unsigned char );

  //*((unsigned long *)p) = rr2->rfid_site;
  _uliw(p, rr2->rfid_site);
  p += sizeof(unsigned long );

  //*((unsigned long *)p) = rr2->rfid_val;
  _uliw(p, rr2->rfid_val);
  p += sizeof(unsigned long );

  //*((unsigned short int *)p) = rr2->rule_param_bucket_id;;
  _usiw(p, rr2->rule_param_bucket_id);
  //p += sizeof(unsigned short int );

  return 0;
}

static int copy_rider_spillover( passdb_slim_context *ctx, int idx, rider_record *rec )
{
  void *p;

  p = passdb_slim_get_record_address( ctx, idx );
  if (!p)
  {
    fprintf(stderr, "ERROR: copy_rider_spillover, got NULL address, bad index? (idx %i)\n", idx);
    return -1;
  }

  memcpy(p, rec, sizeof(rider_record) );

  return 0;

}

// Take rider as stored in rider record, store in either one_cred or two_cred
// DB, depending on content.
//
static int copy_rider( passdb_slim_context *ctx, int idx, rider_record *src)
{
    int rule_param_bucket_id;
    rider_record_slim_one_cred rr1 = {0};
    rider_record_slim_two_cred rr2 = {0};

    if(! (src) )
        return FAIL_PARAM;

    // find rule param bucket
    //
    rule_param_bucket_id = 
      ruleparam_db_find( ctx->ruleparam_db, src->rule_name, src->rule_param );

    if ( rule_param_bucket_id == RULEPARAM_DB_NOT_FOUND )
    {
        fprintf(stderr, "ERROR: copy_rider: rule_param_bucket_id for %s %s not found\n", src->rule_name, src->rule_param);
        return rule_param_bucket_id; 
    }
    else if (rule_param_bucket_id == RULEPARAM_DB_FULL )
    {
        fprintf(stderr, "ERROR: copy_rider: RULEPARAM DB FULL!\n");
        return rule_param_bucket_id;
    }

    if (idx < INDEX_MIDPOINT)
    {
        rr1.rule_param_bucket_id = (unsigned short int)rule_param_bucket_id;
        rr1.seq = src->seq;
        rr1.id = src->id;

        convert_single_credential( &rr1, src );

        copy_rider_one_cred( ctx, idx, &rr1 );
    }
    else if (idx < (2*INDEX_MIDPOINT))
    {


        rr2.rule_param_bucket_id = (unsigned short int)rule_param_bucket_id;
        rr2.seq = src->seq;
        rr2.id = src->id;

        convert_magstripe( &(rr2.magstripe_code), &(rr2.magstripe), src->magstripe_value );
        convert_rfid( &(rr2.rfid_code), &(rr2.rfid_site), &(rr2.rfid_val), src->rfid_value );

        copy_rider_two_cred( ctx, idx, &rr2 );
    }
    else 
    {
        copy_rider_spillover( ctx, idx, src );
    }

    return 0;
}

static int alloc_rider_one_cred(passdb_slim_context *ctx)
{
    rider_node *p;

    p = ctx->freelist_one_cred;

    if(p)
    {
        ctx->freelist_one_cred = ctx->freelist_one_cred->next;

        p->next = ctx->activelist;
        ctx->activelist = p;

        ctx->n_one_cred++;
        return p->idx;
    }
    else
    {
        return FAIL_FULL;
    }
}

static int alloc_rider_two_cred(passdb_slim_context *ctx)
{
    rider_node *p;

    p = ctx->freelist_two_cred;

    if(p)
    {
        ctx->freelist_two_cred = ctx->freelist_two_cred->next;

        p->next = ctx->activelist;
        ctx->activelist = p;

        ctx->n_two_cred++;
        return p->idx;
    }
    else
    {
        return FAIL_FULL;
    }
}


static int alloc_rider_spillover(passdb_slim_context *ctx)
{
    rider_node *p;

    p = ctx->freelist_spillover;

    if(p)
    {
        ctx->freelist_spillover = ctx->freelist_spillover->next;

        p->next = ctx->activelist;
        ctx->activelist = p;

        ctx->n_spillover++;
        return p->idx;
    }
    else
    {
        return FAIL_FULL;
    }
}


static int free_rider(passdb_slim_context *ctx, int idx)
{
    rider_node *p, *q;
    void *record_p;

    q = NULL;
    p = ctx->activelist;

    while(p)
    {
        if( p->idx == idx )
            break;

        q = p;
        p = p->next;
    }

    if(p)
    {
        if(q)
        {
            q->next = p->next;  
        }
        else
        {
            ctx->activelist = p->next;
        }

        if ( idx < INDEX_MIDPOINT )
        {
          p->next = ctx->freelist_one_cred;
          ctx->freelist_one_cred = p;
        }
        else if (idx < (2*INDEX_MIDPOINT))
        {
          p->next = ctx->freelist_two_cred;
          ctx->freelist_two_cred = p;
        }
        else
        {
          p->next = ctx->freelist_spillover;
          ctx->freelist_spillover = p;
        }

        // ID_INVALID _should_ be 0
        //
        record_p = passdb_slim_get_record_address( ctx, idx );
        if (!record_p)
          return -1;

        if ( idx < INDEX_MIDPOINT )
        {
          memset( record_p, 0, sizeof(RIDER_ONE_CRED_SIZE) );
        }
        else if (idx < (2*INDEX_MIDPOINT))
        {
          memset( record_p, 0, sizeof(RIDER_ONE_CRED_SIZE) );
        }
        else
        {
          memset( record_p, 0, sizeof(RIDER_SPILLOVER_SIZE) );
        }

        return 0;
    }
    else
    {
        return WARN_NOTFOUND;
    }
}

static void sync_rider_change(passdb_slim_context *ctx, int idx)
{
    int bank=-1, pos=-1;
    int offset;
    int offset_next;
    int retval;
    int n_pages = 1;

    int tmpa, tmpb;

    retval=-1;

    if(idx < 0)
        return;

    if(!ctx)
        return;

    passdb_slim_get_cred_bank_and_pos( ctx, &bank, &pos, idx );

    if ( idx < INDEX_MIDPOINT )
    {
        offset = (pos * RIDER_ONE_CRED_SIZE) / MEMORY_PAGE_SIZE;        // calculate the beginning page number
        offset *= MEMORY_PAGE_SIZE;					// multiply by page size
        retval = lseek(ctx->passes_one_cred_bank_fd[bank], offset, SEEK_SET);

        offset_next = (((pos+1)*RIDER_ONE_CRED_SIZE) - 1) / MEMORY_PAGE_SIZE;
        offset_next *= MEMORY_PAGE_SIZE;
        if (offset_next != offset )
        {
          n_pages = 2;
        }

        if(retval != offset)
        {
            fprintf(stderr, "lseek() failed in sync_rider_change(). errno = %d\n", errno);
            perror("one");
            return;  
        }

        retval = write(ctx->passes_one_cred_bank_fd[bank], ctx->rider_one_cred_bank[bank] + offset, n_pages * MEMORY_PAGE_SIZE);

        tmpa = offset;
        tmpb = offset + RIDER_ONE_CRED_SIZE;

        if( retval != (n_pages*MEMORY_PAGE_SIZE) )
        {
            fprintf(stderr, "write() failed (one_cred) in sync_rider_change(). errno = %d\n", errno);
            return;
        }

        fsync(ctx->passes_one_cred_bank_fd[bank]);
    }
    else if ( idx < (2*INDEX_MIDPOINT) )
    {
        offset = (pos * RIDER_TWO_CRED_SIZE) / MEMORY_PAGE_SIZE;   // calculate the beginning page number
        offset *= MEMORY_PAGE_SIZE;					              // multiply by page size
        retval = lseek(ctx->passes_two_cred_bank_fd[bank], offset, SEEK_SET);

        offset_next = (((pos+1)*RIDER_TWO_CRED_SIZE) - 1) / MEMORY_PAGE_SIZE;
        offset_next *= MEMORY_PAGE_SIZE;
        if (offset_next != offset )
        {
          n_pages = 2;
        }

        if(retval != offset)
        {
            fprintf(stderr, "lseek() failed in sync_rider_change(). errno = %d\n", errno);
            perror("two");
            return;  
        }

        retval = write(ctx->passes_two_cred_bank_fd[bank], ctx->rider_two_cred_bank[bank] + offset, n_pages * MEMORY_PAGE_SIZE);

        if ( retval != (n_pages*MEMORY_PAGE_SIZE) )
        {
            fprintf(stderr, "write() failed (two_cred) in sync_rider_change(). errno = %d\n", errno);
            return;
        }

        fsync(ctx->passes_two_cred_bank_fd[bank]);
    }
    else 
    {
        // SPILLOVER should be page aligned, but in case it's not, just do the same calculation as we do above
        //

        offset = (pos * RIDER_SPILLOVER_SIZE) / MEMORY_PAGE_SIZE;   // calculate the beginning page number
        offset *= MEMORY_PAGE_SIZE;					              // multiply by page size
        retval = lseek(ctx->passes_spillover_bank_fd[bank], offset, SEEK_SET);

        offset_next = (((pos+1)*RIDER_SPILLOVER_SIZE) - 1) / MEMORY_PAGE_SIZE;
        offset_next *= MEMORY_PAGE_SIZE;
        if (offset_next != offset )
        {
          n_pages = 2;
        }


        if(retval != offset)
        {
            fprintf(stderr, "lseek() failed in sync_rider_change(). errno = %d (\n", errno);
            perror("three");
            return;  
        }

        retval = write(ctx->passes_spillover_bank_fd[bank], ctx->rider_spillover_bank[bank] + offset, n_pages * MEMORY_PAGE_SIZE);


        if ( retval != (n_pages*MEMORY_PAGE_SIZE) )
        {
            fprintf(stderr, "write() failed (spillover) in sync_rider_change(). errno = %d\n", errno);
            return;
        }

        fsync(ctx->passes_spillover_bank_fd[bank]);
    }

}

void sync_all_riders(passdb_slim_context *ctx)
{
    int retval;
    int bank;

    if(!ctx)
        return;

    // flush one cred first
    //

    for (bank=0; bank<ctx->n_one_cred_bank; bank++)
    {
        retval = lseek(ctx->passes_one_cred_bank_fd[bank], 0, SEEK_SET);
        if(retval != 0)
        {
            fprintf(stderr, "lseek() failed in sync_all_riders() for one credential DB (bank %i). errno = %d\n", bank, errno);
            return;  
        }

        retval = write(ctx->passes_one_cred_bank_fd[bank], 
                       ctx->rider_one_cred_bank[bank], 
                       ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE );
        if(retval != (ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE) )
        {
            fprintf(stderr, "write() failed in sync_all_riders() for one credential DB (bank %i). errno = %d\n", bank, errno);
            return;
        }

    }


    // flush two cred next
    //

    for (bank=0; bank<ctx->n_two_cred_bank; bank++)
    {
        retval = lseek(ctx->passes_two_cred_bank_fd[bank], 0, SEEK_SET);
        if(retval != 0)
        {
            fprintf(stderr, "lseek() failed in sync_all_riders() for two credential DB (bank %i). errno = %d\n", bank, errno);
            return;  
        }

        retval = write(ctx->passes_two_cred_bank_fd[bank], 
                       ctx->rider_two_cred_bank[bank], 
                       ctx->n_two_cred_bank_size * RIDER_TWO_CRED_SIZE );
        if(retval != (ctx->n_two_cred_bank_size * RIDER_TWO_CRED_SIZE) )
        {
            fprintf(stderr, "write() failed in sync_all_riders() for two credential DB (bank %i). errno = %d\n", bank, errno);
            return;
        }
    }

    // finall flush our spillover banks
    //

    for (bank=0; bank<ctx->n_spillover_bank; bank++)
    {
        retval = lseek(ctx->passes_spillover_bank_fd[bank], 0, SEEK_SET);
        if(retval != 0)
        {
            fprintf(stderr, "lseek() failed in sync_all_riders() for two credential DB (bank %i). errno = %d\n", bank, errno);
            return;  
        }

        retval = write(ctx->passes_spillover_bank_fd[bank], 
                       ctx->rider_spillover_bank[bank], 
                       ctx->n_spillover_bank_size * RIDER_SPILLOVER_SIZE );
        if(retval != (ctx->n_spillover_bank_size * RIDER_SPILLOVER_SIZE) )
        {
            fprintf(stderr, "write() failed in sync_all_riders() for two credential DB (bank %i). errno = %d\n", bank, errno);
            return;
        }
    }

}

int delete_rider(passdb_slim_context *ctx, rider_record *rec, int sync)
{
    int bank;
    int id_idx;
    void *rider_p;
    rider_record rr;

    if(!ctx)
    {
        return FAIL_PARAM;
    }

    if (!ctx->rider_one_cred_bank) return FAIL_PARAM;
    if (!ctx->rider_two_cred_bank) return FAIL_PARAM;

    for (bank=0; bank<ctx->n_one_cred_bank; bank++)
      if (!ctx->rider_one_cred_bank[bank]) return FAIL_PARAM;

    for (bank=0; bank<ctx->n_two_cred_bank; bank++)
      if (!ctx->rider_two_cred_bank[bank]) return FAIL_PARAM;

    for (bank=0; bank<ctx->n_spillover_bank; bank++)
      if (!ctx->rider_spillover_bank[bank]) return FAIL_PARAM;

    //If this record is older than out current database, ignore it as a duplicate.
    //
    if( ctx->seq >= rec->seq )
    {
        return 0;
    }

    //find the record to be deleted in our ID hash
    //
    id_idx = find_id_in_hash(ctx, rec->id);

    //If we didn't find it, it must have already been deleted...
    //
    if(id_idx < 0)	
    {
        return 0;
    }

    make_rider_record( ctx, &rr, id_idx );


    //delete it from all hashes
    //
    delete_from_id_hash(ctx, id_idx);        
    delete_from_mag_hash(ctx, id_idx);        
    delete_from_rf_hash(ctx, id_idx);        

    //free the record (this zeros out the entire block)
    //
    free_rider(ctx, id_idx);

    passdb_slim_manage_rider_banks(ctx);

    rider_p = passdb_slim_get_record_address( ctx, id_idx );
    if (!rider_p)
      return -1;

    if (id_idx < INDEX_MIDPOINT )
    {
        ctx->n_one_cred--;
        memset( rider_p, 0, RIDER_ONE_CRED_SIZE );
    }
    else if (id_idx < (2*INDEX_MIDPOINT) )
    {
        ctx->n_two_cred--;
        memset( rider_p, 0, RIDER_TWO_CRED_SIZE );
    }
    else
    {
        ctx->n_spillover--;
        memset( rider_p, 0, RIDER_SPILLOVER_SIZE );
    }

    ctx->num_active--;
    ctx->num_free++;


    ctx->ruleparam_db->seq = ctx->seq;
    ruleparam_db_update( ctx->ruleparam_db, rr.rule_name, rr.rule_param, -1 );
    ruleparam_db_rate_limited_sync( ctx->ruleparam_db );


    //and sync our SHM
    //
    if(sync)
    {
        sync_rider_change(ctx, id_idx);
    }

    return 1;
}

// Return the number of non blank credentials
//
int rider_count_credential( rider_record *rec )
{
  int mag_flag = 0;
  int rfid_flag = 0;
  int count = 0;
  int s, e;
  int r;

  unsigned char dummy_code;
  unsigned long dummy_site, dummy_rfid;
  unsigned long long dummy_mag;

  for (s=0; (s<CREDENTIAL_LEN) && (rec->magstripe_value[s] == ' ')  ; s++);
  for (e=s+1; (e<CREDENTIAL_LEN) && (rec->magstripe_value[e]) ; e++ );
  if ((e - s) > 1) mag_flag = 1;

  for (s=0; (s<CREDENTIAL_LEN) && (rec->rfid_value[s] == ' ')  ; s++);
  for (e=s+1; (e<CREDENTIAL_LEN) && (rec->rfid_value[e]) ; e++ );
  if ((e - s) > 1) rfid_flag = 1;

  if (mag_flag)
  {
    r = convert_magstripe( &dummy_code, &dummy_mag, rec->magstripe_value );
    if (r < 0) 
      return r;

    count++;
  }

  if (rfid_flag)
  {
    r = convert_rfid( &dummy_code, &dummy_site, &dummy_rfid, rec->rfid_value );
    if (r < 0)
      return r;
    count++;
  }

  return count;
}

// Called from pass_communication on an UPDATE message
//
int update_rider(passdb_slim_context *ctx, rider_record *rec, int sync)
{
    int id_idx;
    int mag_idx;
    int rf_idx;

    int update_credentials = 0;
    int update_id_hash = 0;

    int retval;

    int credential_count = 0;
    int old_credential_count = 0;
    int is_new_record = 0;
    int bank;

    int decrement_old_rule_ref = 0;
    rider_record rr = {0};

    if(!ctx)
    {
        return FAIL_MEM;
    }


    if (!ctx->rider_one_cred_bank) return FAIL_PARAM;
    if (!ctx->rider_two_cred_bank) return FAIL_PARAM;

    for (bank=0; bank<ctx->n_one_cred_bank; bank++)
      if (!ctx->rider_one_cred_bank[bank]) return FAIL_PARAM;

    for (bank=0; bank<ctx->n_two_cred_bank; bank++)
      if (!ctx->rider_two_cred_bank[bank]) return FAIL_PARAM;


    if( ctx->seq >= rec->seq )
    {
        return 0;
    }

    credential_count = rider_count_credential( rec );
    old_credential_count = credential_count;
    if (credential_count == 0)
    {
      return 0;
    }


    id_idx = find_id_in_hash(ctx, rec->id);
    mag_idx = find_mag_in_hash(ctx, rec->magstripe_value);
    rf_idx = find_rf_in_hash(ctx, rec->rfid_value);


//	We want to allow a short period of magstrip or RFID collision as the lesser of two evils vs.		 	**STUPID**
//  possibly losing a record due to a degenerately stupid administrator doing the following:
//
//	1) Create user 1 with magstripe '1:foo'
//	2) Delete user 1
//	3) Create user 2 with magstripe '1:foo'
//	4) Create user 1 with magstripe '1:bar'	<----  THIS IS NOT ALLOWED (Creating user 1 is dissalowed after a delete of user 1.  This card should be created as user 3).
//
//	The issue here is that if the bus asks what's happensed since sequence number 1, it will get rows 
//  3 and 4.  
//
//	In reality, we'd hope that each bus would complete a sync at least once on a shorter interval
//  than the frequency at which credentials are recycled, but you never know...  And if somebody manually
//  fucks things up such that a user id (card id) is deleted, and then created again (this is a big no-no), we
//  can recover by allowing a hash collision to exist in the meantime.
#ifndef ALLOW_CREDENTIAL_COLLISIONS
    if( (mag_idx >= 0) && (mag_idx != id_idx) )
    {
        //fprintf(stderr, "Refusing to accept change that would introduce duplicate magstripe \"%s\" for records %llu and %llu.\n", rec->magstripe_value, ctx->riders[mag_idx].id, rec->id);
        make_rider_record( ctx, &rr, id_idx );
        fprintf(stderr, "Refusing to accept change that would introduce duplicate magstripe \"%s\" for records %llu and %llu.\n", rec->magstripe_value, rr.id, rec->id);
        return FAIL_DUPKEY;
    }

    if( (rf_idx >= 0) && (rf_idx != id_idx) )
    {
        make_rider_record( ctx, &rr, id_idx );
        fprintf(stderr, "Refusing to accept change that would introduce duplicate RFID \"%s\" for records %llu and %llu.\n", rec->rfid_value, rr.id, rec->id);
        return FAIL_DUPKEY;
    }
#endif

    if(id_idx >= 0)	//if rec->id already exists, we're updating an existing record...
    {
        
        // If EITHER the RFID or MAGSTRIPE values have changed...
        //

        make_rider_record( ctx, &rr, id_idx );
        if( strncmp(rr.magstripe_value, rec->magstripe_value, CREDENTIAL_LEN) || 
            strncmp(rr.rfid_value,      rec->rfid_value,      CREDENTIAL_LEN) )
        {
            update_credentials = 1;
        }

        // EXPERIMENTAL
        //
        if      ( id_idx <   INDEX_MIDPOINT ) { old_credential_count =  1; }
        else if ( id_idx < 2*INDEX_MIDPOINT ) { old_credential_count =  2; }
        else                                  { old_credential_count = -1; }

        /*
        if ( strncmp( rr.magstripe_value , rec->magstripe_value, CREDENTIAL_LEN ) != 0 )
          update_magstrip_credential = 1;
        if ( strncmp( rr.rfid_value, rec->rfid_value, CREDENTIAL_LEN ) != 0 )
          update_rfid_credential = 1;
        */


        // For simplicity, decrement the reference count of the rule for the pre-existing
        // rule record.  The 'new' rule record will be incremented below.  If the 'new'
        // rule record is the same, this has the effect of not changing the reference count.
        // If there is only one reference, this will delete and re-create it, but we
        // expect rule records to most often have more than just a single reference, so
        // this should be a case that is not hit very often.
        // If the rule record has changed, this will correctly decrement the reference
        // to the old rule record here, and increment the reference count of the new
        // rule record below.
        //

        decrement_old_rule_ref = 1;

    }

    // EXPERIMENTAL 
    //
    // If we've changed credential count (most likely 1 to 2 or 2 to 1),
    // then we need to switch the database type (e.g. from the one credential
    // database banks to the two credential database banks).
    // Delete the old entry of the rider, both on disk and in our local
    // hashes in memory, allocate a new index, then proceed as normal.
    //
    // If this is a new record, old_credential_count is default
    // equal to credential count, so this block gets ignored.
    //
    if ( credential_count != old_credential_count )
    {

        // Delete old entry completely.  delete_rider will
        // update counts and delete the index from the hashes.
        //

        // We need to 'trick' delete_rider into accepting the change
        // by setting the sequence number to be the current one from
        // the fetched record.  delete_rider also updates (decmeents etc.)
        // the rule, so we don't need to do it later.
        //

        rr.seq = rec->seq;
        delete_rider( ctx, &rr, 1 );
        decrement_old_rule_ref = 0;

        // Create new index and update counts
        //
        if      ( credential_count == 1 ) { id_idx = alloc_rider_one_cred(ctx); }
        else if ( credential_count == 2 ) { id_idx = alloc_rider_two_cred(ctx); }
        else                              { id_idx = alloc_rider_spillover(ctx); }

        update_credentials = update_id_hash = 1;

    }



    if (id_idx < 0)     // we've got a new record
    //else		// otherwise, we're creating a new record...
    {

        if      ( credential_count == 1 ) { id_idx = alloc_rider_one_cred(ctx); }
        else if ( credential_count == 2 ) { id_idx = alloc_rider_two_cred(ctx); }
        else                              { id_idx = alloc_rider_spillover(ctx); }

        if(DB_FAIL(id_idx))
        {
            fprintf(stderr, "Error (%d) trying to allocate rider\n", id_idx);
            return id_idx;
        }

        ctx->num_active++;
        ctx->num_free--;

        update_credentials = update_id_hash = 1;
        is_new_record = 1;

    }    

    if(update_credentials)
    {
        delete_from_mag_hash(ctx, id_idx);
        delete_from_rf_hash(ctx, id_idx);
    }

    if(update_id_hash)
    {
        delete_from_id_hash(ctx, id_idx);
    }


    // Update our reference count to the rule/rule param for this
    // rider record.  ruleparam_db_update will automatically create
    // a new rule if necessary and, if a new rule is created, will
    // synchronize to it's database.
    //
    // We need to update ruleparam_db so that the record will be found
    //   by copy_record.  After it's been updated, we can decrement
    //   the reference count if need be.
    //
    // At the very end, update ruleparam_db context with most recent
    //   sequence number and save the sequence to disk.
    //
    // - If we die after update but before copy_record completes,
    //   we have old sequence number, at worst a new rule/param will be inserted
    // - If we die after copy_record, the record will have newer sequence number
    //   and get deleted, waiting for a replay
    // - If we die after rule/param decrement, and the rule gets removed from ruleparam.db,
    //   credential will also get removed as it has newer sequnce number.
    // - once we write sequence number, we're in a consistent state
    //

    ruleparam_db_update( ctx->ruleparam_db, rec->rule_name, rec->rule_param, 1 );

#ifdef PASSDB_CONSISTENCY_CHECK
    retval = ruleparam_db_consistency_check( ctx->ruleparam_db );
    if (retval < 0)
    {
        fprintf(stderr, "CONSISTENCY CHECK failed in update_rider: got %i\n", retval );
    }
#endif


    copy_rider( ctx, id_idx, rec  );


    if (decrement_old_rule_ref)
    {
        ruleparam_db_update( ctx->ruleparam_db, rr.rule_name, rr.rule_param, -1 );

#ifdef PASSDB_CONSISTENCY_CHECK
        retval = ruleparam_db_consistency_check( ctx->ruleparam_db );
        if (retval < 0)
        {
            fprintf(stderr, "CONSISTENCY CHECK failed in update_rider: got %i\n", retval );
        }
#endif

    }


    passdb_slim_manage_rider_banks(ctx);

    if(sync)
    {
        sync_rider_change(ctx, id_idx);
    }

    if(update_id_hash)
    {    
        retval = add_to_id_hash(ctx, id_idx);

        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) updating id hash for record seq = %llu\n", retval, (unsigned long long)rec->seq);
        }
    }

    if(update_credentials)
    {
        retval = add_to_mag_hash(ctx, id_idx);

        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) updating magstripe hash for record seq = %llu\n", retval, (unsigned long long)rec->seq);
        }

        retval = add_to_rf_hash(ctx, id_idx);

        if( !DB_OKAY(retval) )
        {
            fprintf(stderr, "Error (%d) updating rf hash for record seq = %llu\n", retval, (unsigned long long)rec->seq);
        }
    }

    ctx->seq = rec->seq;


    // Rate limited sync always updates sequence number file,
    // occassionally updates ruleparam.db.  ruleparam.db updated
    // above with relevant information if we added or removed
    // rules.
    //
    ctx->ruleparam_db->seq = ctx->seq;
    ruleparam_db_rate_limited_sync( ctx->ruleparam_db );

    return 1;
}




void dump_hashes(passdb_slim_context *ctx)
{
    int i;
    rider_node *p;
    rider_record rr = {0};

    if(!ctx)
    {
        printf("NULL Context!\n");
        return;
    }

    printf("ID HASH[%i]:\n", ctx->hash_modulus);
    for(i=0; i < ctx->hash_modulus; i++)
    {
        if(!ctx->logical_card_id_hash[i]) continue;

        printf("\t%d:", i);

        p = ctx->logical_card_id_hash[i];
        while(p)
        {
            make_rider_record(ctx, &rr, p->idx);
            printf("  [%d] %llu", p->idx, rr.id );
            p = p -> next;
        }
        printf("\n");  
    }

    printf("RFID HASH[%i]:\n", ctx->hash_modulus);
    for(i=0; i < ctx->hash_modulus; i++)
    {
        if(!ctx->rider_rf_hash[i]) continue;

        printf("\t%d:", i);

        p = ctx->rider_rf_hash[i];
        while(p)
        {
            make_rider_record(ctx, &rr, p->idx);
            printf("  [%d] %llu", p->idx, rr.id );
            p = p -> next;
        }
        printf("\n");  
    }

    printf("MAGSTRIPE HASH[%i]:\n", ctx->hash_modulus);
    for(i=0; i < ctx->hash_modulus; i++)
    {
        if(!ctx->rider_mag_hash[i]) continue;

        printf("\t%d:", i);

        p = ctx->rider_mag_hash[i];
        while(p)
        {
            make_rider_record(ctx, &rr, p->idx);
            printf("  [%d] %llu", p->idx, rr.id );
            p = p -> next;
        }
        printf("\n");  
    }

};


//-- database expansion functions

int expand_one_cred(passdb_slim_context *ctx)
{
    int i;
    char fn[FN_SZ];
    int cur_bank;
    int fd;
    void *p;
    size_t file_size;
    int r;

    long pagesize;

    rider_node *q = NULL;

    pagesize = sysconf(_SC_PAGE_SIZE);

    cur_bank = ctx->n_one_cred_bank;

    sprintf(fn, "%s%i%s", ctx->one_cred_db_base_fn, cur_bank, ctx->db_fn_suffix );
    r = format_new_passdb( fn, ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE, pagesize );
    if (r<0)
      return r;

    r = init_mmap_passfile( &fd, &p, &file_size, RIDER_ONE_CRED_SIZE, fn );
    if (r<0)
      return r;

    for (i=0; i < ctx->n_one_cred_bank_size; i++)
    {
      q = (rider_node *) malloc( sizeof(rider_node) );

      if (!q)
      {
        fprintf(stderr, "FAIL_MEM: expand_one_cred\n");
        return FAIL_MEM;
      }

      q->next = ctx->freelist_one_cred;
      q->idx = i + (cur_bank * ctx->n_one_cred_bank_size);
      ctx->freelist_one_cred = q;

      ctx->num_free++;
    }


    ctx->passes_one_cred_bank_fd[cur_bank] = fd;
    ctx->rider_one_cred_bank[cur_bank] = p;
    snprintf( ctx->one_cred_db_bank_fn[cur_bank] , FN_SZ, "%s", fn );

    ctx->n_one_cred_bank++;
    return 0;
}

// unused and untested
// EXPERIMENTAL:
// We only expand.  In the future we might want to contract, so this
// function is left here for a starting point.  Contraction still
// needs work, so only use this as a starting point.
//
/*
int contract_one_cred(passdb_slim_context *ctx)
{
    int cur_bank;
    int min_idx, max_idx;
    rider_node *freenod = NULL, *prevnod = NULL, *nexnod = NULL;
    void *p;

    rider_record rr = {0};
    int old_idx, new_idx;

    cur_bank = ctx->n_one_cred_bank-1;
    if (cur_bank < 0) return -1;

    min_idx = cur_bank * ctx->n_one_cred_bank_size ;
    max_idx = (cur_bank + 1) * ctx->n_one_cred_bank_size ;

    // delete all nodes in the free list that fall into the
    // bank we're about to delete.
    // We'll be moving current riders into lower banks
    // and choosing free indexes based on this list,
    // so it needs to happen first.
    //
    freenod = ctx->freelist_one_cred;
    while (freenod)
    {

      nexnod = freenod->next;

      if ( (freenod->idx >= min_idx) && 
           (freenod->idx < max_idx) )
      {
        if (prevnod)
          prevnod->next = nexnod;
        free(freenod);
      }
      else
      {
        prevnod = freenod;
      }

      freenod = nexnod;
    }

    // Now find all records that fall within the index
    // range, delete all references to it in the
    // id, mag and rf hash.  To be "safe", 0
    // out record in the old bank.
    // Allocate a new index (from the lower banks)
    // and put it back in.
    //
    freenod = ctx->activelist;
    while (freenod)
    {

      nexnod = freenod->next;

      if ( (freenod->idx < min_idx) && 
           (freenod->idx >= max_idx) )
      {
        prevnod = freenod;
        freenod = nexnod;
        continue;
      }

      old_idx = freenod->idx;
      make_rider_record( ctx, &rr, old_idx );

      delete_from_id_hash(ctx, old_idx);
      delete_from_mag_hash(ctx, old_idx);
      delete_from_rf_hash(ctx, old_idx);

      //just in case?
      p = passdb_slim_get_record_address( ctx, old_idx );
      memset(p, 0, sizeof(char)*RIDER_ONE_CRED_SIZE );

      new_idx = alloc_rider_one_cred( ctx );
      copy_rider( ctx, new_idx, &rr );

      free(freenod);
      if (prevnod)
        prevnod->next = nexnod;
      freenod = nexnod;

    }

    // finally, munmap the file, free the relevant memory
    // and decrement the number of one credential banks.
    //
    munmap( ctx->rider_one_cred_bank[cur_bank], ctx->n_one_cred_bank_size * RIDER_ONE_CRED_SIZE );
    ctx->rider_one_cred_bank[cur_bank] = NULL;

    close(ctx->passes_two_cred_bank_fd[cur_bank]);
    ctx->passes_one_cred_bank_fd[cur_bank] = -1;

    free( ctx->one_cred_db_bank_fn[cur_bank] );
    ctx->one_cred_db_bank_fn[cur_bank] = NULL;

    ctx->n_one_cred_bank--;

    return 0;
}
*/


//-----

int expand_two_cred(passdb_slim_context *ctx)
{
    int i;
    char fn[FN_SZ];
    int cur_bank;
    int fd;
    void *p;
    size_t file_size;
    int r;

    long pagesize;

    rider_node *q = NULL;

    pagesize = sysconf(_SC_PAGE_SIZE);

    cur_bank = ctx->n_two_cred_bank;

    sprintf(fn, "%s%i%s", ctx->two_cred_db_base_fn, cur_bank, ctx->db_fn_suffix );
    r = format_new_passdb( fn, ctx->n_two_cred_bank_size * RIDER_TWO_CRED_SIZE, pagesize );
    if (r<0)
      return r;

    r = init_mmap_passfile( &fd, &p, &file_size, RIDER_TWO_CRED_SIZE, fn );
    if (r<0)
      return r;

    for (i=0; i < ctx->n_two_cred_bank_size; i++)
    {
      q = (rider_node *) malloc( sizeof(rider_node) );

      if (!q)
      {
        fprintf(stderr, "FAIL_MEM: expand_two_cred\n");
        return FAIL_MEM;
      }


      q->next = ctx->freelist_two_cred;
      q->idx = INDEX_MIDPOINT + i + (cur_bank * ctx->n_two_cred_bank_size);
      ctx->freelist_two_cred = q;

      ctx->num_free++;
    }

    ctx->passes_two_cred_bank_fd[cur_bank] = fd;
    ctx->rider_two_cred_bank[cur_bank] = p;
    snprintf( ctx->two_cred_db_bank_fn[cur_bank] , FN_SZ, "%s", fn );
    ctx->n_two_cred_bank++;

    return 0;
}

// unused and untested
//
// EXPERIMENTAL:
// We only expand.  In the future we might want to contract, so this
// function is left here for a starting point.  Contraction still
// needs work, so only use this as a starting point.
//
/*
int contract_two_cred(passdb_slim_context *ctx)
{
    int cur_bank;
    int min_idx, max_idx;
    rider_node *freenod = NULL, *prevnod = NULL, *nexnod = NULL;
    void *p;

    rider_record rr = {0};
    int old_idx, new_idx;

    cur_bank = ctx->n_two_cred_bank-1;
    if (cur_bank < 0) return -1;

    min_idx = cur_bank * ctx->n_two_cred_bank_size ;
    max_idx = (cur_bank + 1) * ctx->n_two_cred_bank_size ;

    min_idx += INDEX_MIDPOINT;
    max_idx += INDEX_MIDPOINT;

    // delete all nodes in the free list that fall into the
    // bank we're about to delete.
    // We'll be moving current riders into lower banks
    // and choosing free indexes based on this list,
    // so it needs to happen first.
    //
    freenod = ctx->freelist_two_cred;
    while (freenod)
    {

      nexnod = freenod->next;

      if ( (freenod->idx >= min_idx) && 
           (freenod->idx < max_idx) )
      {
        if (prevnod)
          prevnod->next = nexnod;
        free(freenod);
      }
      else
      {
        prevnod = freenod;
      }

      freenod = nexnod;
    }

    // Now find all records that fall within the index
    // range, delete all references to it in the
    // id, mag and rf hash.  To be "safe", 0
    // out record in the old bank.
    // Allocate a new index (from the lower banks)
    // and put it back in.
    //
    freenod = ctx->activelist;
    while (freenod)
    {

      nexnod = freenod->next;

      if ( (freenod->idx < min_idx) && 
           (freenod->idx >= max_idx) )
      {
        prevnod = freenod;
        freenod = nexnod;
        continue;
      }

      old_idx = freenod->idx;
      make_rider_record( ctx, &rr, old_idx );

      delete_from_id_hash(ctx, old_idx);
      delete_from_mag_hash(ctx, old_idx);
      delete_from_rf_hash(ctx, old_idx);

      //just in case?
      p = passdb_slim_get_record_address( ctx, old_idx );
      memset(p, 0, sizeof(char)*RIDER_ONE_CRED_SIZE );

      new_idx = alloc_rider_two_cred( ctx );
      copy_rider( ctx, new_idx, &rr );

      free(freenod);
      if (prevnod)
        prevnod->next = nexnod;
      freenod = nexnod;

    }

    // finally, munmap the file, free the relevant memory
    // and decrement the number of two credential banks.
    //
    munmap( ctx->rider_two_cred_bank[cur_bank], ctx->n_two_cred_bank_size * RIDER_TWO_CRED_SIZE );
    ctx->rider_two_cred_bank[cur_bank] = NULL;

    close(ctx->passes_two_cred_bank_fd[cur_bank]);
    ctx->passes_two_cred_bank_fd[cur_bank] = -1;

    free( ctx->two_cred_db_bank_fn[cur_bank] );
    ctx->two_cred_db_bank_fn[cur_bank] = NULL;

    ctx->n_two_cred_bank--;


    return 0;
}
*/


// ---
// SPILLOVER
//

int expand_spillover(passdb_slim_context *ctx)
{
    int i;
    char fn[FN_SZ];
    int cur_bank;
    int fd;
    void *p;
    size_t file_size;
    int r;
    long pagesize;

    rider_node *q = NULL;

    pagesize = sysconf(_SC_PAGE_SIZE);

    cur_bank = ctx->n_spillover_bank;

    sprintf(fn, "%s%i%s", ctx->spillover_db_base_fn, cur_bank, ctx->db_fn_suffix );
    r = format_new_passdb( fn, ctx->n_spillover_bank_size * RIDER_SPILLOVER_SIZE, pagesize );
    if (r<0)
      return r;

    r = init_mmap_passfile( &fd, &p, &file_size, RIDER_SPILLOVER_SIZE, fn );
    if (r<0)
      return r;

    for (i=0; i < ctx->n_spillover_bank_size; i++)
    {
      q = (rider_node *) malloc( sizeof(rider_node) );

      if (!q)
      {
        fprintf(stderr, "FAIL_MEM: expand_spillover\n");
        return FAIL_MEM;
      }


      q->next = ctx->freelist_spillover;
      q->idx = (2*INDEX_MIDPOINT) + i + (cur_bank * ctx->n_spillover_bank_size);
      ctx->freelist_spillover = q;

      ctx->num_free++;
    }

    ctx->passes_spillover_bank_fd[cur_bank] = fd;
    ctx->rider_spillover_bank[cur_bank] = p;
    snprintf( ctx->spillover_db_bank_fn[cur_bank] , FN_SZ, "%s", fn );
    ctx->n_spillover_bank++;

    return 0;
}



int passdb_slim_manage_rider_banks(passdb_slim_context *ctx)
{
    int r;
    int upper_threshold;
    int lower_threshold;

    float hi_water;
    float lo_water;

    int bank_sz;
    int max_bank;
    int n;
    int n_bank;

    int save_config_flag = 0;
    passdb_slim_config cfg = {0};

    hi_water = ctx->high_watermark_threshold;
    lo_water = ctx->low_watermark_threshold;


    n         = ctx->n_one_cred;
    n_bank    = ctx->n_one_cred_bank;
    bank_sz   = ctx->n_one_cred_bank_size;
    max_bank  = ctx->n_one_cred_max_bank;

    //check one credenteial watermarks
    //
    upper_threshold = (int)( hi_water * (float)(bank_sz * n_bank) );
    lower_threshold = (int)( lo_water * (float)(bank_sz * n_bank) );

    // make sure we haven't maxed out our number of banks
    //
    if ( n_bank < max_bank  )
    {
      if ( ctx->n_one_cred > upper_threshold )
      {
        r = expand_one_cred(ctx);
        if (r<0) return r;

        save_config_flag = 1;
      }
    }


    n         = ctx->n_two_cred;
    n_bank    = ctx->n_two_cred_bank;
    bank_sz   = ctx->n_two_cred_bank_size;
    max_bank  = ctx->n_two_cred_max_bank;


    //check two credenteial watermarks
    //
    upper_threshold = (int)( hi_water * (float)(bank_sz * n_bank) );
    lower_threshold = (int)( lo_water * (float)(bank_sz * n_bank) );


    // make sure we haven't maxed out our number of banks
    //
    if ( n_bank < max_bank )
    {

      if ( n > upper_threshold )
      {
        r = expand_two_cred(ctx);
        if (r<0) return r;

        save_config_flag = 1;
      }

    }


    n         = ctx->n_spillover;
    n_bank    = ctx->n_spillover_bank;
    bank_sz   = ctx->n_spillover_bank_size;
    max_bank  = ctx->n_spillover_max_bank;


    //check two credenteial watermarks
    //
    upper_threshold = (int)( hi_water * (float)(bank_sz * n_bank) );
    lower_threshold = (int)( lo_water * (float)(bank_sz * n_bank) );


    // make sure we haven't maxed out our number of banks
    //
    if ( n_bank < max_bank )
    {

      if ( n > upper_threshold )
      {
        r = expand_spillover(ctx);
        if (r<0) return r;

        save_config_flag = 1;
      }

    }


    // --


    if (save_config_flag)
    {
      passdb_slim_copy_config( &cfg, ctx );
      save_config( &cfg, PASSDB_SLIM_CONFIG_FILE );
    }


    return 0;
}

// ------------------------------------------------
// CONSISTENCY CHECK

int _consist_update_ruleparam_entry( passdb_slim_ruleparam **rule_head, rider_record *rr ) 
{
  static int id=1;
  passdb_slim_ruleparam *nod = NULL, *prv = NULL;

  nod = *rule_head;
  if ( !nod )
  {
    nod = _alloc_ruleparam_db_nod( id++, rr->rule_name, rr->rule_param );
    nod->reference_count++;
    *rule_head = nod;
    return id;
  }

  while (nod)
  {

    if ( (strncmp( nod->name , rr->rule_name  , RULENAME_LEN) == 0) &&
         (strncmp( nod->param, rr->rule_param , PARAM_LEN ) == 0) )
    {
      nod->reference_count++;
      return nod->id;
    }

    prv = nod;
    nod = nod->next;
  }


  prv->next = _alloc_ruleparam_db_nod( id++, rr->rule_name, rr->rule_param );

  nod = prv->next;
  nod->reference_count++;
  return id;

}


int passdb_slim_consistency_check( passdb_slim_context *ctx )
{
  //int idx;
  int acount1=0, acount2=0, acounts=0;
  int fcount1=0, fcount2=0, fcounts=0;
  int tot1=0, tot2=0, tots=0;

  int bank_acount1=0, bank_acount2=0, bank_acounts=0;

  int bank, pos;
  void *rider_p;

  int k;
  int rule_id;
  char name[1024], param[1024];

  rider_node *nod;

  rider_record_slim_one_cred  rr_one_cred;
  rider_record_slim_two_cred  rr_two_cred;
  rider_record                rr_spillover;
  rider_record rr;

  passdb_slim_ruleparam *rule_head = NULL, *rule_nod = NULL;

  nod = ctx->activelist;
  while (nod)
  {

    if ( nod->idx < INDEX_MIDPOINT  )
    {
      acount1++;
      tot1++;
    }
    else if (nod->idx < (2*INDEX_MIDPOINT))
    {
      acount2++;
      tot2++;
    }
    else
    {
      acounts++;
      tots++;
    }

    nod = nod->next;
  }

  nod = ctx->freelist_one_cred;
  while (nod)
  {
    tot1++;
    fcount1++;
    nod = nod->next;
  }


  nod = ctx->freelist_two_cred;
  while (nod)
  {
    tot2++;
    fcount2++;
    nod = nod->next;
  }

  nod = ctx->freelist_spillover;
  while (nod)
  {
    tots++;
    fcounts++;
    nod = nod->next;
  }


  // Check that the active record count for each
  // type appears in the banks and the counts match up

  if ( tot1 != (ctx->n_one_cred_bank * ctx->n_one_cred_bank_size) )
    return -1;

  if ( tot2 != (ctx->n_two_cred_bank * ctx->n_two_cred_bank_size) )
    return -2;

  if ( tots != (ctx->n_spillover_bank * ctx->n_spillover_bank_size) )
    return -3;

  if ( acount1 != ctx->n_one_cred )
    return -4;

  if ( acount2 != ctx->n_two_cred )
    return -5;

  if ( acounts != ctx->n_spillover )
    return -6;




  // go through each of the elements in the banks and
  // make sure they appear properly in the hash
  //
  // OK, this is a bit hard as we don't keep the information
  // about what index is associated with which entry in the actual
  // bank.  That is, from the active list we can infer the position
  // in the bank, but from the bank we would need to search the whole
  // active list.
  // Instead, count all entries.  If the counts match up and all
  // active list entries map properly, we can then infer consistency
  // between the active list and the banks.
  //
  for (bank=0; bank < ctx->n_one_cred_bank; bank++)
  {
    for (pos=0; pos < ctx->n_one_cred_bank_size; pos++)
    {
      rider_p = ctx->rider_one_cred_bank[bank] + (pos*RIDER_ONE_CRED_SIZE);
      populate_one_cred_rider_record( &rr_one_cred, 0, rider_p );
      if (rr_one_cred.id == ID_INVALID)
        continue;

      bank_acount1++;
    }
  }

  for (bank=0; bank < ctx->n_two_cred_bank; bank++)
  {
    for (pos=0; pos < ctx->n_two_cred_bank_size; pos++)
    {
      rider_p = ctx->rider_two_cred_bank[bank] + (pos*RIDER_TWO_CRED_SIZE);
      populate_two_cred_rider_record( &rr_two_cred, 0, rider_p );
      if (rr_two_cred.id == ID_INVALID)
        continue;

      bank_acount2++;
    }
  }

  for (bank=0; bank < ctx->n_spillover_bank; bank++)
  {
    for (pos=0; pos < ctx->n_spillover_bank_size; pos++)
    {
      rider_p = ctx->rider_spillover_bank[bank] + (pos*RIDER_SPILLOVER_SIZE);
      populate_spillover_rider_record( &rr_spillover, 0, rider_p );
      if (rr_spillover.id == ID_INVALID)
        continue;

      bank_acounts++;
    }
  }

  if ( (bank_acount1 != acount1) ||
       (bank_acount2 != acount2) ||
       (bank_acounts != acounts) )
  {

    fprintf(stderr, "CONSISTENCY ERROR: bank_acount1 (%i) !=? acount1 (%i),  bank_acount2 (%i) !=? acount2 (%i) bank_acounts (%i) !=? acounts (%i)\n",
        bank_acount1, acount1,
        bank_acount2, acount2,
        bank_acounts, acounts );

    return -8;
  }

  // Make sure all records in activelist have a valid
  // entry in the appropriate bank.
  //
  nod = ctx->activelist;
  while (nod)
  {
    make_rider_record( ctx, &rr, nod->idx );
    if (rr.id == ID_INVALID)
      return -9;

    nod = nod->next;
  }

  // we get to this point, hash and banks match up:
  // go through all active records, add to a temporary
  // rule param list and count the number of references,
  // make sure it matches up to the ruleparam stored
  //

  // construct our own version of the rule names and parameters
  //
  nod = ctx->activelist;
  while (nod)
  {

    make_rider_record( ctx, &rr, nod->idx );
    _consist_update_ruleparam_entry( &rule_head, &rr );

    nod = nod->next;
  }

  // compare each reference count to what we have in memory
  //
  rule_nod = rule_head;
  while (rule_nod)
  {

    if ( rule_nod->reference_count != (k=ruleparam_db_reference_count( ctx->ruleparam_db, rule_nod->name, rule_nod->param )) )
    {
      rule_id = ruleparam_db_find( ctx->ruleparam_db, rule_nod->name, rule_nod->param );
      ruleparam_db_get( name, param, ctx->ruleparam_db, rule_id );
      fprintf(stderr, "CONSISTENCY ERROR: ruleparam_db reference count error: (%s,%s %i) != (%s,%s %i)\n", 
          rule_nod->name, rule_nod->param, rule_nod->reference_count,
          name, param, k );
      return -10;
    }

    rule_nod = rule_nod->next;
  }

  // free our local copy of the rule param
  //
  rule_nod = rule_head ;
  while (rule_nod)
  {
    rule_nod = rule_nod->next;
    free(rule_head);
    rule_head = rule_nod;
  }


  return 1;
}

/*

int main(int argc, char **argv)
{
    passdb_slim_context ctx = {0};
    int retval;

    rider_record foo = {12362, 6, "SILLYRF", "FOOBAR", "STUPIDRULE", "STUPIDPARAM"};
    rider_record bar = {12354, 7, "WAWAWA", "SILLYMAG", "STUPIDRULE", "STUPIDPARAM"};
    rider_record baz = {12361, 8, "", "BARFOLA_MAG", "STUPIDRULE2", "STUPIDPARAM2"};

    rider_record bat = {12363, 7, "", "", "", ""};

    retval = attach_to_passdb(&ctx);

    dump_hashes(&ctx);

    retval = update_rider(&ctx, &foo);
    retval = update_rider(&ctx, &bar);
    retval = update_rider(&ctx, &baz);

    retval = delete_rider(&ctx, &bat);

    dump_hashes(&ctx);

    return 0;
}

*/