/*
     This file is part of libextractor.
     (C) 2002, 2003, 2004, 2005, 2006 Vidyut Samanta and Christian Grothoff

     libextractor is free software; you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published
     by the Free Software Foundation; either version 2, or (at your
     option) any later version.

     libextractor is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     General Public License for more details.

     You should have received a copy of the GNU General Public License
     along with libextractor; see the file COPYING.  If not, write to the
     Free Software Foundation, Inc., 59 Temple Place - Suite 330,
     Boston, MA 02111-1307, USA.
 */
/**
 * Tool to build a bloomfilter from a dictionary.
 */

#include "platform.h"
#include <string.h>
#include "bloomfilter.h"

/**
 * Sets a bit active in the bitArray. Increment bit-specific
 * usage counter on disk only if below 4bit max (==15).
 *
 * @param bitArray memory area to set the bit in
 * @param bitIdx which bit to set
 */
static void setBit(unsigned char * bitArray,
		   unsigned int bitIdx) {
  unsigned int arraySlot;
  unsigned int targetBit;

  arraySlot = bitIdx / 8;
  targetBit = (1L << (bitIdx % 8));
  bitArray[arraySlot] |= targetBit;
}

/**
 * Callback: increment bit
 *
 * @param bf the filter to manipulate
 * @param bit the bit to increment
 * @param arg not used
 */
static void setBitCallback(Bloomfilter * bf,
			   unsigned int bit,
			   void * arg) {
  setBit(bf->bitArray,
	 bit);
}

/**
 * Add an element to the filter
 *
 * @param bf the filter
 * @param e the element
 */
static void addToBloomfilter(Bloomfilter * bf,
		             const HashCode160 * e) {

  if (NULL == bf)
    return;
  iterateBits(bf,
	      &setBitCallback,
	      NULL,
	      e);
}


#define ADDR_PER_ELEMENT 46


int main(int argc,
	 char ** argv) {
  Bloomfilter bf;
  HashCode160 hc;
  int i;
  int j;
  int cnt;
  char * fn;
  char ** words;
  char line[2048]; /* buffer overflow, here we go */
  FILE *dictin;
  char * bn;
  char * charset = NULL;
#define ALLOCSIZE 1024*1024

  if (argc<3) {
    fprintf(stderr,
	    _("Please provide the name of the language you are building\n"
	      "a dictionary for.  For example:\n"));
    fprintf(stderr, "$ ./dictionary-builder ./en en > en.c\n");
    exit(-1);
  }

  fn = malloc(strlen(argv[1]) + 6);
  strcpy(fn, argv[1]);
  strcat(fn, ".txt");
  dictin=fopen(fn,"r");
  free(fn);
  if (dictin==NULL) {
    fprintf(stderr,
	    _("Error opening file `%s': %s\n"),
	    argv[1],strerror(errno));
    exit(-1);
  }

  words = malloc(sizeof(char*) * ALLOCSIZE); /* don't we LOVE constant size buffers? */
  if (words == NULL) {
    fprintf(stderr,
	    _("Error allocating: %s\n."),
	    strerror(errno));
    exit(-1);
  }
  cnt = 0;
  memset(&line[0], 0, 2048);
  fscanf(dictin, "%s", (char*)&line);
  charset = strdup(line); /* not used (yet) */
  while (1 == fscanf(dictin, "%s", (char*)&line)) {
    words[cnt] = strdup(line);
    cnt++;
    memset(&line[0], 0, 2048);
    if (cnt >= ALLOCSIZE) {
      fprintf(stderr,
	      _("Increase ALLOCSIZE (in %s).\n"),
	      __FILE__);      
      exit(-1);
    }
  }

  bf.addressesPerElement = ADDR_PER_ELEMENT;
  bf.bitArraySize = (1 + (cnt / SUBTABLES)) * sizeof(int) * SUBTABLES;
  bf.bitArray = malloc(bf.bitArraySize);
  memset(bf.bitArray, 0, bf.bitArraySize);

  for (i=0;i<cnt;i++) {
    hash(words[i],
	 strlen(words[i]),
	 &hc);
    addToBloomfilter(&bf, &hc);
  }

  fprintf(stdout,
	  "#include \"bloomfilter-def.h\"\n");

  /* use int[] instead of char[] since it cuts the memory use of
     gcc down to a quarter; don't use long long since various
     gcc versions then output tons of warnings about "decimal constant
     is so large that it is unsigned" (even for unsigned long long[]
     that warning is generated and dramatically increases compile times). */
  for (j=0;j<SUBTABLES;j++) {
    char fn[64];
    FILE * btfile;

    snprintf(fn, 64, "%s_%d.c", argv[1], j);
    btfile = fopen(fn, "w+");
    if (btfile == NULL) {
      fprintf(stderr,
	      _("Error opening file `%s': %s\n"),
	      fn, strerror(errno));
      exit(-1);
    }
    fprintf(btfile,
	    "int %s_bits_%d[] = { ", argv[2], j);
    for (i= j    * (bf.bitArraySize/sizeof(int)/SUBTABLES);
	 i<(j+1) * (bf.bitArraySize/sizeof(int)/SUBTABLES);
	 i++)
      fprintf(btfile,
	      "%dL,",
	      (((int*)bf.bitArray)[i]));
    fprintf(btfile,
	    "};\n");
    fclose(btfile);
    fprintf(stdout,
	    "extern int %s_bits_%d[];\n", argv[2], j);
  }

  fprintf(stdout,
	  "static int * bits[] = { ");
  for (i=0;i<SUBTABLES;i++)
    fprintf(stdout,
	    "%s_bits_%d,",
	    argv[2],
	    i);
  fprintf(stdout,
	  "};\n");
  bn = &argv[1][strlen(argv[1])];
  while ( (bn != argv[1]) &&
	  (bn[0] != '/') )
    bn--;
  if (bn[0] == '/')
    bn++;
  fprintf(stdout,
	  "Bloomfilter libextractor_printable_%s_filter = {\n"
	  "  %u,\n"
	  "  NULL,\n" /* bitarray */
	  "  (unsigned char **)bits,\n" /* sbitArray */
	  "  %u };\n",
	  bn,
	  ADDR_PER_ELEMENT,
	  bf.bitArraySize);
  free(charset);
  return 0;
}