/* vi: set sw=4 ts=4: */
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h> 
#include <ctype.h>
#include <sys/types.h>
#include <string.h>

#include "defs.h"
#include "convert.h"
#include "hzinput.h"
#include "pinyin.h"
#include "safestring.h"
#include "cin2tab.h"

#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>  
#endif

typedef struct 
{
	u_int key1;
	u_int key2;
	union 
	{
		u_int  ch;
		u_char phrase[4];
	}phr;  /* not everybody supports anonymous union! */
	u_short occur_seq;

	u_char type;
} ITEM2;

#define ITEMTABLE_MAXNUM 150000 /* 50000  Rui He Jul 21, 2003 changed to support GBK */

#define ITEMTABLE_TOTALSIZE (sizeof(ITEM2) * ITEMTABLE_MAXNUM)

ITEM2 *ItemTable, *ItemTmp;

#define PHRASE_BUFFER_SIZE 1536000 /* max 1.5M phrase buffer 700k phrase big enough? */ 
u_char *PhraseBuffer;

typedef int (*CMPFUNCPTR)(const void *, const void *);

int qcmp2(ITEM2* a, ITEM2* b)
{
	if (a->key1 > b->key1) return 1;
	if (a->key1 < b->key1) return -1;
	if (a->key2 > b->key2) return 1;
	if (a->key2 < b->key2) return -1;
	if (a->phr.ch > b->phr.ch) return 1;
	if (a->phr.ch < b->phr.ch) return -1;

	return 0;
}

int qcmp(ITEM2* a, ITEM2* b)
{
	if (a->key1 > b->key1) return 1;
	if (a->key1 < b->key1) return -1;
	if (a->key2 > b->key2) return 1;
	if (a->key2 < b->key2) return -1;
	return (int)a->occur_seq - (int)b->occur_seq;
}

#ifdef SUPPORT_ASSOCIATE

typedef struct 
{
	int size;
	int pos;
	unsigned char *buf;
}AssocItem;

#define ASSOCIATE_TRUNK 128

#define ASSOCIATE_SIZE (128 * 192)
AssocItem ascItem[ASSOCIATE_SIZE]; /* all zero */

int AddToAscTable(int Char, char *phrase, int len)
{
	int index,i,l;
	unsigned char *p, *s;
	
	index = CalculateCharIndex(SysCoding, Char >> 8, Char & 0xFF); 
	if (index < 0 || index >= ASSOCIATE_SIZE || len < 2) return -1; /* fail */

	if (ascItem[index].buf == NULL)
	{
		ascItem[index].buf = (unsigned char *)malloc(ASSOCIATE_TRUNK);		
		ascItem[index].size = ASSOCIATE_TRUNK;
	}
	else if (ascItem[index].pos + len >= ascItem[index].size)
	{
		ascItem[index].buf = (unsigned char *)realloc(ascItem[index].buf, 
					ascItem[index].size + ASSOCIATE_TRUNK);
		ascItem[index].size += ASSOCIATE_TRUNK;
	}

	if (!ascItem[index].buf)
	{
		fprintf(stderr, "Out of memory for Associate Items!\n");
		exit(-1);
	}
	s = ascItem[index].buf; /*starting */
	p = ascItem[index].buf + ascItem[index].pos; /*end+1 */

	while (s < p)
	{
		l = strlen(s);
		if (!strcmp(phrase, s)) return -1; /* duplicate */
		s += l+1; /* skip to next string */
	}
	
	for(i = 0; i < len ; i++)
		*p++ = *phrase++;
	*p++ = '\0';
	ascItem[index].pos += len+1;

	return 0;
}

int WriteAssociateFile(char *filename)
{
	AssociateHeader ascHeader;
	int i, min, max, offset;
	FILE *fp;
	
	strcpy(ascHeader.MagicNum, ASSOCIATE_MAGIC_NUMBER);
	ascHeader.Encoding = SysCoding;
	min = -1;
	max = -1;
	offset = 0;
    for(i = 0; i < ASSOCIATE_SIZE; i++)
	{
		ascItem[i].size = offset; /* current offset */
		if (ascItem[i].pos > 0)
		{
			offset += ascItem[i].pos;
			if (min == -1) min = i;
			if (i > max) max = i;
		}
	}
	max += 1; /* add one for dummy */

	if (min == -1)
	{
		fprintf(stderr, "Now associate phrase found! No file generated!\n");
		return -1;
	}
	
	fp = fopen(filename, "w");
	if (fp == NULL)
	{
		fprintf(stderr, "Error opening file %s\n", filename);
		exit(-1);
	}
	
	ascHeader.PhraseOffset = sizeof(ascHeader) + sizeof(int) * (max - min +1 );
	ascHeader.MinIndex = (unsigned short)min;
	ascHeader.MaxIndex = (unsigned short)max;
	
	fprintf(stderr, "Now Generating LianXiang file %s: MinIndex=%d MaxIndex=%d...\n",
			filename, min, max);
	
	fwrite(&ascHeader, sizeof(ascHeader), 1, fp);
	for(i = min;i <= max; i++)
		fwrite(&ascItem[i].size, sizeof(int), 1, fp);
	
	for(i = min; i < max; i++)
	{
		if (ascItem[i].pos > 0)
		{
			fwrite(ascItem[i].buf, ascItem[i].pos, 1, fp);
			free(ascItem[i].buf);
		}
	}

	fclose(fp);
	return 0;
}

int GenerateAssociateFile(FILE *fr, int gen_lx, char *fname)
{
	u_char *p, *s;
	int  IndexChar = 0, len;
	
    while (!feof(fr))
    {
		if (gen_lx == 1)
		{
			get_line(fr, LineBuf);
		   	arg = skip_space(LineBuf);
		   	if (strlen(arg) < 4 || arg[0] == '#' || arg[0] == '%') continue;
		}
		else
		{
			cmd_arg(fr,LineBuf, &cmd, &arg);
			if (cmd[0] == '%' || cmd[0] == '#' || strlen(cmd) < 2) continue;
			IndexChar = ((int)cmd[0]) << 8 | cmd[1];
		}
		
		p = arg;
		while (*p && *p != '\r' && *p != '\n' && *p != '#')
		{
			while (*p && *p < 0x80) p++; /* go to a Chinese/Japanese/Korean char */
			s = p;
			p = to_space(p);
			len = p - s;
			if (*p) *p++ = '\0'; /* still has something */
			if (gen_lx == 1 && len >= 4)
			{
				IndexChar = ((int)s[0]) << 8 | s[1];
				AddToAscTable(IndexChar, s+2, len-2);
			}
			else if (gen_lx == 2)
			{
				if (len >= 2) AddToAscTable(IndexChar, s, len);	
			}
		}
	}

	return WriteAssociateFile(fname);
}
#endif

static void HandleDifferentMbFormat(u_char **pcmd, u_char **parg, int *plen)
{
	u_char *s, *t, tmp1, tmp2;
	int len = 0;
	
	s = *pcmd;
	t = *parg;
	if (!t[0]) /* ZiMa or MaZi? */
	{
		if (s[0] > 0x80)  /* ZiMa */
		{
			t = s;
			while(*t > 0x80 && t[1] >= 0x40) t += 2;
			tmp1 = '\0';
			*pcmd = t+1; /* pointing to ma */
			while(*t)   /*move the Ma one byte right */
			{
				tmp2 = *t;
				*t++ = tmp1;
				tmp1 = tmp2;
				len++;
			}
			*t = tmp1;	/* last byte */
			*parg = s; /* pointing to zi */
			*plen = len;  /* key len */
		}
		else  /* MaZi */
		{
			t = s;
			while(*t && *t < 0x80) t++; /* skip all the ma */
			*parg = t; /* pointing to zi now */
			*plen = (u_int)t - (u_int)s;
		}
	}
	else
	{
		if (s[0] > 0x80 &&  t[0] < 0x80)  /* Zi Ma */
		{
			*pcmd = t;
			*parg = s;
		}

		*plen = strlen(*pcmd); /* calculate key len */
	}
	
}

static int CurIndex,ItemCount;
static int phrasebuf_pointer=0;
static int totalchar = 0, totalphrase = 0;
static int maxphraselen = 0;
static int keysperint;
static int ignoredmap = 0;	

static void read_cin_phrase(FILE *fr, hz_input_table *table)
{
	int i;
	u_char *save_arg;

   	keysperint = 32 / table->key_len; /* 5 or 6 */
	memset(ItemTable, 0, ITEMTABLE_TOTALSIZE );

	CurIndex=0;
	for( ; ; cmd_arg(fr,LineBuf,&cmd,&arg))
	{
		int len;
		u_int key1,key2;
		int k;

		if (!cmd[0]) 
		{
			if (feof(fr)) break;
			continue;
		}
		
		if (cmd[0] == '%')  /* skip some unknown keywords */
		{
			if (!strcasecmp(cmd, "%chardef"))
			{
				if (!strcasecmp(arg, "end")) break;  /* end */
				if (!strcasecmp(arg, "begin")) continue;  /* read one line more */
				continue; /* continue anyway */
			}
			if (table->KeyMap[cmd[0]] <= 0)  /* not a valid input key? */
			{
				fprintf(stderr, "Unknown CIN keyword %s in mapping section, ignored.\n", cmd);
				continue;
			}
		}

		HandleDifferentMbFormat(&cmd, &arg, &len);
		if (len < 1 || !cmd[0] || !arg[0])
		{
			fprintf(stderr, "Warning: Line %d invalid mapping '%s %s' ignored.\n",
					lineno, cmd, arg);
			ignoredmap++;
			continue;
		}
		
		if (len > table->MaxPress) table->MaxPress=len;
		if (len > 2 * keysperint) 
		{
			fprintf(stderr, "Key %s: keylen %d detected, wll take max %d keys!\n", cmd, len, 2 * keysperint);
			len = 2 * keysperint;
		}

		key1=0;
		key2=0;
		for(i=0;i<len;i++) 
        {
			k = table->KeyMap[cmd[i]];

			if (k <= 0 && cmd[i] >= 'A' && cmd[i]	<= 'Z')
			   k = table->KeyMap[tolower(cmd[i])];

			if (k == 0)
			{

				fprintf(stderr, "Line %d Error: key %c(0x%02X) of '%s %s' is not in keymap!\n", 
					lineno, cmd[i], cmd[i], cmd, arg);
				ignoredmap++;
				continue;
			}
			
			if(i < keysperint /* 5 */)  /* 5 * 6 bit/key = 30 bits */
            {
				key1 |= k << ((keysperint - i - 1) * table->key_len);
			}
			else 
            {
				key2 |= k << ((2 * keysperint - i - 1) * table->key_len);
			}
		} /* for */

		for (; *arg ; arg = save_arg)		
		{
			while(*arg == ' ' || *arg == '\t') arg++; /* skip the space */
			len = 0;
			while(arg[len] && arg[len] != ' ' && arg[len] != '\t' 
				&& arg[len] != '\n' && arg[len] != '\r') len++; /* skip to space or end of line */
			save_arg = arg+len;
			if (*save_arg)   *save_arg++ = '\0'; /* terminate the current string */

			if (len < 2 || len % 2 == 1) 
			{
#ifdef DEBUG
				fprintf(stderr, "Warning: Line %d invalid phrase %s(bytes=%d), ignored.\n",
						lineno, arg, len);
#endif
				ignoredmap++;
				continue; /* len -= 1; // some error skip */
			}
			if (!VerifyMapping(cmd, arg, len)) continue; /* skip */

			memcpy(&ItemTable[CurIndex].key1,&key1,4);
			memcpy(&ItemTable[CurIndex].key2,&key2,4);

           	if (len==2 || charonly)  /*  && (int)(*arg) > 0xA0 && (int)(*(arg+1)) > 0xA0)  // FIXME!!! */
			{
		    	while(len > 1)
				{
					memcpy(&ItemTable[CurIndex].phr.ch,arg,2);
					ItemTable[CurIndex].phr.phrase[2] = ItemTable[CurIndex].phr.phrase[3] = '\0';
					ItemTable[CurIndex].type = TYPE_CHAR;  /* mark it as char */
					len -= 2;
					if (len >= 2)
					{
						ItemTable[CurIndex].occur_seq=CurIndex;
						CurIndex++;  /* close the old one */
						if (CurIndex >= ITEMTABLE_MAXNUM) break;
				    	memcpy(&ItemTable[CurIndex].key1,&key1,4);
                    	memcpy(&ItemTable[CurIndex].key2,&key2,4);				
						arg += 2; /* move to next char */
					}
				}
			}
        	else 
        	{
            	int m;
		
				ItemTable[CurIndex].type = TYPE_PHRASE;

				if (len > 4)
				{

					if ((unsigned)(phrasebuf_pointer + len + 1) >= PHRASE_BUFFER_SIZE) 
					{
						fprintf(stderr, "Phrase buffer overrun: %d + %d > %d, please increase it.\n",
							phrasebuf_pointer, len, PHRASE_BUFFER_SIZE);
						abort();
					}
		
					if (len > MAX_PHRASE_LENGTH)
					{
						fprintf(stderr, "Line %d Warning: Phrase '%s' Len %d ignored(MaxLen=%d).\n",
								lineno, arg, len/2, MAX_PHRASE_LENGTH/2);
						ignoredmap++;
						continue;

					}

					ItemTable[CurIndex].phr.phrase[0] = ((u_int)(phrasebuf_pointer) >> 24) & 0xFF;
					ItemTable[CurIndex].phr.phrase[1] = ((u_int)(phrasebuf_pointer) >> 16) & 0xFF;
					ItemTable[CurIndex].phr.phrase[2] = ((u_int)(phrasebuf_pointer) >> 8) & 0xFF;
					ItemTable[CurIndex].phr.phrase[3] = ((u_int)(phrasebuf_pointer)) & 0xFF;

					strcpy(&PhraseBuffer[phrasebuf_pointer],arg);
					phrasebuf_pointer += len+1; /* including the ending 0 */
				}
				else
				{
					strcpy((unsigned char *)&(ItemTable[CurIndex].phr.ch), arg); /* <=4 */
				}

				if (len > maxphraselen) maxphraselen = len;

#ifdef SUPPORT_ASSOCIATE
				if (gen_lx == 3)
				{
					m = ((int)arg[0]) << 8 | arg[1];
					AddToAscTable(m, arg+2, len-2);
				}
#endif
			}  /* else */

			ItemTable[CurIndex].occur_seq=CurIndex;  /* occur_seq for both char/phrase */
			CurIndex++;

			if (CurIndex >= ITEMTABLE_MAXNUM) break;
			
		} /* while (*arg) */

		if (CurIndex >= ITEMTABLE_MAXNUM) 
		{
			fprintf(stderr, "\nERROR: Maximum mapping %d reached, ignoring the remaining!\n\n",
					ITEMTABLE_MAXNUM);
			break;
		}

	} /* for loop */
}

static void eliminate_duplicate_phrase(hz_input_table *table)
{
	u_short CharDef[MAX_INPUT_KEYS];
	int i;
	ITEM2 LastItem;

	qsort(ItemTable,CurIndex,sizeof(ITEM2), (CMPFUNCPTR)qcmp2);  

	memset(&LastItem, 0, sizeof(ITEM2));
	ItemCount=0;
	for(i=0;i<CurIndex;i++) 
	{
		if (ItemTable[i].key1 != LastItem.key1 || ItemTable[i].key2 != LastItem.key2 || ItemTable[i].phr.ch != LastItem.phr.ch)
       	{
  	    	memcpy(&ItemTmp[ItemCount++],&ItemTable[i],sizeof(ITEM2));
	    	memcpy(&LastItem,&ItemTable[i],sizeof(ITEM2));
		}
		else 
		{
#ifdef DEBUG
			fprintf(stderr, "Eliminated: CurIndex=%d key1=%08X key2=%08X char=%c%c%c%c ch=%08X\n", 
			 ItemTable[i].occur_seq, ItemTable[i].key1, ItemTable[i].key2, ItemTable[i].phr.phrase[0], ItemTable[i].phr.phrase[1],
			 ItemTable[i].phr.phrase[2] ? ItemTable[i].phr.phrase[2] : ' ', ItemTable[i].phr.phrase[3] ? ItemTable[i].phr.phrase[3] : ' ',
			ItemTable[i].phr.ch);
#endif
		}
	}

	CurIndex=ItemCount;

	qsort(ItemTmp,CurIndex,sizeof(ITEM2), (CMPFUNCPTR)qcmp);

	memset(CharDef, 0, sizeof(CharDef));
	for(i=0;i<CurIndex;i++) 
	{
    	int kk=(ItemTmp[i].key1 >> ( (keysperint-1)*table->key_len /*24*/ )) & 
			(( 1 << table->key_len)-1); 
    	if (!CharDef[kk]) 
    	{
        	table->KeyIndex[kk]=(u_short)i;  /* start index */
        	CharDef[kk]=1;
    	}
	}

	table->KeyIndex[table->TotalKey]=CurIndex; /* last one is dummy, used for EndKey */

	for(i=table->TotalKey-1; i>0; i--)
		if (!CharDef[i]) table->KeyIndex[i] = table->KeyIndex[i+1];

	if (table->wildcard) /* keyCurIndex */
	{  /* wildcard at first char, will map to KeyIndex[1], last one will map to CurIndex */
		table->KeyIndex[(int)table->wildcard] = table->KeyIndex[1];
		table->KeyIndex[(int)table->wildcard+1] = CurIndex;
	}

#ifdef DEBUG
	for(i = 0; i < 64; i++)
 		fprintf(stderr, "Index %d = %d\n",i,table->KeyIndex[i]);
#endif
}

static void write_tab_file(char *filename, hz_input_table *table)
{
	int i;
	FILE *fw;
	
	if ((fw=fopen(filename,"w"))==NULL) 
	{
		print_error("Failed to create output file!");
	}

	if (table->MaxPress <= keysperint)  /* can use one integer to hold all the keys */
		table->ItemBytes = 8;  /* ch + key1 */
	else table->ItemBytes = 12;  /* ch + key1 + key2 */

	table->PhraseFileSize = phrasebuf_pointer; /* 0 for no file */
	table->TotalEntries = CurIndex;
	table->PhraseBegin = sizeof(*table) + CurIndex * table->ItemBytes;

#ifdef DEBUG
	fprintf(stderr, "PhraseFileSize=%d, PhraseBegin=%d InpTableSize=%d\n", 
		table->PhraseFileSize, table->PhraseBegin, sizeof(*table));
#endif

	fwrite(table,sizeof(*table),1, fw);
	
	for(i=0;i<CurIndex;i++)
	{
		if (ItemTmp[i].type == TYPE_PHRASE) totalphrase++;
		else if (ItemTmp[i].type == TYPE_CHAR) totalchar++;
		else { fprintf(stderr, "Error! non-char and non-phrase!\n"); exit(1); }

		fwrite(&ItemTmp[i].phr.ch, 4, 1, fw);  
		fwrite(&ItemTmp[i].key1, 4, 1, fw);
		if (table->ItemBytes > 8)
			fwrite(&ItemTmp[i].key2, 4, 1, fw);
		
#ifdef DEBUG

#endif
	}

	if (phrasebuf_pointer > 0 )  /* has some phrases with more than 2 chars */
	{
		fwrite(PhraseBuffer,1,phrasebuf_pointer + MAX_PHRASE_LENGTH,fw); /* add some padding at the end */
	}

	fclose(fw);

}

int ConvertTableCIN(FILE *fr, hz_input_table *table, char *cinfname, 
		char *tabfname, char *lxfname)
{
	ItemTable = (ITEM2 *)malloc(ITEMTABLE_TOTALSIZE);
	ItemTmp = (ITEM2 *)malloc(ITEMTABLE_TOTALSIZE);
	PhraseBuffer = (u_char *)malloc(PHRASE_BUFFER_SIZE);

	if (!ItemTable || !ItemTmp || !PhraseBuffer) 
		print_error("ConvertTableCIN: Out of memory error!\n");

	read_cin_phrase(fr, table);

	eliminate_duplicate_phrase(table);

	write_tab_file(tabfname, table);

	fprintf(stdout, "%s:\t" /* BytesPerEntry=%d */ "MaxKeyNum=%d  TotalInputKeys=%d  KeyBitsLen=%d\n"
                    "%s:\t%d Single Chars  %d Phrases(MaxLen=%d)  %d Ignored  Total %d\n", 
		cinfname, /* table->ItemBytes, */ table->MaxPress, 
		table->TotalKey-1, table->key_len,  table->cname,
		totalchar, totalphrase, maxphraselen/2, ignoredmap, ItemCount);

#ifdef SUPPORT_ASSOCIATE
	if (gen_lx == 3)	WriteAssociateFile(lxfname);
#endif

	fclose(fr);

	return 0;
}