/************************** * * * * * * * * * * * * ***************************
    Copyright (c) 1999-2005 Ryan Bobko
                       ryan@ostrich-emulators.com

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.     
************************** * * * * * * * * * * * * ***************************/

#include "qhacc.h"
#include "qhaccext.h"
#include "qhaccutils.h"
#include "qhacctable.h"
#include "qhaccsegmenter.h"
#include "qhaccconstants.h"
#include "localfileplugin.h"

#include <qdir.h>

#include <stdlib.h>
#include <sys/stat.h> 

const int QHaccExt::OLDID=0;
const int QHaccExt::NEWID=1;
const int QHaccExt::DUPE =2;
const int QHaccExt::NCOLS=3;

QHaccExt::QHaccExt( QHacc * eng ){ engine=eng; }
QHaccExt::~QHaccExt(){}

QHaccResultSet * QHaccExt::getRSSet(){
	QHaccResultSet *  rslts=new QHaccResultSet[QC::NUMTABLES];
	for( int i=0; i<QC::NUMTABLES; i++ ){
		Table t=( Table )i;
		rslts[i]=QHaccResultSet( Utils::tcols( t ), Utils::ttypes( t ) );
	}
	return rslts;
}

void QHaccExt::archive( const QDate& date ){ 
	uint rr=0;
	TableSelect ts( QC::TDATE, TableCol( date ), TableSelect::LT );
	if( iarchive( engine->getWhere( TRANSACTIONS,	
																	vector<TableSelect>( 1,	ts ),
																	rr ) ) ){
		engine->resetOBals();
		QString err;
		engine->save( err );
	}
}

void QHaccExt::archive( const Account& acct ){
	TableCol aid=acct[QC::AID];
	auto_ptr<QHaccResultSet> trans( new QHaccResultSet( QC::TCOLS,
																											QC::TCOLTYPES ) );
	uint rr=0;
	vector<TableSelect> v;
	auto_ptr<QHaccResultSet> xt=engine->getXTForA( acct, TableGet(), v, rr );
	for( uint i=0; i<rr; i++ ){
		TableRow tr, sp;
		engine->splitXTrans( xt->at( i ), tr, sp );
		trans->add( tr );
	}
	
	if( iarchive( trans ) ){
		// by this time, we've removed all the transactions and stuff, but we
		// still need to remove the account and repoint any child accounts to
		// some other parent
		
		engine->updateWhere( ACCOUNTS, TableSelect( QC::APID, aid ), 
												 TableUpdate( QC::APID, TableCol( 0 ) ) );
		engine->deleteWhere( ACCOUNTS, TableSelect( QC::AID, aid ) );
		
		engine->resetOBals();
		QString err;
		engine->save( err );
	}
}

bool QHaccExt::iarchive( auto_ptr<QHaccResultSet> trans ){
	// do the actual archiving of the transactions

	// basically, remove all the stuff that would get removed during an
	// engine->remA() on the account, which means:
	// 1: remove scheduled trans that reference the namedtrans
	// 2: remove named trans that include removed transactions
	// 3: remove transactions and splits that reference this account
	// Note that we don't actually do the row deletions from
	// the engine, but against local QHaccTables. This is to
	// (greatly, in some cases) reduce the amount of unnecessary
	// reindexing that has to occur between deletions.


	QHaccResultSet * data=getRSSet();
	engine->exprt( data );
	
	const int NTBLS=4;
	const int TR=0, SP=1, NA=2, JB=3;
	const int TBLS[]={ QC::TRANT, QC::SPLTT, QC::NAMET, QC::JOBST };
	QHaccTable * tbls=new QHaccTable[NTBLS];
	for( int i=0; i<NTBLS; i++ ){
		tbls[i]=QHaccTable( data[TBLS[i]] );
		tbls[i].setPK( Utils::tpk( ( Table )TBLS[i] ) );
	}
	

	uint r=trans->rows(); // how many trans are we going to archive?
	std::ostream *str=0;
	if( Utils::debug( Utils::DBGMINOR, str ) ){
		*str<<"archiving "<<r<<" transaction"<<( r==1 ? "" : "s" )<<endl;
	}

	for( uint i=0; i<r; i++ ){
		const TableRow& row=trans->at( i );
		
		uint rr=0;
		TableSelect tt( QC::NTID, row[QC::TID] );
		vector<TableSelect> v( 1, tt );
		auto_ptr<QHaccResultSet> names=tbls[NA].getWhere( TableGet( QC::NNAME ),
																											v, rr );

		for( uint j=0; j<rr; j++ ){
			// step 1:
			const TableRow& nt=names->at( j );
			tbls[JB].deleteWhere( TableSelect( QC::JWHAT, nt[0] ) );
		}
		
		// step 2:
		tbls[NA].deleteWhere( tt );

		// step 3:
		tbls[SP].deleteWhere( TableSelect( QC::STID, row[QC::TID] ) );
		tbls[TR].deleteWhere( TableSelect( QC::TID, row[QC::TID] ) );
	}
	
	engine->db->setAtom( BEGIN );
	// actually delete ALL the old data from the engine
	for( int i=NTBLS-1; i>=0; i-- )
		engine->deleteWhere( ( Table )TBLS[i], TableSelect() );

	// import the new data into the engine
	bool good=true;
	for( int i=0; i<NTBLS; i++ ){
		good=( good && engine->load( ( Table )TBLS[i], &tbls[i] ) );
	}
	if( good ) engine->db->setAtom( COMMIT );
	else{
		if( Utils::error( Utils::ERROPER, str ) )
			*str<<"Error archiving data"<<endl;
		engine->db->setAtom( ROLLBACK );
	}

	delete [] data;
	return good;
}

void QHaccExt::restore( QHaccIOPlugin * reader ){
	QHaccResultSet * rslts=getRSSet();
	reader->exprt( rslts );
	// we need to check for duplicate accounts, and journals, but luckily,
	// we're just checking by IDs because there is no way to introduce
	// duplicate IDs during the archive/restore process
	engine->db->setAtom( BEGIN );
	uint rr=0;
	vector<TableSelect> v;
	uint lim=rslts[QC::JRNLT].rows();
	for( uint i=0; i<lim; i++ ){
		v.clear();
		v.push_back( TableSelect( QC::LID, rslts[QC::JRNLT][i][QC::LID] ) );
		auto_ptr<QHaccResultSet> leds=engine->getWhere( JOURNALS,
																										TableGet( QC::LID ),
																										v, rr );
		if( rr==0 ) engine->add( JOURNALS, rslts[QC::JRNLT][i] );
	}

	lim=rslts[QC::ACCTT].rows();
	for( uint i=0; i<lim; i++ ){
		v.clear();
		v.push_back( TableSelect( QC::AID, rslts[QC::ACCTT][i][QC::AID] ) );
		auto_ptr<QHaccResultSet> leds=engine->getWhere( ACCOUNTS,
																										TableGet( QC::AID ),
																										v, rr );
		if( rr==0 )	engine->add( ACCOUNTS, rslts[QC::ACCTT][i] );
	}
	
	// now load the other important tables
	int TBLS[]={ QC::TRANT, QC::SPLTT, QC::NAMET, QC::JOBST };
	bool good=true;
	for( int i=0; i<4; i++ ){
		good=( good && engine->load( ( Table )TBLS[i], &rslts[TBLS[i]] ) );
	}
	if( good ){
		engine->db->setAtom( COMMIT );

		engine->resetOBals();
		QString err;
		engine->save( err );
	}
	else{
		engine->db->setAtom( ROLLBACK );

		std::ostream * str=0;
		if( Utils::error( Utils::ERROPER, str ) ){
			*str<<"Error restoring archive"<<endl;
		}
	}
	delete [] rslts;
}

bool QHaccExt::reco( const char * fname ){
	QHaccPlugin * timports=0;
	QString home=engine->getPluginFor( QHacc::PIIMPORTER, fname, timports );
	QHaccIOPlugin * imports=( QHaccDBPlugin * )timports;

	// get the data from the datastore "into" QHacc
	QString err;
	if( imports->connect( engine, home, err ) && imports->load( err ) ){
		
		// create a set of (appropriately duplicated) data for reconciling
		bool forReconcile=true;
		QHaccResultSet * rslts=getMergedImpOf( imports, forReconcile );
		engine->destroyPlugin( QHacc::PIIMPORTER, imports );

		engine->db->setAtom( BEGIN );

		for( uint i=0; i<rslts[QC::SPLTT].rows(); i++ ){
			engine->setRecNR( rslts[QC::SPLTT][i], QC::YREC );
		}

		engine->db->setAtom( COMMIT );

		delete [] rslts;
	}
	else{
		std::ostream * str=0;
		if( Utils::error( Utils::ERROPER, str ) ) *str<<err<<endl;
		return false;
	}
	
	return true;
}

bool QHaccExt::exp( const char * dirname ){
	QHaccResultSet * rslts=getRSSet();
	engine->exprt( rslts );

	QHaccPlugin * texports=0;
	QString home=engine->getPluginFor( QHacc::PIEXPORTER, dirname, texports );
	QHaccIOPlugin * exports=( QHaccIOPlugin * )texports;

	QString err;
	bool good=exports->connect( engine, home, err );
	if( good ){
		exports->imprt( rslts );
		good=exports->save( err );
	}
	
	delete [] rslts;
	std::ostream * str=0;
	if( !good && Utils::error( Utils::ERROPER, str ) ) *str<<err<<endl;
	return good;
}

bool QHaccExt::imp( const char * homedir ){
	QHaccPlugin * timports=0;
	QString home=engine->getPluginFor( QHacc::PIIMPORTER, homedir, timports );
	QHaccIOPlugin * imports=( QHaccIOPlugin * )timports;

	// get the data from the datastore "into" QHacc
	QString err;
	if( imports->connect( engine, home, err ) && imports->load( err ) ){
		
		// create a consistent set of (non-duplicated) data for importing,
		bool forReconcile=false;
		QHaccResultSet * rslts=getMergedImpOf( imports, forReconcile );
		engine->destroyPlugin( QHacc::PIIMPORTER, imports );

		engine->imprt( rslts );

		delete [] rslts;
		if( engine->getBP( "CONSERVEIDSONIMPORT" ) ){
			// we're going to get all the gaps out of the various ID columns
			// from the engine's current data (including what we just imported)
			// this step is totally unnecessary, but if you're anal like I am,
			// it may interest you

			std::ostream * str=0;
			if( Utils::debug( Utils::CURIOSITY, str ) )
				*str<<"conserving table id numbers"<<endl;

			rslts=getRSSet();
			engine->exprt( rslts );
			conserveIDs( rslts );

			replaceEngineData( rslts );
			delete [] rslts;
		}
		
	}
	else{
		std::ostream * str=0;
		if( Utils::error( Utils::ERROPER, str ) ) *str<<err<<endl;
		return false;
	}

	return true;
}

TableGet QHaccExt::getGetter( Table t, bool forReco ){
	// return a vector of columns that define duplicate rows in a table
	vector<int> v;
	switch( t ){
	case JOURNALS:
		v.push_back( QC::LNAME );
		break;
	case ACCOUNTS:
		v.push_back( QC::ANAME );
		break;
	case XTRANS:
		if( forReco ) v.push_back( QC::XTPAYEE );
		v.push_back( QC::XTDATE );
		v.push_back( QC::XSSUM );
		v.push_back( QC::XSACCTID );
		break;
	case NAMEDTRANS:
		v.push_back( QC::NNAME );
		v.push_back( QC::NACCTID );
		v.push_back( QC::NTID );
		break;
	case JOBS:
		v.push_back( QC::JWHAT );
		break;
	default:
		break;
	}

	return TableGet( v );
}

void QHaccExt::dupeError( Table t, const TableRow& row, const QString& print ){
	std::ostream * str=0;
	if( !Utils::error( Utils::ERROPER, str ) ) return;
	
	bool printit=true;
	
	if( t==JOURNALS ) printit=( row[QC::LNAME]!="General" ) ;
	else if( t==ACCOUNTS ) printit=( row[QC::AID]!=0 );
		
	
	if( printit ) 
		*str<<"duplicate "<<QC::TABLENAMES[( int )t]<<": "<<print<<endl;
}

QHaccResultSet * QHaccExt::getMergedImpOf( QHaccIOPlugin * imports,
																					 bool forReco ) const {
	QHaccResultSet * ret=getRSSet();

	// should we add duplicates anyway?
	bool rawload=imports->info().rawloader();

	imports->exprt( ret );

	QHaccTable tbls[]={
		QHaccTable( QC::PCOLS, QC::PCOLTYPES ),
		QHaccTable( QC::LCOLS, QC::LCOLTYPES ),
		QHaccTable( QC::ACOLS, QC::ACOLTYPES ),
		QHaccTable( QC::TCOLS, QC::TCOLTYPES ),
		QHaccTable( QC::SCOLS, QC::SCOLTYPES ),
		QHaccTable( QC::NCOLS, QC::NCOLTYPES ),
		QHaccTable( QC::JCOLS, QC::JCOLTYPES )
	};
	
	// load the resultsets (imported data) into a qhacctable (for updating)
	for( int i=0; i<QC::NUMTABLES; i++ ){
		tbls[i]+=ret[i];
		tbls[i].setPK( Utils::tpk( ( Table )i ) );
	}
	
	delete [] ret;


	////////////// JOURNAL TABLE //////////////
	// we need to update the journal table and the transaction
	// table, since both reference journalids.
	TableGet getter=getGetter( JOURNALS, forReco );
	auto_ptr<QHaccResultSet> rs=getMerged( JOURNALS, tbls[QC::JRNLT], QC::LID,
																				 QC::LID, getter, getter );
	uint rr=rs->rows();
	QHaccTableIndex index( rs.get(), NEWID, CTUINT );
	for( uint i=0; i<rr; i++ ){
		// we go through the index backwards so that we're updating
		// rows from highest to lowest id. this will avoid corrupting
		// the database when a duplicate id comes up
		const TableRow& row=rs->at( index[rr-1-i] );
		const TableCol& oldid=row[OLDID];
		const TableCol& newid=row[NEWID];
		bool dupe=row[DUPE].getb();

		TableSelect myts( QC::LID, oldid );
		const TableRow& l=tbls[QC::JRNLT].getWhere( myts );

		if( dupe && !rawload ){
			dupeError( JOURNALS, l, l[QC::LNAME].gets() );
			tbls[QC::JRNLT].deleteWhere( myts );
		}
		else tbls[QC::JRNLT].updateWhere( myts, TableUpdate( QC::LID, newid ) );

		// update any transactions necessary
		tbls[QC::TRANT].updateWhere( TableSelect( QC::TLID, oldid ),
																 TableUpdate( QC::TLID, newid ) );
	}
	
	////////////// ACCOUNTS TABLE //////////////
	const TableSelect Z( QC::AID, TableCol( 0 ) );
	const Account zero=tbls[QC::ACCTT].getWhere( Z );
	tbls[QC::ACCTT].deleteWhere( Z );

	getter=getGetter( ACCOUNTS, forReco );
	auto_ptr<QHaccResultSet>rs2=getMerged( ACCOUNTS, tbls[QC::ACCTT], QC::AID,
																				 QC::AID, getter, getter );
	rs=rs2;
	rr=rs->rows();

	index=QHaccTableIndex( rs.get(), NEWID, CTUINT );
	for( uint i=0; i<rr; i++ ){
		const TableRow& row=rs->at( index[rr-1-i] );
		const TableCol& oldid=row[OLDID];
		const TableCol& newid=row[NEWID];
		bool dupe=row[DUPE].getb();

		TableSelect myts( QC::AID, oldid );
		const TableRow& a=tbls[QC::ACCTT].getWhere( myts );
		if( dupe && !rawload ){
			if( !forReco ) dupeError( ACCOUNTS, a, a[QC::ANAME].gets() );
				tbls[QC::ACCTT].deleteWhere( myts );

		}
		else tbls[QC::ACCTT].updateWhere( myts, TableUpdate( QC::AID, newid ) );

		// need to repoint child accounts
		tbls[QC::ACCTT].updateWhere( TableSelect( QC::APID, oldid ),
																 TableUpdate( QC::APID, newid ) );
		
		// update splits and namedtrans
		tbls[QC::SPLTT].updateWhere( TableSelect( QC::SACCTID, oldid ),
																 TableUpdate( QC::SACCTID, newid ) );
		tbls[QC::NAMET].updateWhere( TableSelect( QC::NACCTID, oldid ),
																 TableUpdate( QC::NACCTID, newid ) );
	}
	if( !zero.isNull() ) tbls[QC::ACCTT]+=zero;

	////////////// TRANSACTIONS TABLE //////////////
	// there's no good way to find dupes of transactions
	// without also finding dupes of splits, so we'll
	// do these two tables together by creating a XTrans
	// table and finding dupes of those.
	tbls[QC::SPLTT].addIndexOn( QC::STID );
	QHaccTable xtrans( QC::XCOLS, QC::XCOLTYPES );

	rr=tbls[QC::TRANT].rows();
	xtrans.startLoad( rr );

	for( uint i=0; i<rr; i++ ){
		Transaction t=tbls[QC::TRANT][i];
		uint rr2=0;
		auto_ptr<QHaccResultSet> splits=
			tbls[QC::SPLTT].getWhere( TableSelect( QC::STID, t[QC::TID] ), rr2 );

		for( uint j=0; j<rr2; j++ ){
			xtrans+=QHacc::makeXTrans( t, splits->at( j ) );
		}
	}
	xtrans.stopLoad();
	xtrans.addIndexOn( QC::XTID );

	// now we can check for duplicates
	getter=getGetter( XTRANS, forReco );	
	auto_ptr<QHaccResultSet>rs3=getMerged( XTRANS, xtrans, QC::XTID, QC::XTID,
																				 getter, getter );
	rs=rs3;

	// we'll need these in case of duplicates
	const int DF=engine->getIP( "DATEFORMAT" );
	const QString DS=engine->getSP( "DATESEPARATOR" );
	
	TableCol lastid;
	rr=rs->rows();
	index=QHaccTableIndex( rs.get(), NEWID, CTUINT );
	
	if( forReco ) tbls[QC::SPLTT].clear();
		
	for( uint i=0; i<rr; i++ ){
		const TableRow& row=rs->at( index[rr-1-i] );
		const TableCol& oldid=row[OLDID];
		const TableCol& newid=row[NEWID];
		bool dupe=row[DUPE].getb();
		
		if( forReco ){
			if( dupe ){
				// we have a dupe and we're reconciling...we need to figure out
				// what the split that corresponds
				uint rr2=0;
				vector<TableSelect> v( 1, TableSelect( QC::TID, newid ) );
				auto_ptr<QHaccResultSet> rs2=engine->getWhere( TRANSACTIONS, 
																											 TableGet( QC::TID ), 
																											 v , rr2 );
				// we're guaranteed a single row return here
				// we're going to reconcile all the splits
				// associated with this 
				QHaccTable splts=engine->getTSplits( rs2->at( 0 ).getu( 0 ) );
				tbls[QC::SPLTT]+=splts;
			}
		}
		else{
			if( oldid!=lastid ){
				lastid=oldid; 
			
				const TableRow& xt=xtrans.getWhere( TableSelect( QC::XTID, oldid ) );
			
				const TableSelect myts( QC::TID, oldid );
				const TableSelect mysts( QC::STID, oldid );
				const TableSelect mynts( QC::NTID, oldid );
				
				if( dupe && !rawload ){
					dupeError( TRANSACTIONS, xt, 
										 xt[QC::XTPAYEE].gets()+" on "+
										 Utils::stringFromDate( xt[QC::XTDATE].getd(), DS, DF )+
										 " for "+xt[QC::XSSUM].gets() );
					
					tbls[QC::TRANT].deleteWhere( myts );
					tbls[QC::SPLTT].deleteWhere( mysts );
					
					// need to worry about scheduled trans, too
					uint rr2=0;
					auto_ptr<QHaccResultSet> st=tbls[QC::NAMET].getWhere( mynts, rr2 );
					for( uint i=0; i<rr2; i++ ){
						const TableSelect T( QC::JWHAT, st->at( i ).get( QC::NNAME ) );
						tbls[QC::JOBST].deleteWhere( T );
					}
					tbls[QC::NAMET].deleteWhere( mynts );
				}

				Transaction t;
				Split s;
				engine->splitXTrans( xt, t, s );
				
				tbls[QC::TRANT].updateWhere( myts,  TableUpdate( QC::TID,  newid ) );
				tbls[QC::SPLTT].updateWhere( mysts, TableUpdate( QC::STID, newid ) );
				tbls[QC::NAMET].updateWhere( mynts, TableUpdate( QC::NTID, newid ) );
			} // lastid check
		}  // forReco check
	}

	// if we're reconciling, don't change any split ids
	/*
	if( forReco ){
		for( uint i=0; i<tbls[QC::SPLTT].rows(); i++ ){
			cout<<tbls[QC::SPLTT][i].toString()<<endl;
		}
	}
	*/

	if( !forReco ){
		////////////// SPLITS TABLE //////////////
		// we're not merging splits...we're just
		// keeping their SIDs from conflicting
		uint maxsid=engine->max( SPLITS, QC::SID ).getu()+1;
		rr=tbls[QC::SPLTT].rows();
		QHaccResultSet srs( QC::SCOLS, QC::SCOLTYPES, rr );
		for( uint i=0; i<rr; i++ ){
			Split s( tbls[QC::SPLTT][i] );
			s.set( QC::SID, TableCol( maxsid++ ) );
			srs+=s;
		}
		tbls[QC::SPLTT]=srs;
	}


	////////////// NAMEDTRANS TABLE //////////////
	getter=getGetter( NAMEDTRANS, forReco );
	
	auto_ptr<QHaccResultSet>rs4=getMerged( NAMEDTRANS, tbls[QC::NAMET], QC::NID,
																				 QC::NID, getter, getter );
	rs=rs4;
	rr=rs->rows();
	index=QHaccTableIndex( rs.get(), NEWID, CTUINT );
	for( uint i=0; i<rr; i++ ){
		const TableRow& row=rs->at( index[rr-1-i] );
		const TableCol& oldid=row[OLDID];
		const TableCol& newid=row[NEWID];
		bool dupe=row[DUPE].getb();

		TableSelect myts( QC::NID, oldid );
		const TableRow& s=tbls[QC::NAMET].getWhere( myts );
		
		if( dupe && !rawload ){
			if( !forReco ) dupeError( NAMEDTRANS, s, s[QC::NNAME].gets() );
			tbls[QC::NAMET].deleteWhere( myts );
		}
		else tbls[QC::NAMET].updateWhere( myts, TableUpdate( QC::NID, newid ) );
	}


	////////////// JOBS TABLE //////////////
	getter=getGetter( JOBS, forReco );
	
	auto_ptr<QHaccResultSet>rs5=getMerged( JOBS, tbls[QC::JOBST], QC::JID,
																				 QC::JID, getter, getter );
	rs=rs5;

	rr=rs->rows();
	index=QHaccTableIndex( rs.get(), NEWID, CTUINT );
	for( uint i=0; i<rr; i++ ){
		const TableRow& row=rs->at( index[rr-1-i] );
		const TableCol& oldid=row[OLDID];
		const TableCol& newid=row[NEWID];
		bool dupe=row[DUPE].getb();

		TableSelect myts( QC::JID, oldid );
		const TableRow& s=tbls[QC::JOBST].getWhere( myts );
		
		if( dupe && !rawload ){
			if( !forReco ) dupeError( JOBS, s, s[QC::JWHAT].gets() );
			tbls[QC::JOBST].deleteWhere( myts );
		}
		else tbls[QC::JOBST].updateWhere( myts, TableUpdate( QC::JID, newid ) );
	}


	////////////// PREFERENCES TABLE //////////////
	// we're only adding preferences that don't exist
	// we're not trying to reconcile existing prefs
	rr=tbls[QC::PREFT].rows();
	QHaccResultSet hold( QC::PCOLS, QC::PCOLTYPES );
	for( uint i=0; i<rr; i++ ){
		const TableRow& pref=tbls[QC::PREFT][i];
		uint rr2=0;
		vector<TableSelect> ts( 1, TableSelect( QC::PPREF, pref[QC::PPREF] ) );
		auto_ptr<QHaccResultSet> rslt=engine->getWhere( PREFERENCES, ts, rr2 );
		if( rr2==0 ) hold+=pref;
	}
	tbls[QC::PREFT]=hold;
	ret=getRSSet();
	for( int i=0; i<QC::NUMTABLES; i++ ) ret[i]=tbls[i];

	/*
	int tbl[]={ QC::JRNLT, QC::ACCTT, QC::TRANT, QC::SPLTT, QC::NAMET, QC::JOBST };
	for( int i=0; i<QC::NUMTABLES; i++ ){
		cout<<"table: "<<QC::TABLENAMES[tbl[i]]<<endl;
		for( uint j=0; j<tbls[tbl[i]].rows(); j++ )
			cout<<"  "<<tbls[tbl[i]][j].toString()<<endl;
		cout<<"--"<<endl;
	}
	*/

	return ret;
}

auto_ptr<QHaccResultSet> QHaccExt::getMerged( Table model, QHaccTable& import,
																							int modelidcol, int importidcol,
																							const TableGet& modelselection,
																							const TableGet& importselection )
	const {

	uint maxO=engine->max( model, modelidcol ).getu();
	uint maxI=import.max( importidcol ).getu();
	uint nextid=( maxI>maxO ? maxI : maxO )+1;
	
	auto_ptr<QHaccResultSet> ret( new QHaccResultSet( NCOLS ) );
	
	// WARNING: modelselection and importselection must have the
	// same number of selection criteria. Different fields are
	// fine, but the number is essential (and unchecked)
	const uint CNT=modelselection.cnt();

	uint rows=import.rows();
	for( uint i=0; i<rows; i++ ){
		const TableRow& row=import[i];

		// check for duplicates by selecting cols from the engine 
		// and comparing them to the cols of the imports.
		vector<TableSelect> ts;
		for( uint j=0; j<CNT; j++ ){
			ts.push_back( TableSelect( modelselection[j],
																 row[importselection[j]] ) );

			//cout<<TableSelect( modelselection[j],
			//									 row[importselection[j]] ).toString()<<endl;
		}
		
		TableRow adder( NCOLS );
		adder.set( OLDID, row[importidcol] );
		TableCol newid( row[importidcol] );
		
		
		if( CNT>0 ){ // if we're actually looking for something, then look
			uint rr=0;
			auto_ptr<QHaccResultSet> rslts=engine->getWhere( model,
																											 TableGet( modelidcol ),
																											 ts, rr );
			if( rr>0 ){
				//cout<<"found a match! id "<<rslts->at( 0 ).toString()<<endl;
				adder.set( NEWID, rslts->at( 0 ).get( 0 ) );
				adder.set( DUPE, TableCol( true ) );
			}
			else{
				//cout<<"no match"<<endl;
				adder.set( NEWID, TableCol( nextid++ ) );
				adder.set( DUPE, TableCol( false ) );
			}
		} // if CNT>0 loop
		else{ // else just update the id
			adder.set( NEWID, TableCol( nextid++ ) );
			adder.set( DUPE, TableCol( false ) ); 
		}
		ret->add( adder );
	}
	
	return ret;
}

bool QHaccExt::verify( bool andFix ){
	// verify that the database is in a consistent state. this means:
	// 1: accounts have valid parent accounts
	// 2: transactions point to actual journals
	// 3: splits point to transactions
	// 4: splits point to accounts
	// 5: jobs point to namedtrans
	// 6: namedtrans point to transactions
	//
	// return true if the database is in a consistent state

	bool good=true;
	const int CTBS[]={ QC::ACCTT, QC::TRANT, QC::SPLTT,
										 QC::SPLTT, QC::NAMET, QC::JOBST };

	const int PTBS[]={ QC::ACCTT, QC::JRNLT, QC::TRANT,
										 QC::ACCTT, QC::TRANT, QC::NAMET };

	const int CCOLS[]={ QC::APID,    QC::TLID, QC::STID,
											QC::SACCTID, QC::NTID, QC::JWHAT };

	const int PCOLS[]={ QC::AID, QC::LID, QC::TID,
											QC::AID, QC::TID, QC::NNAME };

	const ColType TCOLS[]={ CTUINT, CTUINT, CTUINT, 
													CTUINT, CTUINT, CTSTRING };
	const uint LIM=6;

	QHaccResultSet * rslts=getRSSet();
	engine->exprt( rslts );

	QHaccTable * tbls=new QHaccTable[QC::NUMTABLES];	
	for( int i=0; i<QC::NUMTABLES; i++ ) tbls[i]=QHaccTable( rslts[i] );
	delete [] rslts;
	rslts=getRSSet();

	for( uint i=0; i<LIM; i++ ){
		auto_ptr<QHaccResultSet>rslt=iverify( tbls[CTBS[i]], CCOLS[i],
																					tbls[PTBS[i]], PCOLS[i] );
		rslts[CTBS[i]]+=*rslt;
	}

	// print out the inconsistancies
	std::ostream * str=0;
	bool write=Utils::debug( Utils::ERROPER, str );
	bool haveOrphans=false;
	if( write ){
		for( int i=0; i<QC::NUMTABLES; i++ ){
			if( rslts[i].rows()>0 ) haveOrphans=true;
		}
		if( haveOrphans ) *str<<"inconsistencies found"<<endl;

		for( int i=0; i<QC::NUMTABLES; i++ ){
			const uint ROWS=rslts[i].rows();
			
			if( ROWS>0  ){ // we have some orphans
				*str<<ROWS<<" orphan"<<( ROWS>1 ? "s" : "" )<<" in "
						<<QC::TABLENAMES[i]<<endl;
					
				for( uint j=0; j<ROWS; j++ ) *str<<"  "<<rslts[i][j].toString()<<endl;
			}
		}
	}

	// After the first first pass just just did, we may have lots
	// of orphaned splits, so go back through the transactions
	// and remove any that have lost transactions (this step
	// isn't strictly necessary for a consistent database, but
	// logically, it's important.
	uint rr=rslts[QC::SPLTT].rows();
	for( uint i=0; i<rr; i++ ){
		uint rr2=0;
		const TableSelect TS( QC::TID, rslts[QC::SPLTT][i][QC::STID] );
		auto_ptr<QHaccResultSet> rslt=tbls[QC::TRANT].getWhere( TS, rr2 );
		tbls[QC::TRANT].deleteWhere( TS );
		rslts[QC::TRANT]+=*rslt;
		
		// also get rid of newly-orphaned splits
		const TableSelect TS2( QC::STID, rslts[QC::SPLTT][i][QC::STID] );
		auto_ptr<QHaccResultSet>rslt2=tbls[QC::SPLTT].getWhere( TS2, rr2 );
		tbls[QC::SPLTT].deleteWhere( TS2 );
		rslts[QC::SPLTT]+=*rslt2;
	}
	
	// now, just do the same thing for namedtrans
	rr=rslts[QC::TRANT].rows();
	for( uint i=0; i<rr; i++ ){
		uint rr2=0;
		const TableSelect TS( QC::NTID, rslts[QC::TRANT][i][QC::TID] );
		auto_ptr<QHaccResultSet> rslt=tbls[QC::NAMET].getWhere( TS, rr2 );
		tbls[QC::NAMET].deleteWhere( TS );
		rslts[QC::NAMET]+=*rslt;
	}

	// then do the same thing for jobs and we're done finding orphans
	rr=rslts[QC::NAMET].rows();
	for( uint i=0; i<rr; i++ ){
		uint rr2=0;
		const TableSelect TS( QC::JWHAT, rslts[QC::NAMET][i][QC::NNAME] );
		auto_ptr<QHaccResultSet> rslt=tbls[QC::NAMET].getWhere( TS, rr2 );
		tbls[QC::JOBST].deleteWhere( TS );
		rslts[QC::JOBST]+=*rslt;
	}

	// print out the inconsistancies
	if( write ){
		if( haveOrphans ) *str<<"--fixdb will remove the following rows"<<endl;

		for( int i=0; i<QC::NUMTABLES; i++ ){
			const uint ROWS=rslts[i].rows();
			
			if( ROWS>0  ){ // we have some orphans
				*str<<ROWS<<" orphan"<<( ROWS>1 ? "s" : "" )<<" in "
						<<QC::TABLENAMES[i]<<endl;
					
				for( uint j=0; j<ROWS; j++ ) *str<<"  "<<rslts[i][j].toString()<<endl;
			}
		}
	}

	if( andFix ){
		// we don't really fix anything, we just delete offending rows
		// we do that by deleting all rows and then loading the good ones
		replaceEngineData( tbls );
	}
	/*
	for( uint i=0; i<QC::NUMTABLES; i++ ){
		cout<<QC::TABLENAMES[i]<<":"<<endl;
		for( uint j=0; j<tbls[i].rows(); j++ )
			cout<<"  "<<tbls[i][j].toString()<<endl;
		cout<<"--"<<endl;
	}
	*/

	delete [] tbls;
	delete [] rslts;
	return haveOrphans;
}

auto_ptr<QHaccResultSet> QHaccExt::iverify( QHaccTable& child, int ccol,
																						QHaccTable& parent,	int pcol )
	const {
	// verify that the child table has a row in the parent table. If it doesn't,
	// add the row the resultset to be returned, and DELETE the row from the
	// child table. 

	// basically, get en empty resultset from a table with data in it
	uint rr=0;
	vector<TableSelect> empties;
	empties.push_back( TableSelect( 0, TableCol( 0 ) ) );
	empties.push_back( TableSelect( 0, TableCol( 1 ) ) );
	auto_ptr<QHaccResultSet> ret=child.getWhere( empties, rr );

	uint crows=child.rows();
	uint prows=parent.rows();

	/*
	cout<<"child: "<<crows<<endl;
	for( uint i=0; i<crows; i++ ) cout<<"  "<<child[i].toString()<<endl;
	cout<<"--"<<endl;
	cout<<"parent: "<<prows<<endl;
	for( uint i=0; i<prows; i++ ) cout<<"  "<<parent[i].toString()<<endl;
	cout<<"--"<<endl;
	*/

	if( crows>prows ){
		// we have fewer parents to search through than we have children
		// to check, so remove the parents we do have from the children
		// whatever is left are the orphans. We need a temporary holder
		// because we're deleting the stuff we should be keeping
		QHaccResultSet holder( *ret.get() );

		child.addIndexOn( ccol );
		for( uint i=0; i<prows; i++ ){
			const TableSelect TS( ccol, parent[i][pcol] );

			//cout<<"plooping with for "<<TS.toString()<<" in pcol "<<pcol<<endl;

			auto_ptr<QHaccResultSet> r=child.getWhere( TS, rr );
			holder+=*r.get();
			child.deleteWhere( TS );
		}

		ret->load( &child );
		child.clear();
		child+=holder;
	}
	else{
		// we have fewer children to check than parents to search, so
		// cycle through the children and look for parents. If we 
		// don't find any, we have an orphan
		parent.addIndexOn( pcol );

		// we need a resulteset to cycle through while we're deleting
		// stuff from child
		QHaccResultSet rs( child );

		for( uint i=0; i<crows; i++ ){
			uint rr=0;
			const TableRow& row=rs[i];
			auto_ptr<QHaccResultSet> rslt;
			const TableSelect TS( pcol, row[ccol] );

			//cout<<"clooping with for "<<TS.toString()<<" in ccol "<<ccol<<endl;
			
			TableRow par=parent.getWhere( TS );
			
			if( par.isNull() ){
				ret->add( row );
				const TableSelect TS( ccol, row[ccol] );
				child.deleteWhere( TS );
			}
		}
	}

	return ret;
}


QString QHaccExt::create( const QString& extra ) {
	QHaccPlugin * dbpl2=0;
	QString home=engine->getPluginFor( QHacc::PIDATABASE, extra, dbpl2 );
	QHaccDBPlugin * dbpl=( QHaccDBPlugin * )dbpl2;
	QString str=dbpl->create( home );
	engine->destroyPlugin( QHacc::PIDATABASE, dbpl );
	return str;
}

void QHaccExt::conserveIDs( QHaccResultSet * rslts ) const {
	// get the IDs in the given resultset group as small as possible
	// there are a lot of dependencies here, so be careful.

	std::ostream * str=0;
	bool writer=Utils::debug( Utils::CURIOSITY, str );

	// First, we need tables on which to operate
	QHaccTable * tbls=new QHaccTable[QC::NUMTABLES];
	for( uint i=0; i<QC::NUMTABLES; i++ ){
		tbls[i]=QHaccTable( rslts[i] );
		tbls[i].setPK( Utils::tpk( ( Table )i ) );
	}
	
	for( uint i=0; i<QC::NUMTABLES; i++ ){
		cout<<"before conserving "<<QC::TABLENAMES[i]<<" ids: "<<endl;
		for( uint j=0; j<tbls[i].rows(); j++ )
			cout<<"  "<<tbls[i][j].toString()<<endl;
		cout<<"--"<<endl;
	}

	uint change=false;
	// JOURNALS table first
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::JRNLT]<<" ids"<<endl;
	uint rows=rslts[QC::JRNLT].rows();
	QHaccTableIndex ids( &rslts[QC::JRNLT], QC::LID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::JRNLT][ids[i]][QC::LID]!=i+1 ){
			change=true;
			tbls[QC::TRANT].addIndexOn( QC::TLID );

			const TableCol NEWID( i+1 );
			const TableCol OLDID( rslts[QC::JRNLT][ids[i]][QC::LID] );

			tbls[QC::JRNLT].updateWhere( TableSelect( QC::LID, OLDID ),
																	 TableUpdate( QC::LID, NEWID ) );
			tbls[QC::TRANT].updateWhere( TableSelect( QC::TLID, OLDID ),
																	 TableUpdate( QC::TLID, NEWID ) );
		}
	}
	if( change ){
		rslts[QC::JRNLT]=tbls[QC::JRNLT];
		rslts[QC::TRANT]=tbls[QC::TRANT];
	}


	// ACCOUNTS
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::ACCTT]<<" ids"<<endl;
	change=false;
	rows=rslts[QC::ACCTT].rows();
	ids=QHaccTableIndex( &rslts[QC::ACCTT], QC::AID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::ACCTT][ids[i]][QC::AID]!=i ){ // AID==0 is a valid ID!
			change=true;
			tbls[QC::SPLTT].addIndexOn( QC::SACCTID );
			tbls[QC::NAMET].addIndexOn( QC::NACCTID );

			const TableCol NEWID( i );
			const TableCol OLDID( rslts[QC::ACCTT][ids[i]][QC::AID] );

			tbls[QC::ACCTT].updateWhere( TableSelect( QC::AID, OLDID ),
																	 TableUpdate( QC::AID, NEWID ) );
			tbls[QC::ACCTT].updateWhere( TableSelect( QC::APID, OLDID ),
																	 TableUpdate( QC::APID, NEWID ) );
			tbls[QC::SPLTT].updateWhere( TableSelect( QC::SACCTID, OLDID ),
																	 TableUpdate( QC::SACCTID, NEWID ) );
			tbls[QC::NAMET].updateWhere( TableSelect( QC::NACCTID, OLDID ),
																	 TableUpdate( QC::NACCTID, NEWID ) );
		}
	}
	if( change ){
		rslts[QC::ACCTT]=tbls[QC::ACCTT];
		rslts[QC::SPLTT]=tbls[QC::SPLTT];
		rslts[QC::NAMET]=tbls[QC::NAMET];	
	}

	
	// TRANSACTIONS
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::TRANT]<<" ids"<<endl;
	change=false;
	rows=rslts[QC::TRANT].rows();
	ids=QHaccTableIndex( &rslts[QC::TRANT], QC::TID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::TRANT][ids[i]][QC::TID]!=i+1 ){
			change=true;
			tbls[QC::SPLTT].addIndexOn( QC::STID );
			tbls[QC::NAMET].addIndexOn( QC::NTID );

			const TableCol NEWID( i+1 );
			const TableCol OLDID( rslts[QC::TRANT][ids[i]][QC::TID] );

			tbls[QC::TRANT].updateWhere( TableSelect( QC::TID, OLDID ),
																	 TableUpdate( QC::TID, NEWID ) );
			tbls[QC::SPLTT].updateWhere( TableSelect( QC::STID, OLDID ),
																	 TableUpdate( QC::STID, NEWID ) );
			tbls[QC::NAMET].updateWhere( TableSelect( QC::NTID, OLDID ),
																	 TableUpdate( QC::NTID, NEWID ) );
		}
	}
	if( change ){
		rslts[QC::TRANT]=tbls[QC::TRANT];
		rslts[QC::SPLTT]=tbls[QC::SPLTT];
		rslts[QC::NAMET]=tbls[QC::NAMET];
	}


	// SPLITS
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::SPLTT]<<" ids"<<endl;
	change=false;
	rows=rslts[QC::SPLTT].rows();
	ids=QHaccTableIndex( &rslts[QC::SPLTT], QC::SID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::SPLTT][ids[i]][QC::SID]!=i+1 ){
			change=true;
			const TableCol OLDID( rslts[QC::SPLTT][ids[i]][QC::SID] );
			tbls[QC::SPLTT].updateWhere( TableSelect( QC::SID, OLDID ),
																	 TableUpdate( QC::SID, TableCol( i+1 ) ) );
		}
	}
	if( change ) rslts[QC::SPLTT]=tbls[QC::SPLTT];


	// NAMEDTRANS
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::NAMET]<<" ids"<<endl;
	change=false;
	rows=rslts[QC::NAMET].rows();
	ids=QHaccTableIndex( &rslts[QC::NAMET], QC::NID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::NAMET][ids[i]][QC::NID]!=i+1 ){
			change=true;
			const TableCol OLDID( rslts[QC::NAMET][ids[i]][QC::NID] );
			tbls[QC::NAMET].updateWhere( TableSelect( QC::NID, OLDID ),
																	 TableUpdate( QC::NID, TableCol( i+1 ) ) );
		}
	}
	if( change ) rslts[QC::NAMET]=tbls[QC::NAMET];

	// JOBS
	if( writer ) *str<<"conserving "<<QC::TABLENAMES[QC::JOBST]<<" ids"<<endl;
	change=false;
	rows=rslts[QC::JOBST].rows();
	ids=QHaccTableIndex( &rslts[QC::JOBST], QC::JID, CTUINT );
	for( uint i=0; i<rows; i++ ){
		if( rslts[QC::JOBST][ids[i]][QC::JID]!=i+1 ){
			change=true;
			const TableCol OLDID( rslts[QC::JOBST][ids[i]][QC::JID] );
			tbls[QC::JOBST].updateWhere( TableSelect( QC::JID, OLDID ),
																	 TableUpdate( QC::JID, TableCol( i+1 ) ) );
		}
	}
	if( change ) rslts[QC::JOBST]=tbls[QC::JOBST];

	/*
	for( uint i=0; i<QC::NUMTABLES; i++ ){
		cout<<"after conserving "<<QC::TABLENAMES[i]<<" ids: "<<endl;
		for( uint j=0; j<rslts[i].rows(); j++ )
			cout<<"  "<<rslts[i][j].toString()<<endl;
		cout<<"--"<<endl;
	}
	*/
}

void QHaccExt::replaceEngineData( QHaccResultSet * rslts ){
	// replace the data currently in the engine with the data in the
	// given resultset

	// this is pretty straightforward: delete all the old data and add the
	// new stuff

	engine->db->setAtom( BEGIN );

	// now delete the old data
	for( int i=QC::NUMTABLES-1; i>=0; i-- )
		engine->deleteWhere( ( Table )i, TableSelect() );
		
	for( int i=0; i<QC::NUMTABLES; i++ )
		engine->load( ( Table )i, &rslts[i] );

	engine->db->setAtom( COMMIT );
}