/* Distributed Clearinghouse Checksum database cleaner * * Copyright (c) 2006 by Rhyolite Software, LLC * * This agreement is not applicable to any entity which sells anti-spam * solutions to others or provides an anti-spam solution as part of a * security solution sold to other entities, or to a private network * which employs the DCC or uses data provided by operation of the DCC * but does not provide corresponding data to other users. * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * Parties not eligible to receive a license under this agreement can * obtain a commercial license to use DCC and permission to use * U.S. Patent 6,330,590 by contacting Commtouch at http://www.commtouch.com/ * or by email to nospam@commtouch.com. * * A commercial license would be for Distributed Checksum and Reputation * Clearinghouse software. That software includes additional features. This * free license for Distributed ChecksumClearinghouse Software does not in any * way grant permision to use Distributed Checksum and Reputation Clearinghouse * software * * THE SOFTWARE IS PROVIDED "AS IS" AND RHYOLITE SOFTWARE, LLC DISCLAIMS ALL * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL RHYOLITE SOFTWARE, LLC * BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES * OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS * SOFTWARE. * * Rhyolite Software DCC 1.3.50-1.199 $Revision$ */ #include "srvr_defs.h" #include "dcc_ck.h" #include #include static DCC_EMSG dcc_emsg; static DCC_WF dbclean_wf; static DCC_WHITE_TBL dbclean_white_tbl; static DCC_CLNT_CTXT *ctxt; static AOP_RESP aop_resp; static int flods_off; static int dccd_unlocked; /* dccd has been told to unlock */ static DCC_SRVR_NM srvr = DCC_SRVR_NM_DEF; static DCC_CLNT_ID srvr_clnt_id = DCC_ID_INVALID; static ID_TBL *srvr_clnt_tbl; static u_char info_flags = 0; #ifdef USE_DBCLEAN_F static u_char dbclean_db_mode = DB_OPEN_MMAP_WRITE; #else static u_char dbclean_db_mode = 0; #endif static const DB_HDR def_db_hdr = DB_HDR_DEF; static DB_HDR old_db_hdr; static u_char dccd_started_us; static u_char cleardb; /* 1=clear the database */ static u_char repair; /* 1=only repair the database */ static u_char standalone; /* 1=don't even try talk to dccd */ static u_char quiet; /* 1=don't announce results to stdout */ static u_char keep_white; /* 1=do not rebuild whitelist */ static int exit_value = -1; static const char *homedir; static u_char cur_db_created; static const char *cur_db_nm_str = DB_DCC_NAME; static DCC_PATH cur_db_nm; static DCC_PATH cur_hash_nm; static int old_db_fd = -1; static DB_HADDR old_db_hash_used; static DB_PARMS old_db_parms; static DB_PARMS new_db_parms; static DB_PTR old_db_pos, new_db_csize; static off_t new_db_fsize; static u_int new_db_pagesize, tgt_db_pagesize; static FLOD_MMAPS new_flod_mmaps; static u_char adj_delay_pos; static int lock_db_fd = -1; static DCC_PATH lock_db_nm; static u_char new_db_created; static DCC_PATH new_db_nm; static int new_db_fd = -1; static u_char new_hash_created; static DCC_PATH new_hash_nm; static DCC_PATH old_db_nm; static int expire_secs = -1; static int def_expire_secs = DB_EXPIRE_SECS_DEF; static int expire_spamsecs = -1; static int def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF; static int have_expire_parms = 0; static double def_exp_ratio = 0.0; static DB_EX_SECS new_ex_secs; static DB_EX_TS new_ex_ts; #define DEF_THOLD(keep,g,t) (DB_TEST_NOKEEP((keep).nokeep_cks,t) \ ? DCC_TGTS_INVALID \ : DCC_CK_IS_REP_CMN(g,t) \ ? DCC_TGTS_INVALID \ : (g) ? 1 \ : (t) == DCC_CK_SRVR_ID ? 1 \ : BULK_THRESHOLD) static DB_HADDR new_hash_len; static int expired_rcds, comp_rcds, obs_rcds, expired_cks; static int white_cks, kept_cks; static DCC_TS future_ts; #define RESTART_DELAY (60*5) static struct timeval clean_start; static struct timeval progress_rpt_last; static struct timeval progress_rpt_checked; #define REPORT_INTERVAL (5*60) #define REPORT_INTERVAL_FAST 10 static int progress_rpt_cnt, progress_rpt_base; static u_char progress_rpt_started; static int progress_rpt_percent; static u_char write_new_flush(u_char); static u_char write_new_rcd(const void *, int); static void write_new_hdr(u_char); static void unlink_whine(const char *, u_char); static void rename_bail(const char *, const char *); static u_char expire(DB_PTR); static u_char copy_db(void); static u_char catchup(void); static void parse_white(void); static void build_hash(void); static u_char persist_aop(DCC_AOPS, u_int32_t, u_char, int); static void dccd_new_db(const char *); static void dbclean_msg(const char *p, ...) PATTRIB(1,2); static void finish(void); static void deadman(int); static void sigterm(int); static void usage(u_char die) { const char str[] = { "usage: [-64dDfFNRPSVq] [-i id]" " [-a [server-addr][,server-port]] [-h homedir]\n" " [-G on] [-s hash-size] [-e seconds] [-E spamsecs]\n" " [-t type,allsecs,threshold,spamsecs]" " [-L ltype,facility.level]"}; static u_char complained; /* its important to try to run, so don't give up unless necessary */ if (die) { dcc_logbad(EX_USAGE, complained ? "giving up" : str); } else if (!complained) { dcc_error_msg("%s\ncontinuing", str); complained = 1; } } int NRATTRIB main(int argc, char **argv) { char hostname[MAXHOSTNAMELEN]; const char *rest; char *duparg, *cntstr, *allstr, *spamstr, *p; int allsecs, spamsecs; DCC_TGTS tgts; u_char print_version = 0; u_long l; DCC_CK_TYPES type; struct stat cur_db_sb; int i; gettimeofday(&db_time, 0); clean_start = db_time; dcc_timeval2ts(future_ts, &clean_start, 24*60*60); dcc_syslog_init(1, argv[0], 0); /* this must match DBCLEAN_GETOPTS in cron-dccd.in */ while ((i = getopt(argc, argv, "64dDfFNRPSVqi:a:h:G:s:e:E:t:L:")) != EOF) { switch (i) { case '6': #ifndef NO_IPV6 info_flags = DCC_INFO_FG_IPV6; #endif break; case '4': info_flags = 0; break; case 'd': if (db_debug++) ++dcc_clnt_debug; break; case 'D': dccd_started_us = 1; break; case 'f': dbclean_db_mode &= ~DB_OPEN_MMAP_WRITE; break; case 'F': dbclean_db_mode |= DB_OPEN_MMAP_WRITE; break; case 'N': /* make a new, clear database */ cleardb = 1; standalone = 1; break; case 'R': repair = 1; break; case 'P': if (have_expire_parms > 0) dcc_logbad(EX_USAGE, "do not use -P with -e, -E, or -t"); have_expire_parms = -1; break; case 'S': standalone = 1; break; case 'V': fprintf(stderr, DCC_VERSION"\n"); print_version = 1; break; case 'q': quiet = 1; break; case 'i': l = strtoul(optarg, &p, 0); if (*p != '\0' || l < DCC_SRVR_ID_MIN || l > DCC_SRVR_ID_MAX) dcc_logbad(EX_USAGE, "invalid DCC ID \"-i %s\"", optarg); srvr_clnt_id = l; break; case 'a': rest = dcc_parse_nm_port(dcc_emsg, optarg, srvr.port, hostname, sizeof(hostname), &srvr.port, 0, 0, 0, 0); if (!rest) { dcc_error_msg("%s", dcc_emsg); break; } rest += strspn(rest, DCC_WHITESPACE); if (*rest != '\0') { dcc_error_msg("unrecognized port number in" "\"-a %s\"", optarg); break; } if (hostname[0] == '\0') strcpy(srvr.hostname, DCC_SRVR_NM_DEF_HOST); else BUFCPY(srvr.hostname, hostname); break; case 'h': homedir = optarg; break; case 'G': if (strcasecmp(optarg, "on")) usage(0); dcc_syslog_init(1, argv[0], " grey"); if (have_expire_parms > 0) dcc_logbad(EX_USAGE, "do not use -G with -e, -E, or -t"); grey_on = 1; dcc_syslog_init(1, argv[0], " grey"); have_expire_parms = -1; cur_db_nm_str = DB_GREY_NAME; break; case 's': /* hash table size in entries */ new_hash_len = strtoul(optarg, &p, 0); if (*p != '\0' || new_hash_len < MIN_HASH_ENTRIES || new_hash_len > MAX_HASH_ENTRIES) dcc_logbad(EX_USAGE, "invalid database size \"%s\"", optarg); break; case 'e': /* expiration for non-bulk checksums */ if (grey_on) dcc_logbad(EX_USAGE, "-e cannot be used with -G"); if (have_expire_parms < 0) dcc_logbad(EX_USAGE, "-e cannot be used with -P"); have_expire_parms = 1; expire_secs = dcc_get_secs(optarg, 0, DB_EXPIRE_SECS_MIN, DB_EXPIRE_SECS_MAX, -1); if (expire_secs < 0) dcc_logbad(EX_USAGE, "invalid expiration seconds" " \"-e %s\"", optarg); break; case 'E': /* expiration for bulk checksums */ if (grey_on) dcc_logbad(EX_USAGE, "-E cannot be used with -G"); if (have_expire_parms < 0) dcc_logbad(EX_USAGE, "-E cannot be used with -P"); have_expire_parms = 1; expire_spamsecs = dcc_get_secs(optarg, 0, DB_EXPIRE_SECS_MIN, DB_EXPIRE_SECS_MAX, -1); if (expire_spamsecs < 0) dcc_logbad(EX_USAGE, "invalid long term spam" " expiration seconds" " \"-E %s\"", optarg); break; case 't': if (grey_on) dcc_logbad(EX_USAGE, "-t cannot be used with -G"); if (have_expire_parms < 0) dcc_logbad(EX_USAGE, "-t cannot be used with -P"); have_expire_parms = 1; duparg = dcc_strdup(optarg); allstr = strchr(duparg, ','); if (!allstr) dcc_logbad(EX_USAGE, " missing comma in \"-t %s\"", optarg); *allstr++ = '\0'; cntstr = strchr(allstr, ','); if (!cntstr) { spamstr = 0; } else { *cntstr++ = '\0'; spamstr = strchr(cntstr, ','); if (!spamstr) dcc_logbad(EX_USAGE, "missing comma after" " \"%s\" in \"-t %s\"", cntstr, optarg); *spamstr++ = '\0'; } type = dcc_str2type(duparg); if (type <= DCC_CK_INVALID || type > DCC_CK_G_TRIPLE_R_BULK) dcc_logbad(EX_USAGE, "unrecognized checksum type in" " \"-t %s\"", optarg); allsecs = dcc_get_secs(allstr, 0, DB_EXPIRE_SECS_MIN, DB_EXPIRE_SECS_MAX, -1); if (allsecs < 0) dcc_logbad(EX_USAGE, "invalid seconds \"%s\" in \"%s\"", allstr, optarg); if (!cntstr) { tgts = DCC_TGTS_TOO_MANY; if (allsecs == 0 || DCC_CK_LONG_TERM(type)) { spamsecs = allsecs; } else if (DCC_CK_LONG_TERM(type)) { spamsecs = max(DB_EXPIRE_SPAMSECS_DEF, allsecs); } else { spamsecs = allsecs; } } else { tgts = dcc_str2cnt(cntstr); if (tgts > DCC_TGTS_TOO_MANY || tgts <= 1) dcc_logbad(EX_USAGE, "unrecognized count \"%s\"" " in \"-t %s\"", cntstr, optarg); spamsecs = dcc_get_secs(spamstr, 0, DB_EXPIRE_SECS_MIN, DB_EXPIRE_SECS_MAX, -1); if (spamsecs < 0) dcc_logbad(EX_USAGE, "invalid seconds" " \"%s\" in \"%s\"", spamstr, optarg); if ((spamsecs < allsecs && spamsecs != 0) || (allsecs == 0 && spamsecs != 0)) dcc_logbad(EX_USAGE, "\"%s\"" " must not be smaller than" " \"%s\" in \"%s\"", spamstr, allstr, optarg); } dcc_free(duparg); new_ex_secs[type].all = allsecs; new_ex_secs[type].spam = spamsecs; new_ex_secs[type].clean_thold = tgts; break; case 'L': dcc_parse_log_opt(optarg); break; default: usage(0); } } argc -= optind; argv += optind; if (argc != 0) usage(1); if (srvr_clnt_id == DCC_ID_INVALID && !standalone) { if (print_version) exit(EX_OK); usage(1); } if (srvr.port == 0) srvr.port = DCC_GREY2PORT(grey_on); fnm2path_good(cur_db_nm, cur_db_nm_str, 0); dbclean_msg(DCC_VERSION" %s %s", repair ? "repairing" : "cleaning", cur_db_nm); dcc_clnt_unthread_init(); atexit(finish); signal(SIGALRM, deadman); signal(SIGHUP, sigterm); signal(SIGTERM, sigterm); signal(SIGINT, sigterm); #ifdef SIGXFSZ signal(SIGXFSZ, SIG_IGN); #endif /* move to the target directory */ srvr.clnt_id = srvr_clnt_id; if (!dcc_cdhome(dcc_emsg, homedir)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); if (!standalone) { i = load_ids(dcc_emsg, &srvr_clnt_tbl, srvr_clnt_id); if (i < 0) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); else if (!i) dcc_error_msg("%s", dcc_emsg); memcpy(srvr.passwd, srvr_clnt_tbl->cur_passwd, sizeof(srvr.passwd)); } fnm2path_good(lock_db_nm, cur_db_nm, DB_LOCK_SUFFIX); fnm2path_good(cur_hash_nm, cur_db_nm, DB_HASH_SUFFIX); fnm2path_good(old_db_nm, cur_db_nm, "-old"); fnm2path_good(new_db_nm, cur_db_nm, "-new"); fnm2path_good(new_hash_nm, new_db_nm, DB_HASH_SUFFIX); /* exclude other instances of this program */ lock_db_fd = dcc_lock_open(dcc_emsg, lock_db_nm, O_RDWR|O_CREAT, DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0); if (lock_db_fd < 0) { dcc_logbad(dcc_ex_code, "%s: dbclean already running?", dcc_emsg); } else { char pid[32]; i = 1+snprintf(pid, sizeof(pid), "%ld\n", (long)getpid()); if (i != write(lock_db_fd, pid, i)) dcc_logbad(EX_IOERR, "write(%s, pid): %s", lock_db_nm, ERROR_STR()); /* Let anyone write in it in csae we are running as root * and get interrupted by a crash or gdb. A stray, stale * private lock file cannot be locked */ chmod(lock_db_nm, 0666); } /* create & the lock new database file */ new_db_fd = dcc_lock_open(dcc_emsg, new_db_nm, O_RDWR|O_CREAT, DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0); if (new_db_fd == -1) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); if (0 > ftruncate(new_db_fd, 0)) dcc_logbad(EX_IOERR, "truncate(%s,0): %s", new_db_nm, ERROR_STR()); new_db_fsize = 0; new_db_created = 1; new_db_csize = DB_PTR_BASE; tgt_db_pagesize = 0; if (0 > stat(cur_db_nm, &cur_db_sb)) { if (errno != ENOENT) dcc_logbad(EX_IOERR, "stat(%s): %s", cur_db_nm, ERROR_STR()); /* empty a missing database */ cleardb = 1; } else if (cur_db_sb.st_size == 0) { /* empty an empty database */ cleardb = 1; } else if (grey_on) { tgt_db_pagesize = cur_db_sb.st_size/4; } if (tgt_db_pagesize < MIN_HASH_ENTRIES*sizeof(HASH_ENTRY) && grey_on) tgt_db_pagesize = MIN_HASH_ENTRIES*sizeof(HASH_ENTRY); new_db_pagesize = db_get_pagesize(0, tgt_db_pagesize); write_new_hdr(1); if (standalone) { u_char busy; /* open and lock the current database to ensure * the daemon is not running */ old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm, O_RDWR, DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, &busy); if (busy) dcc_logbad(EX_USAGE, "database %s in use: %s", cur_db_nm, dcc_emsg); if (cleardb && stat(cur_db_nm, &cur_db_sb) >= 0) { if (cur_db_sb.st_size != 0) dcc_logbad(EX_USAGE, "%s already exists", cur_db_nm); cur_db_created = 1; } /* create and lock the current database if it did not exist * to ensure that the server daemon is not running */ if (old_db_fd < 0) { old_db_fd = dcc_lock_open(dcc_emsg, cur_db_nm, O_RDWR|O_CREAT, DCC_LOCK_OPEN_NOWAIT, DCC_LOCK_ALL_FILE, 0); if (old_db_fd < 0) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); cur_db_created = 1; } } else { /* Tell the daemon to start turning off the flooding * so we can adjust its positions in the flood map file * Try very hard to talk to it because releasing the database * can make cause some UNIX flavors stall dccd. */ i = DCC_CLNT_FG_SLOW; if (grey_on) i |= DCC_CLNT_FG_GREY; ctxt = dcc_tmp_clnt_init(dcc_emsg, 0, &srvr, 0, i, info_flags); if (!ctxt) dcc_logbad(dcc_ex_code, "initial contact: %s", dcc_emsg); ++flods_off; if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_SHUTDOWN, 1, 30)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); } /* resolve white-listed host names before locking the database */ parse_white(); /* Tell the daemon to unlock the database between operations * and insist it stop flooding. */ if (!standalone) { /* give the daemon a chance to stop pumping the floods */ for (;;) { if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_CHECK, 1, 30)) dcc_logbad(EX_UNAVAILABLE, "%s", dcc_emsg); i = flod_running(aop_resp.resp.val.string); if (i < 0) dcc_logbad(EX_PROTOCOL, "%s: unrecognized \"%s\"", dcc_aop2str(0, 0, DCC_AOP_FLOD, DCC_AOP_FLOD_CHECK), aop_resp.resp.val.string); if (i == 0) break; if (time(0) > clean_start.tv_sec+45) { if (flods_off < 2) { ++flods_off; if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_HALT, 1, 30)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); continue; } if (time(0) > clean_start.tv_sec+60) dcc_logbad(EX_UNAVAILABLE, "failed to stop floods: %s", aop_resp.resp.val.string); } usleep(100*1000); } dccd_unlocked = 1; if (!persist_aop(DCC_AOP_DB_UNLOCK, 0, 1, 30)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); /* The daemon adds its own and removes our hold on flooding * when we tell it to unlock the database after every * operation. */ --flods_off; } if (cleardb) { dcc_trace_msg(DCC_VERSION" %s database %s", cur_db_created ? "creating" : "clearing", cur_db_nm); } else if (repair) { dcc_error_msg("explicit repair of %s", cur_db_nm); } else { if (!db_open(0, old_db_fd, cur_db_nm, 0, DB_OPEN_RDONLY | dbclean_db_mode | (standalone ? DB_OPEN_LOCK_NOWAIT : DB_OPEN_LOCK_WAIT))) { /* If the hash table is sick, check timestamps only * as much as no hash table allows. * Then rebuild the hash table. */ repair = 1; } else { if (db_debug) dcc_trace_msg("using %s with %s", db_window_size_str, new_db_nm); old_db_parms = db_parms; old_db_hash_used = db_hash_used; /* save a handle on the old database to get * reports that arrive while we expire it */ old_db_fd = dup(db_fd); if (old_db_fd < 0) dcc_logbad(EX_OSERR, "dup(%s): %s", cur_db_nm, ERROR_STR()); /* read old and create new database file */ if (!expire(db_csize)) { old_db_hash_used = 0; repair = 1; } } if (repair) dcc_error_msg("repairing %s", cur_db_nm); } /* if we are repairing the hash table (including now repairing * after encountering problems while expiring), * copy the current file with minimal expiring */ if (repair && !cleardb) { if (!copy_db()) exit(EX_UNAVAILABLE); } build_hash(); /* we have the new database locked * * preserve the current data file as "*-old", delete the current hash * file, and install both new files */ rename_bail(cur_db_nm, old_db_nm); rename_bail(new_hash_nm, cur_hash_nm); strcpy(new_hash_nm, cur_hash_nm); new_hash_created = 0; if (db_hash_fd >= 0) strcpy(db_hash_nm, cur_hash_nm); rename_bail(new_db_nm, cur_db_nm); strcpy(new_db_nm, cur_db_nm); new_db_created = 0; if (db_fd > 0) strcpy(db_nm, cur_db_nm); cur_db_created = 0; if (cleardb) { flod_mmap_path_set(); unlink_whine(flod_mmap_path, 1); if (!db_close(1)) exit(EX_UNAVAILABLE); exit(exit_value = EX_OK); } /* if the daemon was not running, we're finished */ if (standalone) { /* install the flood positions if things are ok */ if (flod_mmaps) { memcpy(flod_mmaps, &new_flod_mmaps, sizeof(new_flod_mmaps)); flod_unmap(0, 0); } if (!db_close(1)) exit(EX_UNAVAILABLE); exit(exit_value = EX_OK); } /* try to finish as much disk I/O as we can to minimize stalling * by dccd when we close the file and hand it over */ make_clean(2); /* Copy any records from the old file to the new file that were * added to the old file while we were creating the new file. */ if (!catchup()) { write_new_hdr(1); exit(EX_UNAVAILABLE); } /* tell the daemon to switch to the new database and stop unlocking * the database on every operation. This will leave the daemon * stuck waiting for us to unlock the new database. * Tell the daemon to forget its changes to the old database. */ dccd_new_db("copy late arrivals"); /* install the flood positions if things are ok */ if (flod_mmaps) { memcpy(flod_mmaps, &new_flod_mmaps, sizeof(new_flod_mmaps)); flod_unmap(0, 0); } /* Copy any records from the old file to the new file in the * race to tell the daemon to switch to the new file. * The new file is still locked from build_hash(). * The daemon should be stuck waiting to open it in the * DCC_AOP_DB_NEW request via the preceding dccd_new_db(). * * Since the daemon has switched and probably cannot go back, * ignore any errors */ catchup(); if (!db_close(1)) exit(EX_UNAVAILABLE); /* finish() will be called via exit() to tell the daemon to resume * flooding if necessary. However, in the normal case, we removed * all counts against flooding before calling dccd_new_db() */ exit(exit_value = EX_OK); } /* adjust output flood positions */ static DB_PTR adj_mmap(void) { FLOD_MMAP *mp; DB_PTR delta, new_pos; delta = new_db_csize - old_db_pos; new_pos = 0; for (mp = new_flod_mmaps.mmaps; mp <= LAST(new_flod_mmaps.mmaps); ++mp) { /* do nothing to marks we have already adjusted */ if (mp->oflod_index >= 0) continue; if (mp->confirm_pos > old_db_pos) { /* note the next mark that will need adjusting * but do not adjust it yet */ if (new_pos == 0 || new_pos > mp->confirm_pos) new_pos = mp->confirm_pos; } else { /* adjust marks not past the current position */ mp->confirm_pos += delta; mp->oflod_index = 0; } } if (adj_delay_pos) { if (new_flod_mmaps.delay_pos > old_db_pos) { if (new_pos == 0 || new_pos > new_flod_mmaps.delay_pos) new_pos = new_flod_mmaps.delay_pos; } else { new_flod_mmaps.delay_pos += delta; adj_delay_pos = 0; } } return new_pos; } static void NRATTRIB deadman(int s UATTRIB) { dcc_logbad(EX_IOERR, "db_lock() timed out; dccd restarted?"); } /* find a checksum * Leave db_sts.rcd2 pointing at the leading record. */ static u_char get_ck(DB_RCD_CK **ckp, DCC_CK_TYPES type, const DCC_SUM sum) { DB_FOUND db_result; /* we must lock the file to keep the daemon from changing the * internal hash table links */ if (!DB_IS_LOCKED()) { alarm(60*60); /* don't stall for more than an hour */ if (0 > db_lock()) return 1; /* cheat and don't turn off the alarm, since we ought * to be back here long before an hour has passed */ } dcc_emsg[0] = '\0'; db_result = db_lookup(dcc_emsg, type, sum, 0, MAX_HASH_ENTRIES, &db_sts.hash, &db_sts.rcd2, ckp); switch (db_result) { case DB_FOUND_LATER: case DB_FOUND_SYSERR: dcc_error_msg("hash lookup for %s from "L_HPAT" = %d: %s", DB_TYPE2STR(type), old_db_pos, db_result, dcc_emsg); break; case DB_FOUND_IT: return 1; case DB_FOUND_EMPTY: case DB_FOUND_CHAIN: case DB_FOUND_INTRUDER: *ckp = 0; return 1; } return 0; } /* Get the leading report for a checksum * Leave db_sts.rcd2 pointing at the leading record. */ static int /* -1=broken database 0=expire 1=keep */ get_lead(DCC_CK_TYPES type, const DCC_SUM sum, DCC_TGTS rcd_tgts) { DB_RCD_CK *ck; DCC_TGTS total_tgts; if (!get_ck(&ck, type, sum)) return -1; if (!ck) { dcc_error_msg("no leader for %s %s at "L_HPAT, DB_TYPE2STR(type), dcc_ck2str_err(type, sum), old_db_pos); return -1; } total_tgts = DB_TGTS_CK(ck); if (DCC_CK_IS_REP_CMN(grey_on, type)) { /* do not keep reputations on systems without reputation code */ return 0; } /* do not keep checksums that later become spam */ if (total_tgts == DCC_TGTS_TOO_MANY && rcd_tgts != DCC_TGTS_TOO_MANY) return 0; return (total_tgts >= new_ex_secs[type].clean_thold); } /* send only to system log if being quiet */ static void PATTRIB(1,2) dbclean_msg(const char *p, ...) { va_list args; va_start(args, p); if (quiet) { vsyslog(dcc_trace_priority, p, args); } else { dcc_vtrace_msg(p, args); } va_end(args); } static void report_progress_init(void) { gettimeofday(&db_time, 0); progress_rpt_checked = db_time; progress_rpt_last = db_time; progress_rpt_cnt = 0; progress_rpt_base = 0; progress_rpt_started = 0; } static void report_progress(u_char force_time, const char *s1, DB_PTR done, const char *s2, DB_PTR total) { time_t us, secs, interval; double percent; /* don't start progress reporting at the end */ if (!total) percent = 100.0; else percent = (done*100.0)/total; if (!progress_rpt_started && percent > 30.0) { progress_rpt_cnt = 1000000; return; } gettimeofday(&db_time, 0); us = tv_diff2us(&db_time, &progress_rpt_checked); progress_rpt_checked = db_time; /* try to check twice per second */ if (!progress_rpt_base) { progress_rpt_base = 1000; } else { /* check every half second */ progress_rpt_base = (progress_rpt_base * 0.5 * DCC_US) / us; if (progress_rpt_base < 1000) progress_rpt_base = 1000; } progress_rpt_cnt = progress_rpt_base; us = tv_diff2us(&db_time, &progress_rpt_last); interval = ((db_debug > 1) ? REPORT_INTERVAL_FAST : REPORT_INTERVAL); if (us >= interval * DCC_US || (force_time && progress_rpt_percent != 100)) { progress_rpt_started = 1; progress_rpt_percent = percent; secs = db_time.tv_sec - clean_start.tv_sec; secs -= secs % interval; progress_rpt_last.tv_sec = clean_start.tv_sec + secs; if (db_debug > 1) dbclean_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%" "\tdb_mmaps=%d hash=%d", s1, done, total, s2, progress_rpt_percent, db_stats.db_mmaps, db_stats.hash_mmaps); else dbclean_msg("%s "L_DPAT" of "L_DPAT" %s or %d%%", s1, done, total, s2, progress_rpt_percent); } } /* mark obsolete old, less fuzzy checksums in the new record */ static void fuzzy_obs(DB_RCD *new, const DB_RCD_CK *end_ck) { DB_RCD_CK *rcd_ck; DCC_CK_TYPES type; for (rcd_ck = new->cks; rcd_ck < end_ck; ++rcd_ck) { type = DB_CK_TYPE(rcd_ck); if (!DCC_TS_OLDER_TS(new->ts, new_ex_ts[type].all)) continue; rcd_ck->type_fgs |= DB_CK_FG_OBS; ++obs_rcds; } } static void adj_def_expire(void) { double new_dbsize, new_dbsize1, rate, db_ratio; int spam_secs, secs; struct timeval tv; /* do this only once */ if (def_exp_ratio != 0.0) return; /* can do nothing the first time dbclean is run */ spam_secs = db_parms.ex_secs[DCC_CK_FUZ2].spam; if (spam_secs == 0) return; /* Compute the ratio of size of the database 24 hours from now * to the size of the window. Assume: * - We will receive about the same number of reports in the next * 24 hours as the last 24. This is a good assumption for * weekdays, but as much as 30% wrong about weekends. * - Dbclean will be run once per day at the current time. * - The size of the database is a linear function of expiration * duration. This is tenuous when the spam expiration duration * is less than 1 day. * Use the maximum of two guesses for tomorrow's database size. * One guess is the current size, base on assuming that * we will use roughly the same expiration durations and * so the database will grow to about size it now has. * The other guess uses the previous database size and the * avarage data rate. It compensates for short term changes * in the rate and for running dbclean more than once per day. */ new_dbsize = db_parms.db_csize; rate = db_add_rate(&db_parms, 0); if (rate > 0.0) { new_dbsize1 = db_parms.old_db_csize; new_dbsize1 += rate * 24*3600; if (new_dbsize < new_dbsize1) new_dbsize = new_dbsize1; } /* pad 10% in case more checksums are seen tomorrow */ new_dbsize *= 1.1; /* Assume there will be 45% as many bytes used in the hash table * as in the database, but the hash table load factor should be * kept below 0.9. That implies a target hash table size * that is 50% of the target database size. */ new_dbsize *= 1.5; if (new_dbsize < db_max_rss/128.0) db_ratio = 128.0; else db_ratio = db_max_rss / new_dbsize; if (db_ratio < 1.0) { def_exp_ratio = (spam_secs * db_ratio) / DB_EXPIRE_SPAMSECS_DEF; /* change the two durations together and so with same errors */ def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF * def_exp_ratio; def_expire_secs = DB_EXPIRE_SECS_DEF * def_exp_ratio; def_expire_secs -= def_expire_secs % (60*60); if (def_expire_secs < DB_EXPIRE_SECS_DEF_MIN) def_expire_secs = DB_EXPIRE_SECS_DEF_MIN; def_expire_spamsecs -= def_expire_spamsecs % (24*60*60); if (def_expire_spamsecs < DB_EXPIRE_SPAMSECS_DEF_MIN) def_expire_spamsecs = DB_EXPIRE_SPAMSECS_DEF_MIN; #if DCC_MIN_DB_MBYTE == 0 && !defined(GOT_PHYSMEM) if (def_expire_secs == DB_EXPIRE_SECS_DEF_MIN || def_expire_spamsecs == DB_EXPIRE_SPAMSECS_DEF_MIN) dbclean_msg("no way to determine physical RAM;" " rebuild with ./configure with-db-memory"); #endif return; } def_exp_ratio = 1.0; /* if the defaults do not need to be reduced now but they * were reduced before, then relax them gently */ if (spam_secs < DB_EXPIRE_SPAMSECS_DEF) { dcc_ts2timeval(&tv, db_parms.ex_spam[DCC_CK_FUZ2]); secs = clean_start.tv_sec - tv.tv_sec; if (secs > 0 && secs < DB_EXPIRE_SPAMSECS_DEF) def_expire_spamsecs = secs; } } /* copy the existing flag file, discard junk and old entries */ static u_char /* 1=done 0=database broken */ expire(DB_PTR old_db_csize) { #define EXPIRE_BAIL() {alarm(0); flod_unmap(0, 0); db_close(0); return 0;} DCC_TS ts; u_char emptied, reduced_defaults; u_char old_ok[DCC_DIM_CKS]; DB_RCD rcd, new; const DB_RCD_CK *rcd_ck, *rcd_ck2; DB_RCD_CK *new_ck; DCC_TGTS tgts_raw, ck_tgts; u_char needed, obs_lvl, timely; int old_num_cks, new_num_cks, nokeep_cks; DB_PTR min_confirm_pos, next_adj_pos; FLOD_MMAP *mp; DCC_CK_TYPES prev_type, type, type2; int rcd_len; struct stat sb; u_char need_unlock; int i; reduced_defaults = 0; if (expire_secs < 0) { adj_def_expire(); if (def_expire_secs > expire_spamsecs && expire_spamsecs > 0) { expire_secs = expire_spamsecs; } else { if (def_expire_secs != DB_EXPIRE_SECS_DEF && def_exp_ratio != 1.0) reduced_defaults = 1; expire_secs = def_expire_secs; } } if (expire_spamsecs < 0) { adj_def_expire(); if (def_expire_spamsecs < expire_secs) { expire_spamsecs = expire_secs; } else { if (def_expire_spamsecs != DB_EXPIRE_SPAMSECS_DEF && def_exp_ratio != 1.0) reduced_defaults = 1; expire_spamsecs = def_expire_spamsecs; } } if (expire_spamsecs > 0 && expire_spamsecs < expire_secs) dcc_logbad(EX_USAGE, "long term spam expiration -E" " must be longer than -e"); expired_rcds = 0; expired_cks = 0; kept_cks = white_cks; need_unlock = 0; report_progress_init(); /* Compute thresholds for records we keep. * Use the values from the previous use of dbclean as defaults * unless they are bogus */ memset(old_ok, 0, sizeof(old_ok)); dcc_secs2ts(ts, clean_start.tv_sec); for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { char buf[30]; DB_EX_SEC *th = &db_parms.ex_secs[type]; if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type)) continue; if ((th->clean_thold <= 0 || th->clean_thold > DCC_TGTS_TOO_MANY) && !DCC_CK_IS_REP_CMN(grey_on, type)) { if (db_debug) dbclean_msg("bad old clean thold %s for %s", dcc_tgts2str(buf, sizeof(buf), th->clean_thold, grey_on), DB_TYPE2STR(type)); continue; } if (th->spam <= 0 || th->spam > DB_EXPIRE_SECS_MAX) { if (db_debug) dbclean_msg("bad old spam threshold %s for %s", dcc_tgts2str(buf, sizeof(buf), th->spam, grey_on), DB_TYPE2STR(type)); continue; } if (th->all <= 0 || th->all > DB_EXPIRE_SECS_MAX) { if (db_debug) dbclean_msg("bad old all threshold %s for %s", dcc_tgts2str(buf, sizeof(buf), th->all, grey_on), DB_TYPE2STR(type)); continue; } if (DCC_TS_NEWER_TS(db_parms.ex_spam[type], ts)) { if (db_debug) dbclean_msg("bad old spam timestamp %s for %s", ts2str_err(db_parms.ex_spam[type]), DB_TYPE2STR(type)); continue; } old_ok[type] = 1; /* the old values are ok */ } for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { DB_EX_SEC *new_th = &new_ex_secs[type]; /* always keep server-ID declarations one week */ if (type == DCC_CK_SRVR_ID) { new_th->all = 7*24*60*60; new_th->spam = 7*24*60*60; new_th->clean_thold = 1; dcc_secs2ts(new_ex_ts[DCC_CK_SRVR_ID].spam, clean_start.tv_sec - 7*24*60*60); memcpy(new_ex_ts[DCC_CK_SRVR_ID].all, new_ex_ts[DCC_CK_SRVR_ID].spam, sizeof(new_ex_ts[DCC_CK_SRVR_ID].all)); continue; } if (new_th->clean_thold == 0) { /* we have no explicit settings for this checksum type. * Use the dccd bulk thresholds by default */ if (grey_on) { new_th->clean_thold = DCC_TGTS_TOO_MANY; } else { new_th->clean_thold = DEF_THOLD(db_parms, 0, type); } if (have_expire_parms < 0 && old_ok[type]) { /* if we have no global expiriation parameters * and the old durations are valid, use them */ new_th->all = db_parms.ex_secs[type].all; new_th->spam = db_parms.ex_secs[type].spam; } else if (grey_on) { if (DCC_CK_IS_GREY_TRIPLE(1, type)) { new_th->all = DEF_GREY_WINDOW; new_th->spam = DEF_GREY_WHITE; } else if (DCC_CK_IS_GREY_MSG(1, type) || type == DCC_CK_BODY) { new_th->all = DEF_GREY_WINDOW; new_th->spam = DEF_GREY_WINDOW; } else { new_th->all = 1; new_th->spam = 1; } } else { new_th->all = expire_secs; new_th->spam = (DCC_CK_LONG_TERM(type) ? expire_spamsecs : expire_secs); if (reduced_defaults) { dbclean_msg("adjust default by %4.2f" " to -e %dhours -E %ddays", def_exp_ratio, expire_secs/(60*60), expire_spamsecs/(24*60*60)); reduced_defaults = 0; } } } /* compute oldest timestamp for this type of checksum, * without going crazy with "-Enever" */ dcc_secs2ts(new_ex_ts[type].spam, clean_start.tv_sec - min(clean_start.tv_sec, new_th->spam)); dcc_secs2ts(new_ex_ts[type].all, clean_start.tv_sec - min(clean_start.tv_sec, new_th->all)); } /* put the thresholds into the new file */ write_new_hdr(1); /* if we are running as root, * don't change the owner of the database */ if (getuid() == 0) { if (0 > fstat(old_db_fd, &sb)) dcc_logbad(EX_IOERR, "fstat(%s): %s", old_db_nm, ERROR_STR()); if (0 > fchown(new_db_fd, sb.st_uid, sb.st_gid)) dcc_logbad(EX_IOERR, "fchown(%s,%d,%d): %s", new_db_nm, (int)sb.st_uid, (int)sb.st_gid, ERROR_STR()); } if (DB_PTR_BASE != lseek(old_db_fd, DB_PTR_BASE, SEEK_SET)) dcc_logbad(EX_IOERR, "lseek(%s,%d): %s", cur_db_nm, DB_PTR_BASE, ERROR_STR()); read_rcd_invalidate(0); flod_mmap(0, db_parms.sn, 0, 1, 1); if (flod_mmaps) memcpy(&new_flod_mmaps, flod_mmaps, sizeof(new_flod_mmaps)); min_confirm_pos = new_flod_mmaps.delay_pos; next_adj_pos = DB_PTR_BASE; for (mp = new_flod_mmaps.mmaps; mp <= LAST(new_flod_mmaps.mmaps); ++mp) { if (mp->rem_hostname[0] == '\0') { mp->oflod_index = 0; } else { mp->oflod_index = -1; if (min_confirm_pos > mp->confirm_pos) min_confirm_pos = mp->confirm_pos; } } adj_delay_pos = (new_flod_mmaps.delay_pos != 0) ? 1 : 0; emptied = cleardb; dcc_timeval2ts(new_flod_mmaps.sn, &clean_start, 0); /* copy the old file to the new, discarding old data as we go */ for (old_db_pos = DB_PTR_BASE; old_db_pos < old_db_csize; old_db_pos += rcd_len) { if (--progress_rpt_cnt <= 0) { need_unlock = 1; report_progress(0, " processed", old_db_pos/1000000, "MBytes", old_db_csize/1000000); } if (old_db_pos == next_adj_pos) next_adj_pos = adj_mmap(); if (!repair) { if (!db_map_rcd(0, &db_sts.rcd, old_db_pos, &rcd_len)) EXPIRE_BAIL(); memcpy(&rcd, db_sts.rcd.d.r, rcd_len); } else { rcd_len = read_rcd(0, &rcd, old_db_fd, old_db_pos, cur_db_nm); if (rcd_len <= 0) { if (rcd_len == 0) dcc_error_msg("unexpected EOF in %s at " L_HPAT" instead of " L_HPAT, cur_db_nm, old_db_pos, old_db_csize); /* give up and ask our neighbors to rewind */ emptied = 1; old_db_pos = old_db_csize; break; } } /* skip end-of-page padding */ if (rcd_len == sizeof(rcd)-sizeof(rcd.cks)) continue; if (DB_RCD_ID(&rcd) == DCC_ID_WHITE) { /* skip whitelist entries if whitelist source is ok */ if (!keep_white) continue; /* refresh whitelist entries if source is bad */ dcc_timeval2ts(rcd.ts, &clean_start, 0); } old_num_cks = DB_NUM_CKS(&rcd); /* expire or throw away deleted reports */ tgts_raw = DB_TGTS_RCD_RAW(&rcd); if (tgts_raw == 0) { ++expired_rcds; expired_cks += old_num_cks; continue; } if (tgts_raw > DCC_TGTS_MAX_DB) { dcc_error_msg("discarding report at "L_HPAT " with bogus target count %#x", old_db_pos, tgts_raw); ++expired_rcds; expired_cks += old_num_cks; continue; } if (DCC_TS_NEWER_TS(rcd.ts, future_ts)) { dcc_error_msg("discarding report at "L_HPAT " from the future %s", old_db_pos, ts2str_err(rcd.ts)); ++expired_rcds; expired_cks += old_num_cks; continue; } needed = 0; obs_lvl = 0; timely = 1; nokeep_cks = 0; memcpy(&new, &rcd, sizeof(new)-sizeof(new.cks)); new.fgs_num_cks &= (DB_RCD_FG_TRIM | DB_RCD_FG_SUMRY | DB_RCD_FG_DELAY); new_ck = new.cks; for (prev_type = DCC_CK_INVALID, rcd_ck = rcd.cks; rcd_ck < &rcd.cks[old_num_cks]; prev_type = type, ++rcd_ck) { type = DB_CK_TYPE(rcd_ck); if (!DCC_CK_OK_DB(grey_on, type)) { static int whines = 0; if (whines < 20) dcc_error_msg("discarding %s" " checksum at "L_HPAT"%s", DB_TYPE2STR(type), old_db_pos, ++whines >= 20 ? "; stop complaining" : ""); ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } if (type <= prev_type && prev_type != DCC_CK_FLOD_PATH) { dcc_error_msg("discarding out of order %s" " checksum at "L_HPAT, DB_TYPE2STR(type), old_db_pos); ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } /* Silently discard pure junk from other servers, * provided it is junk by default */ if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type) && DB_GLOBAL_NOKEEP(grey_on, type) && type != DCC_CK_FLOD_PATH && type != DCC_CK_SRVR_ID && DB_RCD_ID(&rcd) != DCC_ID_WHITE) { ++expired_cks; continue; } /* Keep paths except on old records or records that * have been trimmed or compressed. * Never remove paths from server-ID declarations. */ if (type == DCC_CK_FLOD_PATH) { if (DB_RCD_TRIMMED(&new) || DB_RCD_ID(&new) == DCC_ID_COMP) continue; /* forget line number on old whitelist entry */ if (DB_RCD_ID(&rcd) == DCC_ID_WHITE) continue; rcd_ck2 = rcd_ck+1; for (;;) { type2 = DB_CK_TYPE(rcd_ck2); if (type2 == DCC_CK_SRVR_ID || !DCC_TS_OLDER_TS(rcd.ts, new_ex_ts[type2].all)) { /* keep this path since this report * is a server-ID declaration * or not old */ *new_ck = *rcd_ck; ++new_ck; ++new.fgs_num_cks; ++nokeep_cks; break; } if (++rcd_ck2>=&rcd.cks[old_num_cks]) { /* we are discarding this path */ new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; break; } } continue; } /* throw this checksum away if it is extremely old */ if (DCC_TS_OLDER_TS(rcd.ts, new_ex_ts[type].spam)) { ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } if (!DCC_TS_OLDER_TS(rcd.ts, new_ex_ts[type].all)) { /* This report is recent. * However, obsolete or junk checksums * don't make the report needed */ if (DB_TEST_NOKEEP(db_parms.nokeep_cks, type) && DB_RCD_ID(&rcd) != DCC_ID_WHITE) { ++nokeep_cks; } else if (DB_CK_OBS(rcd_ck)) { /* This checksum is obsolete. * If it has the highest level of * fuzziness, then it controls whether * the whole report is needed,. */ if (obs_lvl < db_ck_fuzziness[type]) { obs_lvl = db_ck_fuzziness[type]; needed = 0; } } else { /* This checksum is not obsolete. * If it is at least as fuzzy as any * other checksum, then it can say * the report is needed */ if (obs_lvl <= db_ck_fuzziness[type]) { obs_lvl = db_ck_fuzziness[type]; needed = 1; } } } else { /* This checksum is old. * Throw away delete requests * and other servers' useless checksums */ if (tgts_raw == DCC_TGTS_DEL || DB_TEST_NOKEEP(db_parms.nokeep_cks, type)) { ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } /* Throw away old obsolete checksums * and entire reports if the fuzziest * checksum is obsolete */ if (DB_CK_OBS(rcd_ck)) { if (obs_lvl < db_ck_fuzziness[type]) { obs_lvl = db_ck_fuzziness[type]; needed = 0; } ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } /* old summaries are unneeded, because * they have already been flooded. * They do not contribute to local counts */ if (DB_RCD_SUMRY(&rcd)) continue; /* The checksum is old enough to compress, * but not old enough to delete, so mark * the record as eligible for splitting. */ timely = 0; /* Discard this checksum if its ultimate total * is low or if it reaches spam after this * report. * To determine the ultimate total, we must * have a hash table to find the newest record, * which contains the final total*/ ck_tgts = DB_TGTS_CK(rcd_ck); if (!repair && ck_tgts < new_ex_secs[type ].clean_thold) { i = get_lead(type, rcd_ck->sum, ck_tgts); if (i < 0) EXPIRE_BAIL(); if (!i) { ++expired_cks; new.fgs_num_cks |= DB_RCD_FG_TRIM; new.fgs_num_cks &= ~DB_RCD_FG_DELAY; continue; } } if (obs_lvl <= db_ck_fuzziness[type]) { /* Since we did not delete this * checksum, we need the record if this * checksum is fuzzy enough to control * our need. */ needed = 1; /* If this is the fuzziest checksum we * have seen, then preceding and so * less fuzzy checksums are obsolete, * if they are old. * Assume that checksums are ordered * in the record by fuzziness. */ if (obs_lvl < db_ck_fuzziness[type]) { obs_lvl = db_ck_fuzziness[type]; if (obs_lvl != DCC_CK_FUZ_LVL_REP && !grey_on) fuzzy_obs(&new, new_ck); } } } /* Keep this checksum if we decide the whole report * is needed. */ *new_ck = *rcd_ck; /* we will want to sum this checksum */ new_ck->type_fgs &= ~DB_CK_FG_DUP; ++new_ck; ++new.fgs_num_cks; } /* let the daemon do some work occassionally */ if (!repair && !standalone && DB_IS_LOCKED() && need_unlock) { need_unlock = 0; if (!db_unlock()) EXPIRE_BAIL(); } /* if none of its checksums are needed, * then discard the entire record */ if (!needed) { expired_cks += DB_NUM_CKS(&new); ++expired_rcds; continue; } new_num_cks = DB_NUM_CKS(&new); kept_cks += new_num_cks - nokeep_cks; /* Put the new record into the new file. * * If all of the record is recent, if it contains 1 checksum, * or if all of its totals are the same, then simply add it. * * Otherwise, divide it into records of identical counts * to allow compression or combining with other records. */ if (new_num_cks > 1 && (!timely || DB_RCD_ID(&new) == DCC_ID_COMP || DB_RCD_TRIMMED(&new))) { for (;;) { /* skip the checksums that have the same total * as the first checksum to leave them with the * original new report */ new_ck = new.cks; ck_tgts = DB_TGTS_CK(new_ck); for (i = 1; i < new_num_cks; ++i) { ++new_ck; if (DB_TGTS_CK(new_ck) != ck_tgts) break; } if (new_num_cks <= i) break; new_num_cks -= i; /* write the checksums with the common total */ new.srvr_id_auth = DCC_ID_COMP; new.fgs_num_cks = i; if (!write_new_rcd(&new, sizeof(new) - sizeof(new.cks) + i*sizeof(new.cks[0]))) EXPIRE_BAIL(); /* handle the remaining checksums */ new.fgs_num_cks = new_num_cks; memmove(&new.cks[0], &new.cks[i], new_num_cks*sizeof(new.cks[0])); } } /* write the rest (or all) of the new record */ if (!write_new_rcd(&new, sizeof(new) - sizeof(new.cks) + new_num_cks*sizeof(new.cks[0]))) EXPIRE_BAIL(); } write_new_flush(1); alarm(0); /* do final adjustment of the flooding positions */ adj_mmap(); /* force them to be right if the system crashed with the * flod.map file on the disk more up to date and so after the * database file on the disk */ for (mp = new_flod_mmaps.mmaps; mp <= LAST(new_flod_mmaps.mmaps); ++mp) { if (mp->rem_hostname[0] != '\0' && mp->confirm_pos > new_db_csize) mp->confirm_pos = new_db_csize; } i = db_close(1); write_new_hdr(emptied); report_progress(1, " processed", old_db_pos/1000000, "MBytes", old_db_csize/1000000); if (grey_on) dbclean_msg("expired %d records and %d checksums in %s", expired_rcds, expired_cks, cur_db_nm); else dbclean_msg("expired %d records and %d checksums," " obsoleted %d checksums in %s", expired_rcds, expired_cks, obs_rcds, cur_db_nm); return i; } /* copy the database copy while doing minimal expiring */ static u_char copy_db(void) { /* do not lock the old database because the daemon must continue * to answer requests */ if (old_db_fd < 0) { old_db_fd = open(cur_db_nm, O_RDONLY, 0); if (old_db_fd == -1) dcc_logbad(EX_IOERR, "open(%s): %s", cur_db_nm, ERROR_STR()); } if (!read_db_hdr(dcc_emsg, &old_db_hdr, old_db_fd, cur_db_nm)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); if (memcmp(old_db_hdr.p.version, def_db_hdr.p.version, sizeof(old_db_hdr.p.version))) dcc_logbad(EX_IOERR, "%s has the wrong magic \"%.*s\"", cur_db_nm, ISZ(old_db_hdr.p.version), old_db_hdr.p.version); old_db_parms = old_db_hdr.p; memcpy(db_parms.sn, old_db_parms.sn, sizeof(db_parms.sn)); memcpy(db_parms.ex_spam, old_db_parms.ex_spam, sizeof(db_parms.ex_spam)); memcpy(&db_parms.ex_secs, &old_db_parms.ex_secs, sizeof(db_parms.ex_secs)); db_parms.nokeep_cks = old_db_parms.nokeep_cks; db_parms.flags = old_db_parms.flags; return expire(old_db_parms.db_csize); } /* Copy any records from the old file to the new file that were * added to the old file while we were creating the new file. */ static u_char catchup(void) { DB_RCD rcd; int rcd_len; u_char result; int count, old_count; /* Because the old file should still be unlocked, the daemon * will have been keeping its magic number block accurate */ result = 1; count = 0; do { old_count = count; if (!read_db_hdr(dcc_emsg, &old_db_hdr, old_db_fd, old_db_nm)) { dcc_error_msg("%s", dcc_emsg); result = 0; break; } if (old_db_hdr.p.db_csize < old_db_pos) { dcc_error_msg("%s mysteriously truncated", old_db_nm); result = 0; break; } if ((off_t)old_db_pos != lseek(old_db_fd, old_db_pos, SEEK_SET)) { dcc_error_msg("lseek(%s, "L_HPAT"): %s", old_db_nm, old_db_pos, ERROR_STR()); result = 0; break; } read_rcd_invalidate(0); while (old_db_pos < old_db_hdr.p.db_csize) { rcd_len = read_rcd(0, &rcd, old_db_fd, old_db_pos, old_db_nm); if (rcd_len <= 0) { if (rcd_len == 0) dcc_error_msg("premature EOF in %s" " at "L_HPAT " instead of "L_HPAT, old_db_nm, old_db_pos, old_db_hdr.p.db_csize); result = 0; break; } /* If something bad happens, we may not be able to * go back to the old file. Carry on to get as much * data as we can although we know the dccd daemon * may croak when we release it */ if (!db_add_rcd(0, &rcd)) { result = 0; break; } old_db_pos += rcd_len; ++count; } } while (result && old_count != count); if (count > 0 && db_debug >= 1) dbclean_msg("added %d late reports", count); return result; } /* try to compress old report pointed to by db_sts.rcd with a predecessor */ static void compress_old(void) { DB_PTR prev, prev1; DB_RCD_CK *new_ck, *prev_ck; int new_ck_num, prev_ck_num; DCC_TGTS new_tgts, prev_tgts; DCC_CK_TYPES new_type, prev_type; DCC_SRVR_ID new_srvr, prev_srvr; /* Before spending the time to map a preceding checksum, * find at least one checksum worth keeping and that might * be combined or compressed with its predecessor. */ if (DB_RCD_TRIMMED(db_sts.rcd.d.r)) new_srvr = DCC_ID_COMP; else new_srvr = DB_RCD_ID(db_sts.rcd.d.r); prev = DB_PTR_NULL; prev_type = DCC_CK_INVALID; for (new_ck_num = DB_NUM_CKS(db_sts.rcd.d.r), new_ck = db_sts.rcd.d.r->cks; new_ck_num != 0; --new_ck_num, ++new_ck) { new_type = DB_CK_TYPE(new_ck); if (DB_TEST_NOKEEP(db_parms.nokeep_cks, new_type)) continue; /* all of the checksums in this record must be old */ if (new_srvr != DCC_ID_COMP && !DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts, new_ex_ts[new_type].all)) return; /* you can compress reports only if you have >=2 */ prev1 = DB_PTR_EX(new_ck->prev); if (prev1 != DB_PTR_NULL) { prev = prev1; prev_type = new_type; } } if (prev_type == DCC_CK_INVALID) return; /* having picked a checksum, * map the record with its predecessor */ prev_ck = db_map_rcd_ck(dcc_emsg, &db_sts.rcd2, prev, prev_type); if (!prev_ck) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); /* The current and previous records must be old * and contain the same useful checksums. */ new_ck_num = DB_NUM_CKS(db_sts.rcd.d.r); new_ck = db_sts.rcd.d.r->cks; if (DB_RCD_TRIMMED(db_sts.rcd2.d.r)) prev_srvr = DCC_ID_COMP; else prev_srvr = DB_RCD_ID(db_sts.rcd2.d.r); prev_ck_num = DB_NUM_CKS(db_sts.rcd2.d.r); prev_ck = db_sts.rcd2.d.r->cks; for (;;) { /* we must run out of checksums in the two reports at the * same time */ if (prev_ck_num == 0 || new_ck_num == 0) { if (prev_ck_num == new_ck_num) break; return; } /* ignore paths */ prev_type = DB_CK_TYPE(prev_ck); if (DB_TEST_NOKEEP(db_parms.nokeep_cks, prev_type)) { --prev_ck_num; ++prev_ck; continue; } new_type = DB_CK_TYPE(new_ck); if (DB_TEST_NOKEEP(db_parms.nokeep_cks, new_type)) { --new_ck_num; ++new_ck; continue; } /* because the checksums are ordered, * give up at the first difference in checksums */ if (new_type != prev_type || memcmp(new_ck->sum, prev_ck->sum, sizeof(new_ck->sum))) return; /* Give up at the first recent and valuable checksum. */ if ((new_srvr != DCC_ID_COMP && !DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts, new_ex_ts[new_type].all)) || (prev_srvr != DCC_ID_COMP && !DCC_TS_OLDER_TS(db_sts.rcd2.d.r->ts, new_ex_ts[new_type].all))) return; --prev_ck_num; ++prev_ck; --new_ck_num; ++new_ck; } /* The current and previous records are compatiable. * Add the count of the previous record to the current record * and mark the previous record useless. * The individual totals in the current record are already correct, * so postpone worrying about the deleted record. */ new_tgts = DB_TGTS_RCD_RAW(db_sts.rcd.d.r); if (new_tgts < DCC_TGTS_TOO_MANY) { prev_tgts = DB_TGTS_RCD(db_sts.rcd2.d.r); if (prev_tgts > DCC_TGTS_TOO_MANY) return; if (prev_tgts == DCC_TGTS_TOO_MANY) { new_tgts = DCC_TGTS_TOO_MANY; } else { new_tgts += prev_tgts; if (new_tgts > DCC_TGTS_TOO_MANY) new_tgts = DCC_TGTS_TOO_MANY; } DB_TGTS_RCD_SET(db_sts.rcd.d.r, new_tgts); } /* mark one of the records to be deleted next time */ DB_TGTS_RCD_SET(db_sts.rcd2.d.r, 0); db_sts.rcd.d.r->srvr_id_auth = DCC_ID_COMP; db_sts.rcd.d.r->fgs_num_cks &= ~(DB_RCD_FG_TRIM | DB_RCD_FG_SUMRY | DB_RCD_FG_DELAY); /* use the newest timestamp */ if (DCC_TS_OLDER_TS(db_sts.rcd.d.r->ts, db_sts.rcd2.d.r->ts)) memcpy(db_sts.rcd.d.r->ts, db_sts.rcd2.d.r->ts, sizeof(db_sts.rcd.d.r->ts)); ++comp_rcds; } /* write a parsed whitelist checksum */ static int white_write(DCC_EMSG emsg, DCC_WF *wf, DCC_CK_TYPES type, DCC_SUM sum, DCC_TGTS tgts) { DB_RCD rcd; int rcd_len; char buf[30]; DCC_FNM_LNO_BUF fnm_buf; /* ignore checksums that clients are never supposed to send * to the server or for some other reason cannot be whitelisted */ switch (type) { case DCC_CK_INVALID: case DCC_CK_ENV_TO: case DCC_CK_G_MSG_R_TOTAL: case DCC_CK_G_TRIPLE_R_BULK: case DCC_CK_SRVR_ID: dcc_pemsg(EX_DATAERR, emsg, "%s checksum cannot be used%s", dcc_type2str_err(type, 0, 0, grey_on), wf_fnm_lno(fnm_buf, wf)); return 0; case DCC_CK_IP: case DCC_CK_ENV_FROM: case DCC_CK_FROM: case DCC_CK_MESSAGE_ID: case DCC_CK_RECEIVED: case DCC_CK_SUB: case DCC_CK_BODY: case DCC_CK_FUZ1: case DCC_CK_FUZ2: break; /* these are ok */ } if (tgts == DCC_TGTS_OK_MX || tgts == DCC_TGTS_OK_MXDCC || tgts == DCC_TGTS_SUBMIT_CLIENT) { dcc_pemsg(EX_DATAERR, emsg,"\"%s\" ignored%s", dcc_tgts2str(buf, sizeof(buf), tgts, 0), wf_fnm_lno(fnm_buf, wf)); return 0; } /* greylist whitelist entries cannot involve blacklisting * and use DCC_TGTS_GREY_WHITE to signal whitelisting */ if (grey_on) { /* ignore anything except whitelisting */ if (tgts != DCC_TGTS_OK) { dcc_pemsg(EX_DATAERR, emsg, "\"%s\" ignored%s", dcc_tgts2str(buf, sizeof(buf), tgts, 0), wf_fnm_lno(fnm_buf, wf)); return 0; } tgts = DCC_TGTS_GREY_WHITE; } memset(&rcd, 0, sizeof(rcd)); dcc_timeval2ts(rcd.ts, &clean_start, 0); rcd.srvr_id_auth = DCC_ID_WHITE; DB_TGTS_RCD_SET(&rcd, tgts); rcd.cks[0].type_fgs = DCC_CK_FLOD_PATH; memcpy(rcd.cks[0].sum, &wf->lno, sizeof(wf->lno)); rcd.cks[0].sum[sizeof(wf->lno)] = wf->fno; rcd.cks[1].type_fgs = type; memcpy(rcd.cks[1].sum, sum, sizeof(rcd.cks[1])); rcd_len = sizeof(rcd) - sizeof(rcd.cks) + 2*sizeof(rcd.cks[0]); rcd.fgs_num_cks = 2; ++white_cks; if (!write_new_rcd(&rcd, rcd_len)) return -1; return 1; } /* Add the whitelist of certified non-spam senders and otherwise * start the database */ static void parse_white(void) { int white_fd; white_cks = 0; if (!keep_white) { memset(&dbclean_white_tbl, 0,sizeof(dbclean_white_tbl)); dcc_wf_init(&dbclean_wf, 0); fnm2path_good(dbclean_wf.ascii_nm, WHITELIST_NM(grey_on), 0); dbclean_wf.info = &dbclean_white_tbl; white_fd = open(dbclean_wf.ascii_nm, O_RDONLY, 0); if (white_fd < 0) { /* worry only if the file exists but can't be used */ if (errno != ENOENT) { dcc_error_msg("open(%s): %s", dbclean_wf.ascii_nm, ERROR_STR()); keep_white = 1; } } else { if (0 > dcc_parse_whitefile(0, &dbclean_wf, white_fd, white_write, 0)) keep_white = 1; if (0 > close(white_fd)) dcc_error_msg("close(%s): %s", dbclean_wf.ascii_nm, ERROR_STR()); } } if (keep_white) { /* If the whitelist was bad, purge the new database of * the bad new whitelist. We will use the existing * whitelist */ write_new_flush(1); new_db_csize = DB_PTR_BASE; if (0 > ftruncate(new_db_fd, DB_PTR_BASE)) dcc_logbad(EX_IOERR, "truncate(%s, %d): %s", new_db_nm, DB_PTR_BASE, ERROR_STR()); new_db_fsize = DB_PTR_BASE; white_cks = 0; } /* update the counts in the database file */ write_new_hdr(1); } /* check for conflicts in the whitelist file in the record pointed to * by db_sts.rcd */ static void check_white(void) { static int msgs; static int prev_lno1, prev_lno2; static int prev_fno1, prev_fno2; const DB_RCD_CK *rcd_ck, *prev_ck; int lno1, lno2; int fno1, fno2; DCC_TGTS tgts1, tgts2; char tgts1_buf[30], tgts2_buf[30]; const char *fname1, *fname2; DCC_CK_TYPES type; DB_PTR prev; /* don't check if we have already complained enough */ if (msgs > 20) return; rcd_ck = db_sts.rcd.d.r->cks; /* it is pointless without line numbers, which are lacking only * if we saved the old whitelist entries because the file is * broken */ if (DB_NUM_CKS(db_sts.rcd.d.r) != 2 || DB_CK_TYPE(rcd_ck) != DCC_CK_FLOD_PATH) return; /* conflict is impossible with a single line */ ++rcd_ck; prev = DB_PTR_EX(rcd_ck->prev); if (prev == DB_PTR_NULL) return; type = DB_CK_TYPE(rcd_ck); prev_ck = db_map_rcd_ck(dcc_emsg, &db_sts.rcd2, prev, type); if (!prev_ck) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); tgts1 = DB_TGTS_RCD(db_sts.rcd2.d.r); tgts2 = DB_TGTS_RCD(db_sts.rcd.d.r); if (tgts1 == tgts2) return; /* no conflict */ memcpy(&lno1, db_sts.rcd2.d.r->cks[0].sum, sizeof(lno1)); fno1 = db_sts.rcd2.d.r->cks[0].sum[sizeof(lno1)]; memcpy(&lno2, db_sts.rcd.d.r->cks[0].sum, sizeof(lno2)); fno2 = db_sts.rcd.d.r->cks[0].sum[sizeof(lno2)]; if (lno1 == prev_lno1 && fno1 == prev_fno1 && lno2 == prev_lno2 && fno2 == prev_fno2) return; fname1 = wf_fnm(&dbclean_wf, fno1); fname2 = wf_fnm(&dbclean_wf, fno2); if (fname1 == fname2) { fname1 = ""; } else { fname1 = path2fnm(fname1); } dcc_error_msg("\"%s\" in line %d%s%s conflicts with \"%s\"" " in line %d of %s", dcc_tgts2str(tgts1_buf, sizeof(tgts1_buf), tgts1, grey_on), lno1, *fname1 != '\0' ? " of " : "", fname1, dcc_tgts2str(tgts2_buf, sizeof(tgts2_buf), tgts2, grey_on), lno2, fname2); ++msgs; prev_lno1 = lno1; prev_fno1 = fno1; prev_lno2 = lno2; prev_fno2 = fno2; } /* rebuild the hash table and the totals and links within the database file * finish with the file locked */ static void build_hash(void) { DB_PTR rcd_pos; DB_HADDR haddr_window, haddr_lo, haddr_hi; int pass, total_passes; u_int rcd_len; int rcd_cks, rcd_sums, rcds, sums; const DB_RCD_CK *rcd_ck; DB_HADDR guess_hash_len; int i; db_buf_init(new_db_pagesize, 0); if (!new_hash_len) { /* Try to choose a hash table size now so that when it * is next time to rebuild after 24 hours of incoming * checksums, the alpha or load factor will still be 0.9. * We probably ran 24 hours ago, so the old hash size * is a good estimate of the size tomorrow. */ new_hash_len = old_db_hash_used; if (new_hash_len < MIN_HASH_ENTRIES) new_hash_len = MIN_HASH_ENTRIES; /* guess the number of checksums that will arrive in the * next 24 hours based on the current number of checksums */ if (kept_cks != 0) { guess_hash_len = kept_cks+white_cks; i = expire_spamsecs / (24*60*60); if (i == 0) i = 1; else if (i > 10) i = 10; guess_hash_len += guess_hash_len / i; if (new_hash_len < guess_hash_len) { if (db_debug) dcc_trace_msg("guess hash size=%d" " instead of %d" " from kept_cks+white_cks" "=%d", guess_hash_len, new_hash_len, kept_cks+white_cks); new_hash_len = guess_hash_len; } } /* Guess the numbers of checksums tomorrow based on the * current average rate, provided we have at lease one day. */ if (new_db_parms.rate_secs > 24*60*60) { double rate = (24*60*60*1.0) / new_db_parms.rate_secs; rate *= new_db_parms.hash_added; /* Increase the rate by 10% to account * for the 30% decrease often seen on weekends. */ guess_hash_len = rate * 1.10; guess_hash_len += kept_cks+white_cks; if (new_hash_len < guess_hash_len) { if (db_debug) dcc_trace_msg("guess hash size=%d" " instead of %d" " from %d rate", guess_hash_len, new_hash_len, (int)rate); new_hash_len = guess_hash_len; } } /* go for alpha=0.9 in 24 hours */ new_hash_len += new_hash_len/10; if (new_hash_len > db_max_hash_entries) { if (db_debug) dcc_trace_msg("default hash size of %d" " is larger than maximum %d", new_hash_len, db_max_hash_entries); else dbclean_msg("default hash size of %d" " is larger than maximum %d", new_hash_len, db_max_hash_entries); } if (new_hash_len < MIN_HASH_ENTRIES) new_hash_len = MIN_HASH_ENTRIES; if (new_hash_len < DEF_HASH_ENTRIES && !grey_on) new_hash_len = DEF_HASH_ENTRIES; } /* Open and lock the new database */ unlink_whine(new_hash_nm, 1); new_hash_created = 1; if (!db_open(0, -1, new_db_nm, new_hash_len, DB_OPEN_LOCK_NOWAIT | dbclean_db_mode)) { dcc_logbad(dcc_ex_code, "could not start database %s", new_db_nm); } if (db_debug) dcc_trace_msg("using %s with %s", db_window_size_str, new_db_nm); /* guess which checksums we will keep so that we can count them */ if (old_db_parms.nokeep_cks != 0) db_parms.nokeep_cks = old_db_parms.nokeep_cks; /* add every record in the database file to the hash table and * fix its accumulated counts and reverse links */ comp_rcds = 0; sums = 0; rcds = 0; report_progress_init(); haddr_window = db_hash_page_len*((db_buf_total*3)/4); total_passes = (db_hash_len+haddr_window-1)/haddr_window; for (haddr_lo = 0, pass = 1; haddr_lo < db_hash_len; haddr_lo = haddr_hi, ++pass) { if (haddr_lo > db_hash_len-haddr_window) haddr_hi = MAX_HASH_ENTRIES; else haddr_hi = haddr_lo+haddr_window; for (rcd_pos = DB_PTR_BASE; rcd_pos < db_csize; rcd_pos += rcd_len) { /* skip reports crossing page bounardies */ if (rcd_pos%db_pagesize > db_page_max) { rcd_len = DB_RCD_HDR_LEN; continue; } if (--progress_rpt_cnt <= 0) { report_progress(0, " hash rebuilt", sums/total_passes, "checksums", kept_cks); } if (!db_map_rcd(0, &db_sts.rcd, rcd_pos, &rcd_len)) { dcc_logbad(dcc_ex_code, "hash build failed reading" " record at "L_HPAT, rcd_pos); } /* skip end of page padding */ if (db_sts.rcd.d.r->fgs_num_cks == 0) continue; ++rcds; /* count the checksums we'll link in this record */ rcd_cks = DB_NUM_CKS(db_sts.rcd.d.r); rcd_sums = 0; for (rcd_ck = db_sts.rcd.d.r->cks; rcd_ck < &db_sts.rcd.d.r->cks[rcd_cks]; ++rcd_ck) { if (!DB_TEST_NOKEEP(db_parms.nokeep_cks, DB_CK_TYPE(rcd_ck))) ++rcd_sums; } sums += rcd_sums; db_set_flush(&db_sts.rcd, 0, rcd_len); if (!db_link_rcd(dcc_emsg, haddr_lo, haddr_hi)) { dcc_logbad(dcc_ex_code, "relinking record at "L_HPAT": %s", rcd_pos, dcc_emsg); } /* check for conflicts in the whitelist file */ if (DB_RCD_ID(db_sts.rcd.d.r) == DCC_ID_WHITE) check_white(); compress_old(); } } db_parms.old_hash_used = db_hash_used; db_parms.hash_used = db_hash_used; db_parms.old_db_csize = db_csize; if (!db_flush_parms(dcc_emsg)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); report_progress(1, " hash rebuilt", sums/total_passes, "checksums", kept_cks); dbclean_msg("hashed %d records containing %d checksums," " compressed %d records", rcds, sums/total_passes, comp_rcds); dbclean_msg("%d hash entries total, %d or %d%% used", db_hash_len, HASH_LEN_EXT(db_hash_used), (int)((HASH_LEN_EXT(db_hash_used)*100.0) / HASH_LEN_EXT(db_hash_len))); } static u_char write_new_db(const void *buf, int buflen, off_t pos, u_char fatal) { int i; if (pos != lseek(new_db_fd, pos, SEEK_SET)) { if (fatal) { dcc_logbad(EX_IOERR, "lseek(%s, 0): %s", new_db_nm, ERROR_STR()); } else { dcc_error_msg("lseek(%s, 0): %s", new_db_nm, ERROR_STR()); } return 0; } i = write(new_db_fd, buf, buflen); if (i == buflen) { if (new_db_fsize < pos+buflen) new_db_fsize = pos+buflen; return 1; } if (fatal) { if (i < 0) dcc_logbad(EX_IOERR, "write(%s): %s", new_db_nm, ERROR_STR()); else dcc_logbad(EX_IOERR, "write(%s)=%d instead of %d", new_db_nm, i, buflen); } else { if (i < 0) dcc_error_msg("write(%s): %s", new_db_nm, ERROR_STR()); else dcc_error_msg("write(%s)=%d instead of %d", new_db_nm, i, buflen); } return 0; } /* use a large buffer to encourage the file system to avoid fragmentation */ static u_char write_new_db_buf[MIN_MIN_DB_MBYTE*1024*1024/4]; static u_int write_new_db_buflen = 0; static DB_PTR write_new_base; static u_char write_new_flush(u_char fatal) { u_char result = 1; if (write_new_db_buflen != 0) { if (!write_new_db(write_new_db_buf, write_new_db_buflen, write_new_base, fatal)) result = 0; } write_new_base = new_db_csize; write_new_db_buflen = 0; return result; } static u_char write_new_buf(const void *buf, int buflen) { if (write_new_db_buflen + buflen > ISZ(write_new_db_buf) && !write_new_flush(0)) return 0; memcpy(&write_new_db_buf[write_new_db_buflen], buf, buflen); write_new_db_buflen += buflen; return 1; } /* add a record to the new file */ static u_char write_new_rcd(const void *buf, int buflen) { static const u_char zeros[128] = {0}; DB_PTR new_page_num; u_char result; int pad, i; /* pad accross page boundaries */ new_page_num = (new_db_csize + buflen)/new_db_pagesize; if (new_page_num != new_db_csize/new_db_pagesize) { pad = new_page_num*new_db_pagesize - new_db_csize; pad = (((pad + DB_RCD_HDR_LEN-1) / DB_RCD_HDR_LEN) * DB_RCD_HDR_LEN); do { i = sizeof(zeros); if (i > pad) i = pad; if (!write_new_buf(zeros, i)) return 0; pad -= i; new_db_csize += i; } while (pad != 0); } result = write_new_buf(buf, buflen); new_db_csize += buflen; return result; } /* write the magic string at the head of the database file */ static void write_new_hdr(u_char emptied) { DB_HDR new; struct timeval old_sn; time_t new_rate_secs; DCC_CK_TYPES type; DCC_TGTS thold; int i; write_new_flush(1); new = def_db_hdr; dcc_timeval2ts(new.p.sn, &clean_start, 0); new.p.db_csize = new_db_csize; /* update the total traffic counts */ if (!cleardb && old_db_parms.db_csize >= old_db_parms.old_db_csize && old_db_parms.hash_used >= old_db_parms.old_hash_used) { new.p.rate_secs = old_db_parms.rate_secs; if (new.p.rate_secs > 0) { new.p.db_added = old_db_parms.db_added; new.p.hash_added = old_db_parms.hash_added; } dcc_ts2timeval(&old_sn, old_db_parms.sn); new_rate_secs = old_db_parms.last_rate_sec - old_sn.tv_sec; if (new_rate_secs > 0) { new_rate_secs += new.p.rate_secs; if (new_rate_secs > DB_MAX_RATE_SECS) { double trim, new_val; trim = DB_MAX_RATE_SECS; trim /= new_rate_secs; new_val = new.p.db_added; new_val *= trim; new.p.db_added = new_val; new_val = new.p.hash_added; new_val *= trim; new.p.hash_added = new_val; new_rate_secs = DB_MAX_RATE_SECS; } new.p.db_added += (old_db_parms.db_csize - old_db_parms.old_db_csize); new.p.hash_added += (old_db_parms.hash_used - old_db_parms.old_hash_used); new.p.rate_secs = new_rate_secs; } } if (grey_on) new.p.flags |= DB_PARM_FG_GREY; if (repair || dccd_started_us) { if (old_db_parms.flags & DB_PARM_FG_SELF_CLEAN) new.p.flags |= DB_PARM_FG_SELF_CLEAN2; new.p.flags |= DB_PARM_FG_SELF_CLEAN; } new.p.nokeep_cks = (emptied || old_db_parms.nokeep_cks == 0 ? def_nokeep_cks() : old_db_parms.nokeep_cks); for (type = DCC_CK_TYPE_FIRST; type <= DCC_CK_TYPE_LAST; ++type) { thold = new_ex_secs[type].clean_thold; if (thold != 0) { new.p.ex_secs[type].clean_thold = thold; new.p.ex_secs[type].all = new_ex_secs[type].all; new.p.ex_secs[type].spam = new_ex_secs[type].spam; memcpy(new.p.ex_spam[type], new_ex_ts[type].spam, sizeof(new.p.ex_spam[type])); } else { new.p.ex_secs[type].clean_thold = DEF_THOLD(new.p, grey_on, type); new.p.ex_secs[type].all = def_expire_secs; new.p.ex_secs[type].spam = (DCC_CK_LONG_TERM(type) ? def_expire_spamsecs : def_expire_secs); } } if (emptied || (old_db_parms.flags & DB_PARM_FG_CLEARED)) new.p.flags |= DB_PARM_FG_CLEARED; else new.p.flags &= ~DB_PARM_FG_CLEARED; new.p.pagesize = new_db_pagesize; new_db_parms = new.p; write_new_db(&new, sizeof(new), 0, 1); /* ensure that the last page of the file is complete */ if (new_db_pagesize != 0) { i = new_db_fsize % new_db_pagesize; if (i != 0) { if (!db_extend(dcc_emsg, new_db_fd, new_db_nm, new_db_fsize + (new_db_pagesize - i), new_db_fsize, new_db_pagesize)) dcc_logbad(dcc_ex_code, "%s", dcc_emsg); new_db_fsize += (new_db_pagesize - i); } } } static void unlink_whine(const char *nm, u_char enoent_ok) { if (0 > unlink(nm) && (!enoent_ok || errno != ENOENT)) dcc_error_msg("unlink(%s): %s", cur_db_nm, ERROR_STR()); } static void rename_bail(const char *from, const char *to) { if (0 > rename(from, to)) dcc_logbad(EX_IOERR, "rename(%s, %s): %s", from, to, ERROR_STR()); } /* try for a long time or until the server hears */ static u_char /* 1=ok, 0=failed */ persist_aop(DCC_AOPS aop, u_int32_t val1, u_char late_ok, /* accept late responses */ int secs) /* try for this long */ { struct timeval begin, now; char buf1[DCC_OPBUF]; char buf2[DCC_OPBUF]; u_char first; u_int32_t op_nums_r; DCC_OPS result; gettimeofday(&begin, 0); first = 1; op_nums_r = dcc_clnt_info->proto_hdr.op_nums.r+1; for (;;) { /* This kludge on the transaction ID makes all of our * tries appear to be retransmissions of a single request. * This is nice for operations such as DCC_AOP_FLOD_SHUTDOWN * that are not idempotent when the server has been stalled by * the operating system. */ if (late_ok) dcc_clnt_info->proto_hdr.op_nums.r = op_nums_r-1; result = dcc_aop(dcc_emsg, ctxt, grey_on, DCC_NO_SRVR, aop, val1, 0, 0, 0, 0, 0, &aop_resp, 0); /* finished if that worked */ if (result == DCC_OP_ADMN || result == DCC_OP_OK) return 1; if (result != DCC_OP_ERROR && result != DCC_OP_INVALID) dcc_pemsg(EX_UNAVAILABLE, dcc_emsg, "%s: %s", dcc_aop2str(buf1, sizeof(buf1), aop, val1), dcc_hdr_op2str(buf2, sizeof(buf2), &aop_resp.hdr)); /* deal with time change */ gettimeofday(&now, 0); if (begin.tv_sec > now.tv_sec || (begin.tv_sec == now.tv_sec && begin.tv_usec > now.tv_usec)) begin = now; /* eventually give up */ if (now.tv_sec > begin.tv_sec + secs) return 0; /* we may need a message */ if ((result != DCC_OP_ERROR && result != DCC_OP_INVALID) && (db_debug || first)) { first = 0; dcc_error_msg("%s", dcc_emsg); } sleep(5); } return 1; } /* tell the daemon to switch to the new database */ static void dccd_new_db(const char *msg) { /* Send a round of NOPs and ask about status to ensure the server * has dealt with requests that arrived while we had the database * locked and otherwise caught up. We want to try to ensure that * the server is listening when we re-open the database so that * it does not leave flooding off. * On some systems with lame mmap() support including BSD/OS, the * the daemon can stall for minutes in close(). */ if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_LIST, 0, RESTART_DELAY)) dcc_error_msg("%s: %s", msg, dcc_emsg); dccd_unlocked = 0; if (!persist_aop(DCC_AOP_DB_NEW, 0, 1, RESTART_DELAY)) { /* cannot be a fatal error, * lest we leave the database broken */ dcc_error_msg("%s: %s", msg, dcc_emsg); } } static void finish(void) { int bailing = 0; /* delete the new files */ #ifndef DCC_DBCLEAN_KEEP_NEW /* for debugging */ if (new_db_created) { unlink_whine(new_db_nm, 0); new_db_created = 0; bailing = -1; } /* we don't really know if the new hash file was created, * so don't worry about problems */ if (new_hash_created) { unlink_whine(new_hash_nm, 1); new_hash_created = 0; bailing = -1; } #endif if (cur_db_created) { unlink_whine(cur_db_nm, 0); unlink_whine(cur_hash_nm, 1); cur_db_created = 0; bailing = -1; } if (new_db_fd >= 0) { if (0 > close(new_db_fd)) dcc_error_msg("close(%s): %s", new_db_nm, ERROR_STR()); new_db_fd = -1; } if (old_db_fd >= 0) { /* In most cases nothing cares about the old database. * So push it out of RAM to the disk so it won't lurk in the * buffer cache or elsewhere to slow a system reboot */ if (exit_value == EX_OK && 0 > fsync(old_db_fd)) dcc_error_msg("fsync(%s): %s", old_db_nm, ERROR_STR()); if (0 > close(old_db_fd)) dcc_error_msg("close(%s): %s", old_db_nm, ERROR_STR()); old_db_fd = -1; } flod_unmap(0, 0); if (lock_db_fd >= 0) { unlink_whine(lock_db_nm, 0); close(lock_db_fd); lock_db_fd = -1; } /* release the daemon, but if the database is still open, it's bad */ db_close(bailing); /* tell the daemon to switch database */ if (dccd_unlocked) dccd_new_db("finish"); while (flods_off > 0) { --flods_off; if (!persist_aop(DCC_AOP_FLOD, DCC_AOP_FLOD_RESUME, 1, RESTART_DELAY)) dcc_error_msg("%s", dcc_emsg); } } /* terminate with a signal */ static void NRATTRIB sigterm(int s) { dcc_error_msg("interrupted"); exit(-s); }