//
// $Id: httpgrabber.c,v 1.2 2001/08/31 13:28:56 mavetju Exp $
//
// httpgrabber.c - Grab HTTP traffic by Edwin Groothuis <edwin@mavetju.org>
//
// See the LICENSE file on distribution and copying.
//

#include <stdio.h>
#include <pcap.h>

#include <sys/types.h>
#include <sys/socket.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/tcp.h>
#include <netinet/udp.h>
#include <netinet/ip_icmp.h>

#include <arpa/inet.h>
#include <ctype.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <sys/param.h>
#include <signal.h>
#include <time.h>

#include "ngrep-lib.h"

//
// Flags
//
int VERBOSE=0;
int verbose=0;
int alarmtime=30;
int tcpttl=600;
char *tcpport=NULL;
char *outputdir=NULL;
char *device=NULL;
int imageonly=0;

//
// Unique next session number
//
int next_sessionnumber=1;

typedef struct session SESSION;

struct session { 
    struct in_addr	ip_src;		// ip source address
    struct in_addr	ip_dst;		// ip destination address
    u_short		th_sport;	// tcp source port
    u_short		th_dport;	// tcp destination port

    int			is_sender;	// true if session is sending
					// the HTTP request

    tcp_seq		th_startseq;	// tcp sequence number
    int			bytes;		// bytes already transmitted

    int			finished_httpheader;	// initialized HTTP header
    long		sizeof_httpheader;	// already.

    time_t		time_started;	// statistical information
    time_t		time_lastpacket;// for timeout

    FILE *		fout;		// output file for this session

    char *		tempname;	// temporary output filename
    char *		filename;	// real output filename
    char *		url;		// info from GET field 
    char *		host;		// info from Host: field
    char *		contenttype;	// info from Content-Type: field

    int			sessionnumber;	// unique session number
    SESSION *		partner;	// partner of this TCP session
    SESSION *		next;
};
SESSION *session_head=NULL;

void (*oldinfosignal)()=NULL;
void (*oldalarmsignal)()=NULL;

void siginfo(int sig);
void sigalarm(int sig);

void process_args(int argc,char **argv);

void my_tcp(struct ip *iphdr, struct tcphdr *tcphdr,char *data,int len);
void my_info(char *interface,char *filter,char *match,char *output);
void my_stats(long received,long dropped);

void show_sessions(void);
void cleanup_sessions(void);

SESSION *create_session(struct ip *iphdr, struct tcphdr *tcp,int is_sender);
void close_session(SESSION *session);

SESSION *find_partner(struct ip *iphdr, struct tcphdr *tcp);
SESSION *find_session(struct ip *iphdr, struct tcphdr *tcp);

void write_data(SESSION *session,struct tcphdr *tcp,char *data,int len);
void parse_sender_header(SESSION *session,struct tcphdr *tcp,char *data,int len);
void parse_receiver_header(SESSION *session,struct tcphdr *tcp,char *data,int len);
void write_data(SESSION *session,struct tcphdr *tcp,char *data,int len);

void my_tcp(struct ip *iphdr,struct tcphdr *tcp,char *data,int len);


//
// Read the arguments, set signal handlers, set some callback functions and run!
//
int main(int argc,char **argv) {
    process_args(argc,argv);

    oldinfosignal=signal(SIGINFO,siginfo);
    oldalarmsignal=signal(SIGALRM,sigalarm);
    alarm(alarmtime);

    ngrep_callback_process(NULL,&my_tcp,NULL,NULL);
    ngrep_callback_info(&my_info,&my_stats);
    ngrep_setfilter(tcpport,NULL,device);
    ngrep_run();
    return 0;
}

//
// Print the usage screen
//
void usage(void) {
    fprintf(stderr,
	"Usage: httpgrabber [options...]\n"
	"Options are:\n"
	"-d device          which interface to use\n"
	"-i                 images only, default everything is grabbed\n"
	"-o outputdir       outputdir, default .\n"
	"-p port            which port to listen on, default 80\n"
	"-t time            TTL of idle connections, default 600 seconds\n"
	"-v                 verbose\n"
	"-V                 very verbose\n"
    );
    exit(0);
}

//
// process the arguments
//
void process_args(int argc,char **argv) {
    int ch;

    while ((ch=getopt(argc,argv,"d:io:p:t:vV"))>0) {
	switch (ch) {
	case 'd':
	    asprintf(&device,"-d %s",optarg);
	    if (verbose)
		printf("Using %s for grabbing\n",optarg);
	    break;
	case 'i':
	    imageonly=1;
	    break;
	case 'o':
	    outputdir=optarg;
	    break;
	case 'p':
	    asprintf(&tcpport,"port %s",optarg);
	    if (verbose)
		printf("Using %s\n",tcpport);
	    break;
	case 't':
	    tcpttl=atoi(optarg);
	    break;
	case 'v':
	    verbose=1;
	    break;
	case 'V':
	    VERBOSE=1;
	    break;
	default:
	    usage();
	}
    }

    if (tcpport==NULL) tcpport="port 80";
    if (outputdir==NULL) outputdir=".";
}

//
// SIGINFO handler (^T pressed)
//
void siginfo(int sig) {
    show_sessions();
    if (oldinfosignal!=NULL) (*oldinfosignal)();
}


//
// Periodic removal of old sessions
//
void sigalarm(int sig) {
    cleanup_sessions();
    if (oldalarmsignal!=NULL) (*oldalarmsignal)();
    alarm(alarmtime);
}


//
// Information from the pcap-filter compiler
//
void my_info(char *interface,char *filter,char *match,char *output) {
    if (VERBOSE)
	printf("interface: %s\nfilter: %s\nmatch: %s\noutput: %s\n",
	    interface,filter,match,output);
}


//
// Information from the packet filter afterwards
//
void my_stats(long received, long dropped) {
    if (verbose)
	printf("received %ld packets, dropped %ld packets\n",
	    received,dropped);
}


//
// Create a directory based on a pathname and filename.
//
void makedir(char *s) {
    char *ps;
    char path[MAXPATHLEN];

    if (VERBOSE) printf("Creating directory: %s\n",s);

    getcwd(path,MAXPATHLEN);
    while ((ps=strchr(s,'/'))!=NULL) {
	ps[0]=0;
	mkdir(s,0755);
	chdir(s);
	ps[0]='/';
	s=ps+1;
    }
    chdir(path);
}


//
// Allocate a new session and initialize it.
// Also open the file which belongs to it, if it's an receiving session.
//
SESSION *create_session(struct ip *iphdr, struct tcphdr *tcp,int is_sender) {
    SESSION *new_session;
    char s[100];

    new_session=(SESSION *)calloc(1,sizeof(SESSION));
    memcpy(&new_session->ip_src,&iphdr->ip_src,sizeof(struct in_addr));
    memcpy(&new_session->ip_dst,&iphdr->ip_dst,sizeof(struct in_addr));
    new_session->th_dport=tcp->th_dport;
    new_session->th_sport=tcp->th_sport;
    new_session->th_startseq=tcp->th_seq;

    new_session->finished_httpheader=0;
    new_session->sizeof_httpheader=0;

    new_session->sessionnumber=next_sessionnumber++;
    new_session->time_started=time(NULL);
    new_session->time_lastpacket=time(NULL);

    if (!is_sender) {
	//
	// Create temp-dir (if needed) and open the output file.
	//
	strcpy(s,"temp/a");
	makedir(s);
	asprintf(&new_session->tempname,"temp/%d",new_session->sessionnumber);
	if ((new_session->fout=fopen(new_session->tempname,"w"))==NULL) {
	    perror(s);
	    exit;
	}
    }

    new_session->is_sender=is_sender;

    new_session->next=session_head;
    session_head=new_session;

    return new_session;
}


//
// Find a session with the same IP addresses and ports
//
SESSION *find_session(struct ip *iphdr, struct tcphdr *tcp) {
    SESSION *session;

    for (session=session_head;session!=NULL;session=session->next) {
	if (session->th_dport!=tcp->th_dport) continue;
	if (session->th_sport!=tcp->th_sport) continue;
	if (memcmp(&session->ip_src,&iphdr->ip_src,sizeof(struct in_addr))!=0)
	    continue;
	if (memcmp(&session->ip_dst,&iphdr->ip_dst,sizeof(struct in_addr))!=0)
	    continue;
	return session;
    }
    return NULL;
}


//
// Find a session with the same IP addresses and ports but inverted.
//
SESSION *find_partner(struct ip *iphdr, struct tcphdr *tcp) {
    SESSION *session;

    for (session=session_head;session!=NULL;session=session->next) {
	if (session->th_sport!=tcp->th_dport) continue;
	if (session->th_dport!=tcp->th_sport) continue;
	if (memcmp(&session->ip_dst,&iphdr->ip_src,sizeof(struct in_addr))!=0)
	    continue;
	if (memcmp(&session->ip_src,&iphdr->ip_dst,sizeof(struct in_addr))!=0)
	    continue;
	return session;
    }
    return NULL;
}


//
// Finish a session. That means:
// - Close the file. If it's zero-length, delete it. Move it to host/url.
// - Free the data
// - Close the linked list
//
void close_session(SESSION *session) {
    SESSION *previous;
    long size;
    int isimage;

    if (!session->is_sender) {
	fseek(session->fout,0,SEEK_END);
	size=ftell(session->fout);
	fclose(session->fout);

	isimage=0;
	if (strncmp(session->contenttype,"image/",6)==0)
	    isimage=1;

	if (size==0 || (!isimage && imageonly) ) {
	    unlink(session->tempname);
	    if (verbose) {
		if (session->host==NULL && session->url==NULL)
		    printf("file/empty: %s\n",inet_ntoa(session->ip_src));
		else
		    printf("file/empty: %s%s\n",
			session->host,
			session->url);
	    }
	} else {
	    asprintf(&session->filename,"%s%s%s",
		(session->host==NULL || session->host[0]==0)?
		    inet_ntoa(session->ip_src):session->host,
		(session->url==NULL || session->url[0]==0)?
		    "/":session->url,
		 session->url==NULL || session->url[strlen(session->url)-1]=='/'?
		    "index.html":"");
	    makedir(session->filename);
	    rename(session->tempname,session->filename);
	    printf("%s: %s\n",session->contenttype,session->filename);
	}
    }

    //
    // get rid of the partners partner (that's this one).
    //
    if (session->partner!=NULL) session->partner->partner=NULL;

    //
    // free the mallocs!
    //
    if (session->filename   !=NULL) free(session->filename);
    if (session->tempname   !=NULL) free(session->tempname);
    if (session->contenttype!=NULL) free(session->contenttype);
    if (session->host       !=NULL) free(session->host);
    if (session->url        !=NULL) free(session->url);

    //
    // repair the linked list
    //
    if (session==session_head) {
	session_head=session_head->next;
	free(session);
    } else {
	previous=session_head;

	while (previous->next!=session)
	    previous=previous->next;
	previous->next=previous->next->next;
	free(session);
    }
}


//
// Parse the HTTP header for the sender data
// It searches for two fields:
// - The HTTP request
// - The Host: field
//
void parse_sender_header(SESSION *session,struct tcphdr *tcp,char *data,int len) {
    char *get;
    char *host;

    //
    // search for the GET field.
    // syntax is either "GET /directory/file HTTP/x.y" or "GET /directory/file"
    //
    if ((get=strstr(data,"GET "))!=NULL) {
	char *http;

	get+=4;
	http=strstr(get,"HTTP/");
	if (http==NULL)
	    http=strchr(get,'\x0d');
	http[-1]=0;
	asprintf(&session->partner->url,"%s",get);
	http[-1]=' ';
    }

    //
    // get the Host field.
    // syntax is "Host: host.domain"
    //
    if ((host=strstr(data,"Host: "))!=NULL) {
	char *od=strstr(host,"\x0d");

	host+=6;
	*od=0;
	asprintf(&session->partner->host,"%s",host);
    }
}


//
// Parse the HTTP header for the returned data.
// Actually, it searches for two things:
// - the Content-Type field
// - the double 0x0d/0x0a at the end of the header
//
void parse_receiver_header(SESSION *session,struct tcphdr *tcp,char *data,int len) {
    int i;
    char *p;
    int seencrlf=0;

    //
    // search for Content-Type:
    // syntax is "Content-Type: mimetype"
    //
    if ((p=strstr(data,"Content-Type: "))!=NULL) {
	char *od=strchr(p,'\x0d');
	p+=14;
	*od=0;
	asprintf(&session->contenttype,"%s",p);
	*od='\x0d';
    }

    //
    // search for the empty line at the end of the HTTP header.
    //
    p=data;
    i=0;
    while (i<len) {
	if (*p==0x0a)
	    seencrlf++;
	if (seencrlf==2)
	    break;
	if (*p!=0x0d && *p!=0x0a)
	    seencrlf=0;
	p++;
	i++;
    }

    if (seencrlf==2) {
	p++;
	session->finished_httpheader=1;
	write_data(session,tcp,p,len-(p-data));
    }
    session->sizeof_httpheader+=i+1;
}


//
// Write the data into a file.
// Only after an HTTP header has received. Will probably break if
// a the returned HTTP header is spread over more than one IP packet.
// TCP is a stream protocol, not a packet protocol! What is done here
// is bad behaviour.
//
void write_data(SESSION *session,struct tcphdr *tcp,char *data,int len) {
    int offset;

    if (len==0) return;

    //
    // check if the HTTP header is already processed. If not, do it now.
    //
    if (!session->finished_httpheader) {
	if (session->is_sender)
	    parse_sender_header(session,tcp,data,len);
	else
	    parse_receiver_header(session,tcp,data,len);
	return;
    }

    //
    // statistical data
    //
    session->bytes+=len;
    session->time_lastpacket=time(NULL);

    //
    // we don't write information for senders, only the returned data
    //
    if (session->is_sender) return;

    offset=ntohl(tcp->th_seq)-
	   ntohl(session->th_startseq)-
	   session->sizeof_httpheader-1;

    fseek(session->fout,offset,SEEK_SET);
    fwrite(data,len,1,session->fout);
}


//
// Print some information about the sessions. Press ^T to see it.
//
void show_sessions(void) {
    SESSION *session=session_head;
    time_t timepassed;

    printf("Current sessions:\n");
    while (session!=NULL) {
	if (session->is_sender) {
	    printf("%5d S ",session->sessionnumber);
	    if (session->partner!=NULL)
		printf("belongs to session %d\n",
		    session->partner->sessionnumber);
	    else
		printf("doesn't have a partner session.\n");
	    printf("        From %s:%d ",
		inet_ntoa(session->ip_src),ntohs(session->th_sport));
	    printf("to %s:%d\n",
		inet_ntoa(session->ip_dst),ntohs(session->th_dport));
	    timepassed=time(NULL)-session->time_lastpacket;
	    printf("        idle for %ld seconds\n",timepassed);
	} else {
	    printf("%5d R ",
		session->sessionnumber);
	    if (session->url!=NULL)
		printf("http://%s%s\n",session->host,session->url);
	    else
		printf("no url send yet\n");

	    if (session->contenttype!=NULL)
		printf("        context-type: %s\n",session->contenttype);

	    timepassed=time(NULL)-session->time_lastpacket;
	    printf("        idle for %ld seconds\n",timepassed);

	    timepassed=time(NULL)-session->time_started;
	    printf("        speed: %d bytes received in %ld seconds (%ld bps)\n",
		session->bytes,
		timepassed,
		timepassed==0?0:session->bytes/timepassed);
	}
	session=session->next;
    }
    if (session_head==NULL)
	printf("none!\n");
}


//
// Periodic called to clean up old TCP sessions which are dead.
// Normally they're closed via a FIN packet, but you never know
// in this world.
//
void cleanup_sessions(void) {
    SESSION *session,*session_next;
    time_t now=time(NULL);
    time_t idle;

    for (session=session_head;session!=NULL;session=session_next) {
	session_next=session->next;

	idle=session->time_lastpacket;
	if (session->is_sender && session->partner!=NULL)
	    idle=session->partner->time_lastpacket;
	if (idle>now-tcpttl)
	    continue;

	close_session(session);
    }
}


//
// Callback function from ngrep-lib.
// It checks is the function which is processing the IP/TCP headers
// and checks for new sessions, data to be written and closing of sessions.
//
void my_tcp(struct ip *iphdr,struct tcphdr *tcp,char *data,int len) {
    SESSION *session;

    //
    // packets with the SYN-flag are initiators on new sessions.
    // but, they might be send more than once during initialisation 
    // so we have to check for that.
    //
    if (tcp->th_flags & TH_SYN) {
	if ((session=find_session(iphdr,tcp))==NULL) {
	    if (tcp->th_flags & TH_ACK)
		session=create_session(iphdr,tcp,0);
	    else
		session=create_session(iphdr,tcp,1);
	} else
	    if (VERBOSE)
		printf("refound %d\n",session->sessionnumber);
    } else {
	if (tcp->th_flags & TH_RST) {
	    if ((session=find_partner(iphdr,tcp))!=NULL)
		close_session(session);
	    if ((session=find_session(iphdr,tcp))!=NULL)
		close_session(session);
	    return;
	}
	if ((session=find_session(iphdr,tcp))==NULL)
	    return;
    }

    //
    // connect this session with its partner
    //
    if (session->partner==NULL) {
	session->partner=find_partner(iphdr,tcp);
	if (session->partner!=NULL)
	    session->partner->partner=session;
    }

    if (VERBOSE) {
	char ip_src[16];
	char ip_dst[16];

	strcpy(ip_src,inet_ntoa(iphdr->ip_src));
	strcpy(ip_dst,inet_ntoa(iphdr->ip_dst));

	printf("%2d %s%s%s%s %4d bytes from %s:%d to %s:%d\n",
	    session->sessionnumber,
	    tcp->th_flags & TH_SYN ? "S": " ",
	    tcp->th_flags & TH_ACK ? "A": " ",
	    tcp->th_flags & TH_RST ? "R": " ",
	    tcp->th_flags & TH_FIN ? "F": " ",
	    len,
	    ip_src,
	    ntohs(tcp->th_sport),
	    ip_dst,
	    ntohs(tcp->th_dport));
    }

    write_data(session,tcp,data,len);

    //
    // FIN flags means end of that TCP-session
    //
    if (tcp->th_flags & TH_FIN)
	close_session(session);
}