// // bidwatcher // copyright (c) 1999-2005 // Trent McNair (trent@rmci.net) // Tom McNair (tmcnair@cyberhighway.net) // Wayne Schlitt (wayne@midwestcs.com) // Ben Byer (bushing@users.sourceforge.net) // Kevin Dwyer (kevin@pheared.net) // // use of this code is restricted to the terms // of the GNU GPL, which should have been included in this // distribution. If not, see www.gnu.org/copyleft/gpl.html. // Here is the short version: // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation; either version 2 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // #include #include #include #include #include #include #include #include #include #include #include #include #include "bidwatcher.h" using namespace std; // The next two functions are essentially taken from the examples // that come with the curl library. For speed we don't want to // start with an empty buffer, so memsize should be larger than size. struct MemoryBufferStruct { char *memory; // This is the character array size_t size; // This is the length of the string size_t memsize; // This is the actual length of the array }; size_t WriteMemoryBuffer(void *ptr, size_t size, size_t nmemb, void *data) { //DPRINTF(DHIGH, ("WMB: ptr:%#x size:%i nmemb:%i data:%#x\n", // ptr, size, nmemb, data)); register int realsize = size * nmemb; struct MemoryBufferStruct *mem = (struct MemoryBufferStruct *)data; if ((size_t) (realsize) >= (mem->memsize - mem->size)) { mem->memory = (char *)realloc(mem->memory, mem->size + realsize + 1); mem->memsize = mem->size + realsize + 1; } if (mem->memory) { memcpy(&(mem->memory[mem->size]), ptr, realsize); mem->size += realsize; // Update the string length mem->memory[mem->size] = 0; // Terminate the string with a NULL } return realsize; } struct MemoryBufferStruct curl_memory_chunk; bool cancelPressed; double avg_load_time = INITIAL_LOAD_TIME; URL::URL(char *newurl, char *postval, URL *proxy) { DPRINTF(DMED, ("New URL(%s).\n", newurl)); proxyuser = NULL; proxypass = NULL; create(newurl, postval, proxy); } URL::URL(char *newurl, char *postval, URL *proxy, char *user, char *pass) { DPRINTF(DMED, ("New URL(%s).\n", newurl)); proxyuser = strdup(user); proxypass = strdup(pass); create(newurl, postval, proxy); } inline void constFree(const void *ptr) { free((void*)ptr); } /* void URL::freeHostInfo(bool free_all) { if(!hostinfo) return; if (hostinfo->h_name) constFree(hostinfo->h_name); int i; for(i=0; i < numaliases; i++) constFree(hostinfo->h_aliases[i]); if (i > 0) constFree(hostinfo->h_aliases); for(i=0; i < numaddrs; i++) constFree(hostinfo->h_addr_list[i]); if (i > 0) constFree(hostinfo->h_addr_list); if(free_all) constFree(hostinfo); } */ URL::~URL() { // Clean this memory waster up. DPRINTF(DMED, ("Destroying URL(%s)\n", fullurl)); // If we are using a proxy, don't kill this. /*if (!useproxy && hostinfo) { freeHostInfo(true); }*/ if(proxyuser != NULL) free(proxyuser); if(proxypass != NULL) free(proxypass); } void URL::create(char *newurl, char *postval, URL *proxy) { sprintf(fullurl, "%s", newurl); sprintf(fullurlpost, "%s", postval); string tmp = fullurl; string hostport; unsigned int hostStart = strlen("http://"); unsigned int hostEnd; hostEnd = tmp.find(":", hostStart); if (hostEnd == string::npos) { // No port hostEnd = tmp.find("/", hostStart); } else { // There is a port unsigned int portStart = hostEnd + 1; unsigned int portEnd = tmp.find("/", hostEnd); hostport = tmp.substr(portStart, portEnd - portStart); port = atoi(hostport.c_str()); } hostname = tmp.substr(hostStart, hostEnd - hostStart); if (proxy == NULL) { useproxy = FALSE; } else { #ifdef DEBUG_NETWORK printf("Configured for proxy - host: %s, port: %d\n", proxy->hostname, proxy->port); #endif useproxy = TRUE; proxyuser = strdup(proxy->proxyuser); proxypass = strdup(proxy->proxypass); } } bool checkredir(URL *url, char *buff) { // We are extracting the URL from a string of the form // location: http://... if (stristr(buff,"location: http") != NULL) { char *loc = stristr(buff,"location: http"); loc = loc + 10; loc[strcspn(loc,"\r\n")] = '\0'; sprintf(url->fullurl,"%s\n",loc); return TRUE; } // We are extracting the URL from a string of the form // // First we seek to the end of the "url =" part, then start // counting until we hit the ending " mark. Could we just // link to a regular expression library instead? if (strstr(buff,"fullurl,"%.*s\n",substring_len,loc); return TRUE; } return FALSE; } int fetchURL(URL *url, int Post, char **Buff, char *cookiejar, int redir) { size_t avebuffsize = 1 << 15; // 2 to the 15th power CURL *curl_handle; CURLM *multi_handle; curl_memory_chunk.memory = (char *) malloc(avebuffsize*sizeof(char)); curl_memory_chunk.size = 0; curl_memory_chunk.memsize = avebuffsize; // Initialize the curl session curl_handle = curl_easy_init(); multi_handle = curl_multi_init(); // Return both the headers and the body curl_easy_setopt(curl_handle, CURLOPT_HEADER, 1); curl_easy_setopt(curl_handle, CURLOPT_NOBODY, 0); // Automatically handle all the cookies curl_easy_setopt(curl_handle, CURLOPT_COOKIEJAR, cookiejar); curl_easy_setopt(curl_handle, CURLOPT_COOKIEFILE, cookiejar); // Act like we are the mozilla browser curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) " "Gecko/20040113"); // This next feature doesn't appear to work properly so we follow by hand curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1); if (url->useproxy) { char proxyuserpwd[200]; memset(proxyuserpwd, 0, sizeof proxyuserpwd); if (strlen(proxyurl->proxyuser)) { snprintf(proxyuserpwd, sizeof proxyuserpwd - 1, "%s:%s", proxyurl->proxyuser, proxyurl->proxypass); curl_easy_setopt(curl_handle, CURLOPT_PROXYUSERPWD, proxyuserpwd); } char proxyinfo[200]; memset(proxyinfo, 0, sizeof proxyinfo); snprintf(proxyinfo, sizeof proxyinfo - 1, "%s:%d", proxyurl->hostname.c_str(), proxyurl->port); curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxyinfo); } // Set the URL we wish to fetch curl_easy_setopt(curl_handle, CURLOPT_URL, url->fullurl); // Follow redirects curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1); // If we are posting, set the post fields if (Post) { curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, url->fullurlpost); } // All the data will be sent to this function by curl curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryBuffer); // We will need our memory buffer to be pass to the write function curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&curl_memory_chunk); curl_multi_add_handle(multi_handle, curl_handle); int still_running; while(CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multi_handle, &still_running)); // Now get that URL while(still_running) { struct timeval timeout; int rc; /* select() return code */ fd_set fdread; fd_set fdwrite; fd_set fdexcep; int maxfd; FD_ZERO(&fdread); FD_ZERO(&fdwrite); FD_ZERO(&fdexcep); /* set a suitable timeout to play around with */ timeout.tv_sec = 0; timeout.tv_usec = 1; /* get file descriptors from the transfers */ curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd); rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout); switch(rc) { case -1: /* select error */ break; case 0: /* timeout */ default: /* timeout or readable/writable sockets */ gtk_main_iteration_do(FALSE); while(CURLM_CALL_MULTI_PERFORM == curl_multi_perform(multi_handle, &still_running)); break; } } int remaining_msgs = 1; int error = 0; CURLMsg *msg; while (remaining_msgs) { msg = curl_multi_info_read(multi_handle, &remaining_msgs); if (msg != NULL) { if (CURLE_OK != msg->data.result) error = 1; } } curl_multi_cleanup(multi_handle); if (error == 0) { // If we haven't hit redirect limit, follow any redirects //if (redir && checkredir(url,curl_memory_chunk.memory)) { /* cleanup curl stuff */ // curl_easy_cleanup(curl_handle); // return fetchURL(url, 0, Buff, cookiejar, redir - 1); //} // Set the buffer pointer to the curl memory buffer *Buff = curl_memory_chunk.memory; /* cleanup curl stuff */ curl_easy_cleanup(curl_handle); return NET_SUCCESS; } else { return NET_NETERROR; } }