#include "cygwin.h" #include "tiger.h" #include "sha1.h" #include "md4.h" // 1. SIZE = The file Size // 2. NODES_8K = (SIZE + 10239) / 10240 static char *base32Chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; static void EncodeBase32 (const unsigned char *buffer, unsigned int bufLen, char *base32Buffer) {{{ unsigned int i, index; unsigned char word; for(i = 0, index = 0; i < bufLen;) { /* Is the current word going to span a byte boundary? */ if (index > 3) { word = (buffer[i] & (0xFF >> index)); index = (index + 5) % 8; word <<= index; if (i < bufLen - 1) word |= buffer[i + 1] >> (8 - index); i++; } else { word = (buffer[i] >> (8 - (index + 5))) & 0x1F; index = (index + 5) % 8; if (index == 0) i++; } assert(word < 32); *(base32Buffer++) = (char)base32Chars[word]; } *base32Buffer = 0; }}} const unsigned char *hash_bin2hex (const unsigned char *hash) {{{ const unsigned char hexdigits[] = "0123456789abcdef"; static unsigned char m_hash_str[33]; int j = 0; for (j = 0; j<16; j++) { m_hash_str[(j<<1) ] = hexdigits[(((hash[j]) & 0xf0) >> 4)]; m_hash_str[(j<<1)+1] = hexdigits[(((hash[j]) & 0x0f) )]; } return m_hash_str; }}} void tiger_compose (word64 *data, size_t count, word64 *ret) {{{ word64 *temp = (word64*) malloc(TIGERSIZE * count); memcpy(temp, data, count * TIGERSIZE); while(count > 1) { size_t i = 0; for(i = 0; 2 * i + 1< count; i++) { tiger (temp + i * 6, TIGERSIZE * 2, ret); memcpy(temp + i * 3, ret , TIGERSIZE); } if (count != 2 * i) memcpy(temp + i * 3, temp + i * 6, TIGERSIZE); count -= (count >> 1); } memcpy(ret, temp, TIGERSIZE); free (temp); }}} void tiger_tree_8k (word64 *data, size_t size , word64 *ret) {{{ word64 temp[3*8]; size_t count = 0; while (size > 0) { size_t todo = 1024; if (todo > size) todo = size; tiger(data, todo, temp+count*3); size -= todo; ((char*)data) += todo; count++; } tiger_compose(temp, count, ret); }}} #define BLOCKSIZE (1024*8) #define PARTSIZE 9728000 /* 1. Details 3 Access methods to the new Subhash System Potocol: OP_TREE_REQ OP_TREE_RES [(size+8092-1)/8092] OP_TREE_DETAIL_REQ <8 MB Index> OP_TREE_DETAIL_RES [max 1024 => 24kb] Dateiname: ..tree Dateistruktur: SIZE Orginal dateigröße 4 Byte SHA1_HASH SHA1 full Hash 20 Byte ED2K_HASH eDonkey Hash 16 Byte ED2K_LIST eDonkey_Part Hash 16 Byte * (SIZE + 9728000 - 1) / 9728000 TIGER_TOP Tiger Tree TOP 24 Byte TIGER_BLOCK Tiger Block size 4 Byte // Basic is still 1K TIGER_8K Tiger Tree NODES (8k) 24 Byte * (SIZE + TIGER_BLOCK - 1) / TIGER_BLOCK File-Meta-Tags: => 24 Byte Hash => 20 Byte Hash => 16 Byte Hash 2. Comments + The file access allow an granularity from 1K up to single node. + The file store only one level since all higher can be reconstruct + The protocol acces allow the use without transfer the compleate file + Partial Blocks can be created after an an eDonkey part_hash is verified + There are already 700.000 files index with sha1/tiger_hash - sha1 in the file is overhead but allow access to bitzi.com - with 8K blocksize the file take 2,4 MB for an 800 MB source file + For verifycation of only an defekt block you need the TOP-Hash, 8MB-Hash and the 8K hash for the defekt range. this is less than 300k !!! Compared with an 9,2 MB refetch of avereage with ICH 4,5 even an big improve. 3. Suggestion store an list who send wich part for blocking him on the refetch. */ class cTiger { private: size_t file_size; unsigned char sha1 [SHA_DIGESTSIZE]; unsigned char ed2k [MD4_DIGEST_LENGTH]; unsigned char tiger[TIGERSIZE]; unsigned char *ed2k_list; size_t tiger_block; unsigned char *tiger_list; bool ed2k_ok; bool tiger_ok; public: cTiger (void ); ~cTiger (void ); bool Store (void ); bool Load (int fd ); bool Build (int fd ); int Check (int fd, size_t from, size_t till); bool Print (void ); }; cTiger:: cTiger(void) {{{ file_size = 0; ed2k_list = NULL; tiger_list = NULL; ed2k_ok = false; tiger_ok = false; tiger_block = BLOCKSIZE; }}} cTiger::~cTiger(void) {{{ if (ed2k_list != NULL) free (ed2k_list); if (tiger_list != NULL) free (tiger_list); }}} bool cTiger::Store (void) {{{ size_t nodes_tiger = ((file_size + tiger_block - 1) / tiger_block); size_t nodes_ed2k = ((file_size + PARTSIZE - 1) / PARTSIZE ); size_t LEN = 4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(nodes_ed2k+1)+4+TIGERSIZE*(nodes_tiger+1); char *BUF = (char*)malloc(LEN); memcpy(BUF , &file_size , 4); memcpy(BUF+4 , sha1 , SHA_DIGESTSIZE); memcpy(BUF+4+SHA_DIGESTSIZE , ed2k , MD4_DIGEST_LENGTH); memcpy(BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH , ed2k_list , MD4_DIGEST_LENGTH * nodes_ed2k ); memcpy(BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(1+nodes_ed2k) , tiger , TIGERSIZE); memcpy(BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(1+nodes_ed2k)+TIGERSIZE,&tiger_block, 4); memcpy(BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(1+nodes_ed2k)+28, tiger_list , TIGERSIZE * nodes_tiger); unsigned char md4[MD4_DIGEST_LENGTH]; MD4_CTX md4_ctx; MD4Init (&md4_ctx); MD4Update(&md4_ctx, (unsigned char*)BUF, LEN); MD4Final (md4, &md4_ctx); char dateiname[512]; snprintf(dateiname, 512, "%s.tree", hash_bin2hex(md4)); int fd = open (dateiname, O_WRONLY|O_TRUNC|O_CREAT, 0600); write(fd, BUF, LEN); close(fd); free (BUF); }}} bool cTiger::Print (void) {{{ char buffer[128]; printf("size = %9u\n", file_size); EncodeBase32(sha1, SHA_DIGESTSIZE, buffer); printf("SHA1(32) = %s\n", buffer); EncodeBase32(tiger, TIGERSIZE, buffer); printf("TIGER(32)= %s\n", buffer); printf("ED2K = %s\n", hash_bin2hex(ed2k)); }}} bool cTiger::Build (int fd) {{{ if (fd == -1) return false; file_size = lseek (fd, 0 , SEEK_END); lseek (fd, 0, SEEK_SET); size_t nodes_tiger = ((file_size + tiger_block - 1) / tiger_block); size_t nodes_ed2k = ((file_size + PARTSIZE - 1) / PARTSIZE ); tiger_list= (unsigned char*) malloc(TIGERSIZE * nodes_tiger); ed2k_list = (unsigned char*) malloc(MD4_DIGEST_LENGTH * nodes_ed2k ); size_t size = file_size; unsigned idx_tiger = 0; unsigned idx_ed2k = 0; SHA_INFO scontext; MD4_CTX md4_ctx; MD4Init(&md4_ctx); sha_init(&scontext); size_t ed_todo = PARTSIZE; while (size > 0) {{{ char BUF_8K[tiger_block]; if (ed_todo > size) ed_todo = size; size_t done = read(fd, BUF_8K, tiger_block); if (ed_todo <= done) { MD4Update(&md4_ctx, (unsigned char*)BUF_8K, ed_todo); MD4Final (ed2k_list + idx_ed2k*MD4_DIGEST_LENGTH, &md4_ctx); idx_ed2k++; MD4Init(&md4_ctx); if (done - ed_todo > 0) MD4Update(&md4_ctx, (unsigned char*)BUF_8K + ed_todo, done - ed_todo); ed_todo = PARTSIZE - done + ed_todo; } else { MD4Update(&md4_ctx, (unsigned char*)BUF_8K, done); ed_todo -= done; } assert (done > 0 && ( done == tiger_block || done == size)); sha_update(&scontext, (BYTE*)BUF_8K, done); tiger_tree_8k ((word64*)BUF_8K, done,(word64*) (tiger_list+(idx_tiger*TIGERSIZE))); size -= done; idx_tiger++; }}} fd = -1; sha_final(sha1, &scontext); tiger_compose((word64*)tiger_list, idx_tiger, (word64*)tiger); MD4Init(&md4_ctx); MD4Update(&md4_ctx, ed2k_list, idx_ed2k*MD4_DIGEST_LENGTH); MD4Final (ed2k, &md4_ctx); Store (); Print (); }}} bool cTiger::Load (int fd) {{{ if (fd == -1) return false; off_t LEN = lseek (fd, 0 , SEEK_END); char *BUF = (char*) malloc(LEN); lseek (fd, 0, SEEK_SET); read(fd, BUF, LEN); memcpy (&file_size, BUF, 4); size_t nodes_ed2k = (file_size + PARTSIZE - 1) / PARTSIZE; ed2k_list = (unsigned char*)malloc(nodes_ed2k *MD4_DIGEST_LENGTH); memcpy (sha1 , BUF+4 , SHA_DIGESTSIZE); memcpy (ed2k , BUF+4+SHA_DIGESTSIZE , MD4_DIGEST_LENGTH); memcpy (ed2k_list , BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH , MD4_DIGEST_LENGTH*nodes_ed2k ); memcpy (tiger , BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(nodes_ed2k+1) , TIGERSIZE); memcpy (&tiger_block, BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(nodes_ed2k+1)+TIGERSIZE, 4); size_t nodes_tiger = (file_size + tiger_block - 1) / tiger_block; tiger_list = (unsigned char*)malloc(nodes_tiger*TIGERSIZE); memcpy (tiger_list, BUF+4+SHA_DIGESTSIZE+MD4_DIGEST_LENGTH*(nodes_ed2k+1)+28, TIGERSIZE * nodes_tiger); unsigned char md4[MD4_DIGEST_LENGTH]; MD4_CTX md4_ctx; MD4Init (&md4_ctx); MD4Update(&md4_ctx, ed2k_list, MD4_DIGEST_LENGTH*nodes_ed2k); MD4Final (md4, &md4_ctx); if (0 == memcmp(md4, ed2k , MD4_DIGEST_LENGTH)) printf("ED2K OK\n"); char tree[TIGERSIZE]; tiger_compose((word64*)tiger_list, nodes_tiger, (word64*)tree); if (0 == memcmp(tree, tiger, TIGERSIZE)) printf("Tiger OK\n"); }}} int cTiger::Check (int fd, size_t start, size_t ende) {{{ int begin = (start + tiger_block - 1) / tiger_block; int end = ende / tiger_block; lseek (fd, begin * tiger_block, SEEK_SET); char buf[tiger_block]; char check[TIGERSIZE]; for (int i = begin; i < end; i++) { size_t done = read(fd, buf, tiger_block); tiger_tree_8k ((word64*)buf, done, (word64*)check); if (0 != memcmp(check, tiger_list + i * TIGERSIZE, TIGERSIZE)) return i * tiger_block; } return -1; }}} int main(int a, char **b) {{{ class cTiger Tiger; int fd = open (b[1], O_RDONLY); int load = atol(b[2]); if (!load) { Tiger.Build(fd); close(fd); } else { Tiger.Load(fd); close(fd); Tiger.Print(); if (a == 3) return 0; int fd = open (b[3], O_RDONLY); printf("Check %i\n", Tiger.Check(fd, 0, 37103)); close(fd); } return 0; }}}