From cac251d0bc4c68b7ab36026990aff3c783913ae6 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 21 Feb 2006 20:39:25 -0500 Subject: [PATCH 1/3] relax delta selection filtering in pack-objects This change provides a 8% saving on the pack size with a 4% CPU time increase for git-repack -a on the current git archive. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- pack-objects.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pack-objects.c b/pack-objects.c index 0c9f4c9d23..e3946db5ca 100644 --- a/pack-objects.c +++ b/pack-objects.c @@ -629,11 +629,10 @@ static int try_delta(struct unpacked *cur, struct unpacked *old, unsigned max_de } size = cur_entry->size; - if (size < 50) - return -1; oldsize = old_entry->size; sizediff = oldsize > size ? oldsize - size : size - oldsize; - if (sizediff > size / 8) + + if (size < 50) return -1; if (old_entry->depth >= max_depth) return 0; From fe474b588b3cb1c23c987a3d0f9e869a160d82d2 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 21 Feb 2006 20:41:41 -0500 Subject: [PATCH 2/3] diff-delta: fold two special tests into one plus cleanups Testing for realloc and size limit can be done with only one test per loop. Make it so and fix a theoretical off-by-one comparison error in the process. The output buffer memory allocation is also bounded by max_size when specified. Finally make some variable unsigned to allow the handling of files up to 4GB in size instead of 2GB. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- diff-delta.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/diff-delta.c b/diff-delta.c index c2f656ae39..ac992e21b8 100644 --- a/diff-delta.c +++ b/diff-delta.c @@ -148,16 +148,20 @@ static void delta_cleanup(bdfile_t *bdf) cha_free(&bdf->cha); } +/* provide the size of the copy opcode given the block offset and size */ #define COPYOP_SIZE(o, s) \ (!!(o & 0xff) + !!(o & 0xff00) + !!(o & 0xff0000) + !!(o & 0xff000000) + \ !!(s & 0xff) + !!(s & 0xff00) + 1) +/* the maximum size for any opcode */ +#define MAX_OP_SIZE COPYOP_SIZE(0xffffffff, 0xffffffff) + void *diff_delta(void *from_buf, unsigned long from_size, void *to_buf, unsigned long to_size, unsigned long *delta_size, unsigned long max_size) { - int i, outpos, outsize, inscnt, csize, msize, moff; + unsigned int i, outpos, outsize, inscnt, csize, msize, moff; unsigned int fp; const unsigned char *ref_data, *ref_top, *data, *top, *ptr1, *ptr2; unsigned char *out, *orig; @@ -169,6 +173,8 @@ void *diff_delta(void *from_buf, unsigned long from_size, outpos = 0; outsize = 8192; + if (max_size && outsize >= max_size) + outsize = max_size + MAX_OP_SIZE + 1; out = malloc(outsize); if (!out) { delta_cleanup(&bdf); @@ -259,17 +265,15 @@ void *diff_delta(void *from_buf, unsigned long from_size, *orig = i; } - if (max_size && outpos > max_size) { - free(out); - delta_cleanup(&bdf); - return NULL; - } - - /* next time around the largest possible output is 1 + 4 + 3 */ - if (outpos > outsize - 8) { + if (outpos >= outsize - MAX_OP_SIZE) { void *tmp = out; outsize = outsize * 3 / 2; - out = realloc(out, outsize); + if (max_size && outsize >= max_size) + outsize = max_size + MAX_OP_SIZE + 1; + if (max_size && outpos > max_size) + out = NULL; + else + out = realloc(out, outsize); if (!out) { free(tmp); delta_cleanup(&bdf); From 8e1454b5ad285ec5dd25758e799c589045aff9d4 Mon Sep 17 00:00:00 2001 From: Nicolas Pitre Date: Tue, 21 Feb 2006 20:43:17 -0500 Subject: [PATCH 3/3] diff-delta: big code simplification This is much smaller and hopefully clearer code now. Signed-off-by: Nicolas Pitre Signed-off-by: Junio C Hamano --- diff-delta.c | 231 +++++++++++++++++++-------------------------------- 1 file changed, 85 insertions(+), 146 deletions(-) diff --git a/diff-delta.c b/diff-delta.c index ac992e21b8..2ed5984b1c 100644 --- a/diff-delta.c +++ b/diff-delta.c @@ -19,8 +19,9 @@ */ #include +#include +#include #include "delta.h" -#include "zlib.h" /* block size: min = 16, max = 64k, power of 2 */ @@ -29,123 +30,54 @@ #define MIN(a, b) ((a) < (b) ? (a) : (b)) #define GR_PRIME 0x9e370001 -#define HASH(v, b) (((unsigned int)(v) * GR_PRIME) >> (32 - (b))) - -static unsigned int hashbits(unsigned int size) -{ - unsigned int val = 1, bits = 0; - while (val < size && bits < 32) { - val <<= 1; - bits++; - } - return bits ? bits: 1; -} - -typedef struct s_chanode { - struct s_chanode *next; - int icurr; -} chanode_t; - -typedef struct s_chastore { - int isize, nsize; - chanode_t *ancur; -} chastore_t; - -static void cha_init(chastore_t *cha, int isize, int icount) -{ - cha->isize = isize; - cha->nsize = icount * isize; - cha->ancur = NULL; -} - -static void *cha_alloc(chastore_t *cha) -{ - chanode_t *ancur; - void *data; - - ancur = cha->ancur; - if (!ancur || ancur->icurr == cha->nsize) { - ancur = malloc(sizeof(chanode_t) + cha->nsize); - if (!ancur) - return NULL; - ancur->icurr = 0; - ancur->next = cha->ancur; - cha->ancur = ancur; - } - - data = (void *)ancur + sizeof(chanode_t) + ancur->icurr; - ancur->icurr += cha->isize; - return data; -} - -static void cha_free(chastore_t *cha) -{ - chanode_t *cur = cha->ancur; - while (cur) { - chanode_t *tmp = cur; - cur = cur->next; - free(tmp); - } -} +#define HASH(v, shift) (((unsigned int)(v) * GR_PRIME) >> (shift)) -typedef struct s_bdrecord { - struct s_bdrecord *next; - unsigned int fp; +struct index { const unsigned char *ptr; -} bdrecord_t; + unsigned int val; + struct index *next; +}; -typedef struct s_bdfile { - chastore_t cha; - unsigned int fphbits; - bdrecord_t **fphash; -} bdfile_t; - -static int delta_prepare(const unsigned char *buf, int bufsize, bdfile_t *bdf) +static struct index ** delta_index(const unsigned char *buf, + unsigned long bufsize, + unsigned int *hash_shift) { - unsigned int fphbits; - int i, hsize; - const unsigned char *data, *top; - bdrecord_t *brec; - bdrecord_t **fphash; - - fphbits = hashbits(bufsize / BLK_SIZE + 1); - hsize = 1 << fphbits; - fphash = malloc(hsize * sizeof(bdrecord_t *)); - if (!fphash) - return -1; - for (i = 0; i < hsize; i++) - fphash[i] = NULL; - cha_init(&bdf->cha, sizeof(bdrecord_t), hsize / 4 + 1); - - top = buf + bufsize; - data = buf + (bufsize / BLK_SIZE) * BLK_SIZE; - if (data == top) + unsigned int hsize, hshift, entries, blksize, i; + const unsigned char *data; + struct index *entry, **hash; + void *mem; + + /* determine index hash size */ + entries = (bufsize + BLK_SIZE - 1) / BLK_SIZE; + hsize = entries / 4; + for (i = 4; (1 << i) < hsize && i < 16; i++); + hsize = 1 << i; + hshift = 32 - i; + *hash_shift = hshift; + + /* allocate lookup index */ + mem = malloc(hsize * sizeof(*hash) + entries * sizeof(*entry)); + if (!mem) + return NULL; + hash = mem; + entry = mem + hsize * sizeof(*hash); + memset(hash, 0, hsize * sizeof(*hash)); + + /* then populate it */ + data = buf + entries * BLK_SIZE - BLK_SIZE; + blksize = bufsize - (data - buf); + while (data >= buf) { + unsigned int val = adler32(0, data, blksize); + i = HASH(val, hshift); + entry->ptr = data; + entry->val = val; + entry->next = hash[i]; + hash[i] = entry++; + blksize = BLK_SIZE; data -= BLK_SIZE; + } - for ( ; data >= buf; data -= BLK_SIZE) { - brec = cha_alloc(&bdf->cha); - if (!brec) { - cha_free(&bdf->cha); - free(fphash); - return -1; - } - brec->fp = adler32(0, data, MIN(BLK_SIZE, top - data)); - brec->ptr = data; - i = HASH(brec->fp, fphbits); - brec->next = fphash[i]; - fphash[i] = brec; - } - - bdf->fphbits = fphbits; - bdf->fphash = fphash; - - return 0; -} - -static void delta_cleanup(bdfile_t *bdf) -{ - free(bdf->fphash); - cha_free(&bdf->cha); + return hash; } /* provide the size of the copy opcode given the block offset and size */ @@ -161,23 +93,24 @@ void *diff_delta(void *from_buf, unsigned long from_size, unsigned long *delta_size, unsigned long max_size) { - unsigned int i, outpos, outsize, inscnt, csize, msize, moff; - unsigned int fp; - const unsigned char *ref_data, *ref_top, *data, *top, *ptr1, *ptr2; - unsigned char *out, *orig; - bdrecord_t *brec; - bdfile_t bdf; + unsigned int i, outpos, outsize, inscnt, hash_shift; + const unsigned char *ref_data, *ref_top, *data, *top; + unsigned char *out; + struct index *entry, **hash; - if (!from_size || !to_size || delta_prepare(from_buf, from_size, &bdf)) + if (!from_size || !to_size) return NULL; - + hash = delta_index(from_buf, from_size, &hash_shift); + if (!hash) + return NULL; + outpos = 0; outsize = 8192; if (max_size && outsize >= max_size) outsize = max_size + MAX_OP_SIZE + 1; out = malloc(outsize); if (!out) { - delta_cleanup(&bdf); + free(hash); return NULL; } @@ -205,28 +138,32 @@ void *diff_delta(void *from_buf, unsigned long from_size, } inscnt = 0; - moff = 0; - while (data < top) { - msize = 0; - fp = adler32(0, data, MIN(top - data, BLK_SIZE)); - i = HASH(fp, bdf.fphbits); - for (brec = bdf.fphash[i]; brec; brec = brec->next) { - if (brec->fp == fp) { - csize = ref_top - brec->ptr; - if (csize > top - data) - csize = top - data; - for (ptr1 = brec->ptr, ptr2 = data; - csize && *ptr1 == *ptr2; - csize--, ptr1++, ptr2++); - csize = ptr1 - brec->ptr; - if (csize > msize) { - moff = brec->ptr - ref_data; - msize = csize; - if (msize >= 0x10000) { - msize = 0x10000; - break; - } + while (data < top) { + unsigned int moff = 0, msize = 0; + unsigned int blksize = MIN(top - data, BLK_SIZE); + unsigned int val = adler32(0, data, blksize); + i = HASH(val, hash_shift); + for (entry = hash[i]; entry; entry = entry->next) { + const unsigned char *ref = entry->ptr; + const unsigned char *src = data; + unsigned int ref_size = ref_top - ref; + if (entry->val != val) + continue; + if (ref_size > top - src) + ref_size = top - src; + while (ref_size && *src++ == *ref) { + ref++; + ref_size--; + } + ref_size = ref - entry->ptr; + if (ref_size > msize) { + /* this is our best match so far */ + moff = entry->ptr - ref_data; + msize = ref_size; + if (msize >= 0x10000) { + msize = 0x10000; + break; } } } @@ -241,13 +178,15 @@ void *diff_delta(void *from_buf, unsigned long from_size, inscnt = 0; } } else { + unsigned char *op; + if (inscnt) { out[outpos - inscnt - 1] = inscnt; inscnt = 0; } data += msize; - orig = out + outpos++; + op = out + outpos++; i = 0x80; if (moff & 0xff) { out[outpos++] = moff; i |= 0x01; } @@ -262,7 +201,7 @@ void *diff_delta(void *from_buf, unsigned long from_size, msize >>= 8; if (msize & 0xff) { out[outpos++] = msize; i |= 0x20; } - *orig = i; + *op = i; } if (outpos >= outsize - MAX_OP_SIZE) { @@ -276,7 +215,7 @@ void *diff_delta(void *from_buf, unsigned long from_size, out = realloc(out, outsize); if (!out) { free(tmp); - delta_cleanup(&bdf); + free(hash); return NULL; } } @@ -285,7 +224,7 @@ void *diff_delta(void *from_buf, unsigned long from_size, if (inscnt) out[outpos - inscnt - 1] = inscnt; - delta_cleanup(&bdf); + free(hash); *delta_size = outpos; return out; }