Browse Source

Merge branch 'np/index-pack'

* np/index-pack:
  remove .keep pack lock files when done with refs update
  have index-pack create .keep file more carefully
  improve fetch-pack's handling of kept packs
  git-fetch can use both --thin and --keep with fetch-pack now
  Teach receive-pack how to keep pack files based on object count.
  Allow pack header preprocessing before unpack-objects/index-pack.
  Remove unused variable in receive-pack.
  Revert "send-pack --keep: do not explode into loose objects on the receiving end."
  missing small substitution
  Teach git-index-pack how to keep a pack file.
  Only repack active packs by skipping over kept packs.
  Allow short pack names to git-pack-objects --unpacked=.
  send-pack --keep: do not explode into loose objects on the receiving end.
  index-pack: minor fixes to comment and function name
  enhance clone and fetch -k experience
  mimic unpack-objects when --stdin is used with index-pack
  add progress status to index-pack
  make index-pack able to complete thin packs.
  enable index-pack streaming capability
maint
Junio C Hamano 18 years ago
parent
commit
66f3b35fde
  1. 11
      Documentation/config.txt
  2. 3
      Documentation/git-fetch-pack.txt
  3. 52
      Documentation/git-index-pack.txt
  4. 2
      Makefile
  5. 15
      builtin-unpack-objects.c
  6. 5
      cache.h
  7. 276
      fetch-clone.c
  8. 112
      fetch-pack.c
  9. 14
      git-fetch.sh
  10. 27
      git-repack.sh
  11. 539
      index-pack.c
  12. 157
      receive-pack.c
  13. 22
      sha1_file.c

11
Documentation/config.txt

@ -301,7 +301,16 @@ imap:: @@ -301,7 +301,16 @@ imap::
The configuration variables in the 'imap' section are described
in gitlink:git-imap-send[1].

receive.denyNonFastforwads::
receive.unpackLimit::
If the number of objects received in a push is below this
limit then the objects will be unpacked into loose object
files. However if the number of received objects equals or
exceeds this limit then the received pack will be stored as
a pack, after adding any missing delta bases. Storing the
pack from a push can make the push operation complete faster,
especially on slow filesystems.

receive.denyNonFastForwards::
If set to true, git-receive-pack will deny a ref update which is
not a fast forward. Use this to prevent such an update via a push,
even if that push is forced. This configuration variable is

3
Documentation/git-fetch-pack.txt

@ -32,7 +32,8 @@ OPTIONS @@ -32,7 +32,8 @@ OPTIONS
-k::
Do not invoke 'git-unpack-objects' on received data, but
create a single packfile out of it instead, and store it
in the object database.
in the object database. If provided twice then the pack is
locked against repacking.

--exec=<git-upload-pack>::
Use this to specify the path to 'git-upload-pack' on the

52
Documentation/git-index-pack.txt

@ -8,7 +8,8 @@ git-index-pack - Build pack index file for an existing packed archive @@ -8,7 +8,8 @@ git-index-pack - Build pack index file for an existing packed archive

SYNOPSIS
--------
'git-index-pack' [-o <index-file>] <pack-file>
'git-index-pack' [-v] [-o <index-file>] <pack-file>
'git-index-pack' --stdin [--fix-thin] [--keep] [-v] [-o <index-file>] [<pack-file>]


DESCRIPTION
@ -21,6 +22,9 @@ objects/pack/ directory of a git repository. @@ -21,6 +22,9 @@ objects/pack/ directory of a git repository.

OPTIONS
-------
-v::
Be verbose about what is going on, including progress status.

-o <index-file>::
Write the generated pack index into the specified
file. Without this option the name of pack index
@ -29,6 +33,52 @@ OPTIONS @@ -29,6 +33,52 @@ OPTIONS
fails if the name of packed archive does not end
with .pack).

--stdin::
When this flag is provided, the pack is read from stdin
instead and a copy is then written to <pack-file>. If
<pack-file> is not specified, the pack is written to
objects/pack/ directory of the current git repository with
a default name determined from the pack content. If
<pack-file> is not specified consider using --keep to
prevent a race condition between this process and
gitlink::git-repack[1] .

--fix-thin::
It is possible for gitlink:git-pack-objects[1] to build
"thin" pack, which records objects in deltified form based on
objects not included in the pack to reduce network traffic.
Those objects are expected to be present on the receiving end
and they must be included in the pack for that pack to be self
contained and indexable. Without this option any attempt to
index a thin pack will fail. This option only makes sense in
conjunction with --stdin.

--keep::
Before moving the index into its final destination
create an empty .keep file for the associated pack file.
This option is usually necessary with --stdin to prevent a
simultaneous gitlink:git-repack[1] process from deleting
the newly constructed pack and index before refs can be
updated to use objects contained in the pack.

--keep='why'::
Like --keep create a .keep file before moving the index into
its final destination, but rather than creating an empty file
place 'why' followed by an LF into the .keep file. The 'why'
message can later be searched for within all .keep files to
locate any which have outlived their usefulness.


Note
----

Once the index has been created, the list of object names is sorted
and the SHA1 hash of that list is printed to stdout. If --stdin was
also used then this is prefixed by either "pack\t", or "keep\t" if a
new .keep file was successfully created. This is useful to remove a
.keep file used as a lock to prevent the race with gitlink:git-repack[1]
mentioned above.


Author
------

2
Makefile

@ -257,7 +257,7 @@ LIB_OBJS = \ @@ -257,7 +257,7 @@ LIB_OBJS = \
quote.o read-cache.o refs.o run-command.o dir.o object-refs.o \
server-info.o setup.o sha1_file.o sha1_name.o strbuf.o \
tag.o tree.o usage.o config.o environment.o ctype.o copy.o \
fetch-clone.o revision.o pager.o tree-walk.o xdiff-interface.o \
revision.o pager.o tree-walk.o xdiff-interface.o \
write_or_die.o trace.o list-objects.o grep.o \
alloc.o merge-file.o path-list.o help.o unpack-trees.o $(DIFF_OBJS) \
color.o wt-status.o archive-zip.o archive-tar.o

15
builtin-unpack-objects.c

@ -371,6 +371,21 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix) @@ -371,6 +371,21 @@ int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
recover = 1;
continue;
}
if (!strncmp(arg, "--pack_header=", 14)) {
struct pack_header *hdr;
char *c;

hdr = (struct pack_header *)buffer;
hdr->hdr_signature = htonl(PACK_SIGNATURE);
hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
if (*c != ',')
die("bad %s", arg);
hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
if (*c)
die("bad %s", arg);
len = sizeof(*hdr);
continue;
}
usage(unpack_usage);
}


5
cache.h

@ -376,6 +376,7 @@ extern struct packed_git *parse_pack_index_file(const unsigned char *sha1, @@ -376,6 +376,7 @@ extern struct packed_git *parse_pack_index_file(const unsigned char *sha1,
char *idx_path);

extern void prepare_packed_git(void);
extern void reprepare_packed_git(void);
extern void install_packed_git(struct packed_git *pack);

extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
@ -415,10 +416,6 @@ extern int copy_fd(int ifd, int ofd); @@ -415,10 +416,6 @@ extern int copy_fd(int ifd, int ofd);
extern void write_or_die(int fd, const void *buf, size_t count);
extern int write_or_whine(int fd, const void *buf, size_t count, const char *msg);

/* Finish off pack transfer receiving end */
extern int receive_unpack_pack(int fd[2], const char *me, int quiet, int);
extern int receive_keep_pack(int fd[2], const char *me, int quiet, int);

/* pager.c */
extern void setup_pager(void);
extern int pager_in_use;

276
fetch-clone.c

@ -1,276 +0,0 @@ @@ -1,276 +0,0 @@
#include "cache.h"
#include "exec_cmd.h"
#include "pkt-line.h"
#include "sideband.h"
#include <sys/wait.h>
#include <sys/time.h>

static int finish_pack(const char *pack_tmp_name, const char *me)
{
int pipe_fd[2];
pid_t pid;
char idx[PATH_MAX];
char final[PATH_MAX];
char hash[41];
unsigned char sha1[20];
char *cp;
int err = 0;

if (pipe(pipe_fd) < 0)
die("%s: unable to set up pipe", me);

strcpy(idx, pack_tmp_name); /* ".git/objects/pack-XXXXXX" */
cp = strrchr(idx, '/');
memcpy(cp, "/pidx", 5);

pid = fork();
if (pid < 0)
die("%s: unable to fork off git-index-pack", me);
if (!pid) {
close(0);
dup2(pipe_fd[1], 1);
close(pipe_fd[0]);
close(pipe_fd[1]);
execl_git_cmd("index-pack", "-o", idx, pack_tmp_name, NULL);
error("cannot exec git-index-pack <%s> <%s>",
idx, pack_tmp_name);
exit(1);
}
close(pipe_fd[1]);
if (read(pipe_fd[0], hash, 40) != 40) {
error("%s: unable to read from git-index-pack", me);
err = 1;
}
close(pipe_fd[0]);

for (;;) {
int status, code;

if (waitpid(pid, &status, 0) < 0) {
if (errno == EINTR)
continue;
error("waitpid failed (%s)", strerror(errno));
goto error_die;
}
if (WIFSIGNALED(status)) {
int sig = WTERMSIG(status);
error("git-index-pack died of signal %d", sig);
goto error_die;
}
if (!WIFEXITED(status)) {
error("git-index-pack died of unnatural causes %d",
status);
goto error_die;
}
code = WEXITSTATUS(status);
if (code) {
error("git-index-pack died with error code %d", code);
goto error_die;
}
if (err)
goto error_die;
break;
}
hash[40] = 0;
if (get_sha1_hex(hash, sha1)) {
error("git-index-pack reported nonsense '%s'", hash);
goto error_die;
}
/* Now we have pack in pack_tmp_name[], and
* idx in idx[]; rename them to their final names.
*/
snprintf(final, sizeof(final),
"%s/pack/pack-%s.pack", get_object_directory(), hash);
move_temp_to_file(pack_tmp_name, final);
chmod(final, 0444);
snprintf(final, sizeof(final),
"%s/pack/pack-%s.idx", get_object_directory(), hash);
move_temp_to_file(idx, final);
chmod(final, 0444);
return 0;

error_die:
unlink(idx);
unlink(pack_tmp_name);
exit(1);
}

static pid_t setup_sideband(int sideband, const char *me, int fd[2], int xd[2])
{
pid_t side_pid;

if (!sideband) {
fd[0] = xd[0];
fd[1] = xd[1];
return 0;
}
/* xd[] is talking with upload-pack; subprocess reads from
* xd[0], spits out band#2 to stderr, and feeds us band#1
* through our fd[0].
*/
if (pipe(fd) < 0)
die("%s: unable to set up pipe", me);
side_pid = fork();
if (side_pid < 0)
die("%s: unable to fork off sideband demultiplexer", me);
if (!side_pid) {
/* subprocess */
close(fd[0]);
if (xd[0] != xd[1])
close(xd[1]);
if (recv_sideband(me, xd[0], fd[1], 2))
exit(1);
exit(0);
}
close(xd[0]);
close(fd[1]);
fd[1] = xd[1];
return side_pid;
}

int receive_unpack_pack(int xd[2], const char *me, int quiet, int sideband)
{
int status;
pid_t pid, side_pid;
int fd[2];

side_pid = setup_sideband(sideband, me, fd, xd);
pid = fork();
if (pid < 0)
die("%s: unable to fork off git-unpack-objects", me);
if (!pid) {
dup2(fd[0], 0);
close(fd[0]);
close(fd[1]);
execl_git_cmd("unpack-objects", quiet ? "-q" : NULL, NULL);
die("git-unpack-objects exec failed");
}
close(fd[0]);
close(fd[1]);
while (waitpid(pid, &status, 0) < 0) {
if (errno != EINTR)
die("waiting for git-unpack-objects: %s",
strerror(errno));
}
if (WIFEXITED(status)) {
int code = WEXITSTATUS(status);
if (code)
die("git-unpack-objects died with error code %d",
code);
return 0;
}
if (WIFSIGNALED(status)) {
int sig = WTERMSIG(status);
die("git-unpack-objects died of signal %d", sig);
}
die("git-unpack-objects died of unnatural causes %d", status);
}

/*
* We average out the download speed over this many "events", where
* an event is a minimum of about half a second. That way, we get
* a reasonably stable number.
*/
#define NR_AVERAGE (4)

/*
* A "binary msec" is a power-of-two-msec, aka 1/1024th of a second.
* Keeping the time in that format means that "bytes / msecs" means
* the same as kB/s (modulo rounding).
*
* 1000512 is a magic number (usecs in a second, rounded up by half
* of 1024, to make "rounding" come out right ;)
*/
#define usec_to_binarymsec(x) ((int)(x) / (1000512 >> 10))

int receive_keep_pack(int xd[2], const char *me, int quiet, int sideband)
{
char tmpfile[PATH_MAX];
int ofd, ifd, fd[2];
unsigned long total;
static struct timeval prev_tv;
struct average {
unsigned long bytes;
unsigned long time;
} download[NR_AVERAGE] = { {0, 0}, };
unsigned long avg_bytes, avg_time;
int idx = 0;

setup_sideband(sideband, me, fd, xd);

ifd = fd[0];
snprintf(tmpfile, sizeof(tmpfile),
"%s/pack/tmp-XXXXXX", get_object_directory());
ofd = mkstemp(tmpfile);
if (ofd < 0)
return error("unable to create temporary file %s", tmpfile);

gettimeofday(&prev_tv, NULL);
total = 0;
avg_bytes = 0;
avg_time = 0;
while (1) {
char buf[8192];
ssize_t sz, wsz, pos;
sz = read(ifd, buf, sizeof(buf));
if (sz == 0)
break;
if (sz < 0) {
if (errno != EINTR && errno != EAGAIN) {
error("error reading pack (%s)", strerror(errno));
close(ofd);
unlink(tmpfile);
return -1;
}
sz = 0;
}
pos = 0;
while (pos < sz) {
wsz = write(ofd, buf + pos, sz - pos);
if (wsz < 0) {
error("error writing pack (%s)",
strerror(errno));
close(ofd);
unlink(tmpfile);
return -1;
}
pos += wsz;
}
total += sz;
if (!quiet) {
static unsigned long last;
struct timeval tv;
unsigned long diff = total - last;
/* not really "msecs", but a power-of-two millisec (1/1024th of a sec) */
unsigned long msecs;

gettimeofday(&tv, NULL);
msecs = tv.tv_sec - prev_tv.tv_sec;
msecs <<= 10;
msecs += usec_to_binarymsec(tv.tv_usec - prev_tv.tv_usec);

if (msecs > 500) {
prev_tv = tv;
last = total;

/* Update averages ..*/
avg_bytes += diff;
avg_time += msecs;
avg_bytes -= download[idx].bytes;
avg_time -= download[idx].time;
download[idx].bytes = diff;
download[idx].time = msecs;
idx++;
if (idx >= NR_AVERAGE)
idx = 0;

fprintf(stderr, "%4lu.%03luMB (%lu kB/s) \r",
total >> 20,
1000*((total >> 10) & 1023)>>10,
avg_bytes / avg_time );
}
}
}
close(ofd);
return finish_pack(tmpfile, me);
}

112
fetch-pack.c

@ -3,6 +3,9 @@ @@ -3,6 +3,9 @@
#include "pkt-line.h"
#include "commit.h"
#include "tag.h"
#include "exec_cmd.h"
#include "sideband.h"
#include <sys/wait.h>

static int keep_pack;
static int quiet;
@ -416,6 +419,103 @@ static int everything_local(struct ref **refs, int nr_match, char **match) @@ -416,6 +419,103 @@ static int everything_local(struct ref **refs, int nr_match, char **match)
return retval;
}

static pid_t setup_sideband(int fd[2], int xd[2])
{
pid_t side_pid;

if (!use_sideband) {
fd[0] = xd[0];
fd[1] = xd[1];
return 0;
}
/* xd[] is talking with upload-pack; subprocess reads from
* xd[0], spits out band#2 to stderr, and feeds us band#1
* through our fd[0].
*/
if (pipe(fd) < 0)
die("fetch-pack: unable to set up pipe");
side_pid = fork();
if (side_pid < 0)
die("fetch-pack: unable to fork off sideband demultiplexer");
if (!side_pid) {
/* subprocess */
close(fd[0]);
if (xd[0] != xd[1])
close(xd[1]);
if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
exit(1);
exit(0);
}
close(xd[0]);
close(fd[1]);
fd[1] = xd[1];
return side_pid;
}

static int get_pack(int xd[2], const char **argv)
{
int status;
pid_t pid, side_pid;
int fd[2];

side_pid = setup_sideband(fd, xd);
pid = fork();
if (pid < 0)
die("fetch-pack: unable to fork off %s", argv[0]);
if (!pid) {
dup2(fd[0], 0);
close(fd[0]);
close(fd[1]);
execv_git_cmd(argv);
die("%s exec failed", argv[0]);
}
close(fd[0]);
close(fd[1]);
while (waitpid(pid, &status, 0) < 0) {
if (errno != EINTR)
die("waiting for %s: %s", argv[0], strerror(errno));
}
if (WIFEXITED(status)) {
int code = WEXITSTATUS(status);
if (code)
die("%s died with error code %d", argv[0], code);
return 0;
}
if (WIFSIGNALED(status)) {
int sig = WTERMSIG(status);
die("%s died of signal %d", argv[0], sig);
}
die("%s died of unnatural causes %d", argv[0], status);
}

static int explode_rx_pack(int xd[2])
{
const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
return get_pack(xd, argv);
}

static int keep_rx_pack(int xd[2])
{
const char *argv[6];
char keep_arg[256];
int n = 0;

argv[n++] = "index-pack";
argv[n++] = "--stdin";
if (!quiet)
argv[n++] = "-v";
if (use_thin_pack)
argv[n++] = "--fix-thin";
if (keep_pack > 1) {
int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
strcpy(keep_arg + s, "localhost");
argv[n++] = keep_arg;
}
argv[n] = NULL;
return get_pack(xd, argv);
}

static int fetch_pack(int fd[2], int nr_match, char **match)
{
struct ref *ref;
@ -447,17 +547,13 @@ static int fetch_pack(int fd[2], int nr_match, char **match) @@ -447,17 +547,13 @@ static int fetch_pack(int fd[2], int nr_match, char **match)
goto all_done;
}
if (find_common(fd, sha1, ref) < 0)
if (!keep_pack)
if (keep_pack != 1)
/* When cloning, it is not unusual to have
* no common commit.
*/
fprintf(stderr, "warning: no common commits\n");

if (keep_pack)
status = receive_keep_pack(fd, "git-fetch-pack", quiet, use_sideband);
else
status = receive_unpack_pack(fd, "git-fetch-pack", quiet, use_sideband);

status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
if (status)
die("git-fetch-pack: fetch failed.");

@ -494,7 +590,7 @@ int main(int argc, char **argv) @@ -494,7 +590,7 @@ int main(int argc, char **argv)
continue;
}
if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
keep_pack = 1;
keep_pack++;
continue;
}
if (!strcmp("--thin", arg)) {
@ -518,8 +614,6 @@ int main(int argc, char **argv) @@ -518,8 +614,6 @@ int main(int argc, char **argv)
}
if (!dest)
usage(fetch_pack_usage);
if (keep_pack)
use_thin_pack = 0;
pid = git_connect(fd, dest, exec);
if (pid < 0)
return 1;

14
git-fetch.sh

@ -20,7 +20,7 @@ verbose= @@ -20,7 +20,7 @@ verbose=
update_head_ok=
exec=
upload_pack=
keep=--thin
keep=
while case "$#" in 0) break ;; esac
do
case "$1" in
@ -51,7 +51,7 @@ do @@ -51,7 +51,7 @@ do
verbose=Yes
;;
-k|--k|--ke|--kee|--keep)
keep=--keep
keep='-k -k'
;;
--reflog-action=*)
rloga=`expr "z$1" : 'z-[^=]*=\(.*\)'`
@ -368,9 +368,10 @@ fetch_main () { @@ -368,9 +368,10 @@ fetch_main () {
;; # we are already done.
*)
( : subshell because we muck with IFS
pack_lockfile=
IFS=" $LF"
(
git-fetch-pack $exec $keep "$remote" $rref || echo failed "$remote"
git-fetch-pack --thin $exec $keep "$remote" $rref || echo failed "$remote"
) |
while read sha1 remote_name
do
@ -378,6 +379,12 @@ fetch_main () { @@ -378,6 +379,12 @@ fetch_main () {
failed)
echo >&2 "Fetch failure: $remote"
exit 1 ;;
# special line coming from index-pack with the pack name
pack)
continue ;;
keep)
pack_lockfile="$GIT_OBJECT_DIRECTORY/pack/pack-$remote_name.keep"
continue ;;
esac
found=
single_force=
@ -408,6 +415,7 @@ fetch_main () { @@ -408,6 +415,7 @@ fetch_main () {
append_fetch_head "$sha1" "$remote" \
"$remote_name" "$remote_nick" "$local_name" "$not_for_merge"
done
if [ "$pack_lockfile" ]; then rm -f "$pack_lockfile"; fi
) || exit ;;
esac


27
git-repack.sh

@ -45,11 +45,19 @@ case ",$all_into_one," in @@ -45,11 +45,19 @@ case ",$all_into_one," in
args='--unpacked --incremental'
;;
,t,)
args=

# Redundancy check in all-into-one case is trivial.
existing=`test -d "$PACKDIR" && cd "$PACKDIR" && \
find . -type f \( -name '*.pack' -o -name '*.idx' \) -print`
if [ -d "$PACKDIR" ]; then
for e in `cd "$PACKDIR" && find . -type f -name '*.pack' \
| sed -e 's/^\.\///' -e 's/\.pack$//'`
do
if [ -e "$PACKDIR/$e.keep" ]; then
: keep
else
args="$args --unpacked=$e.pack"
existing="$existing $e"
fi
done
fi
[ -z "$args" ] && args='--unpacked --incremental'
;;
esac

@ -86,17 +94,16 @@ fi @@ -86,17 +94,16 @@ fi

if test "$remove_redundant" = t
then
# We know $existing are all redundant only when
# all-into-one is used.
if test "$all_into_one" != '' && test "$existing" != ''
# We know $existing are all redundant.
if [ -n "$existing" ]
then
sync
( cd "$PACKDIR" &&
for e in $existing
do
case "$e" in
./pack-$name.pack | ./pack-$name.idx) ;;
*) rm -f $e ;;
pack-$name) ;;
*) rm -f "$e.pack" "$e.idx" "$e.keep" ;;
esac
done
)

539
index-pack.c

@ -6,9 +6,11 @@ @@ -6,9 +6,11 @@
#include "commit.h"
#include "tag.h"
#include "tree.h"
#include <sys/time.h>
#include <signal.h>

static const char index_pack_usage[] =
"git-index-pack [-o index-file] pack-file";
"git-index-pack [-v] [-o <index-file>] [{ ---keep | --keep=<msg> }] { <pack-file> | --stdin [--fix-thin] [<pack-file>] }";

struct object_entry
{
@ -33,21 +35,71 @@ union delta_base { @@ -33,21 +35,71 @@ union delta_base {

struct delta_entry
{
struct object_entry *obj;
union delta_base base;
int obj_no;
};

static const char *pack_name;
static struct object_entry *objects;
static struct delta_entry *deltas;
static int nr_objects;
static int nr_deltas;
static int nr_resolved_deltas;

static int from_stdin;
static int verbose;

static volatile sig_atomic_t progress_update;

static void progress_interval(int signum)
{
progress_update = 1;
}

static void setup_progress_signal(void)
{
struct sigaction sa;
struct itimerval v;

memset(&sa, 0, sizeof(sa));
sa.sa_handler = progress_interval;
sigemptyset(&sa.sa_mask);
sa.sa_flags = SA_RESTART;
sigaction(SIGALRM, &sa, NULL);

v.it_interval.tv_sec = 1;
v.it_interval.tv_usec = 0;
v.it_value = v.it_interval;
setitimer(ITIMER_REAL, &v, NULL);

}

static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc)
{
unsigned percent = n * 100 / total;
if (percent != last_pc || progress_update) {
fprintf(stderr, "%4u%% (%u/%u) done\r", percent, n, total);
progress_update = 0;
}
return percent;
}

/* We always read in 4kB chunks. */
static unsigned char input_buffer[4096];
static unsigned long input_offset, input_len, consumed_bytes;
static SHA_CTX input_ctx;
static int input_fd;
static int input_fd, output_fd, mmap_fd;

/* Discard current buffer used content. */
static void flush()
{
if (input_offset) {
if (output_fd >= 0)
write_or_die(output_fd, input_buffer, input_offset);
SHA1_Update(&input_ctx, input_buffer, input_offset);
memcpy(input_buffer, input_buffer + input_offset, input_len);
input_offset = 0;
}
}

/*
* Make sure at least "min" bytes are available in the buffer, and
@ -59,11 +111,7 @@ static void *fill(int min) @@ -59,11 +111,7 @@ static void *fill(int min)
return input_buffer + input_offset;
if (min > sizeof(input_buffer))
die("cannot fill %d bytes", min);
if (input_offset) {
SHA1_Update(&input_ctx, input_buffer, input_offset);
memmove(input_buffer, input_buffer + input_offset, input_len);
input_offset = 0;
}
flush();
do {
int ret = xread(input_fd, input_buffer + input_len,
sizeof(input_buffer) - input_len);
@ -86,13 +134,31 @@ static void use(int bytes) @@ -86,13 +134,31 @@ static void use(int bytes)
consumed_bytes += bytes;
}

static void open_pack_file(void)
static const char *open_pack_file(const char *pack_name)
{
input_fd = open(pack_name, O_RDONLY);
if (input_fd < 0)
die("cannot open packfile '%s': %s", pack_name,
strerror(errno));
if (from_stdin) {
input_fd = 0;
if (!pack_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/pack_XXXXXX", get_object_directory());
output_fd = mkstemp(tmpfile);
pack_name = xstrdup(tmpfile);
} else
output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
if (output_fd < 0)
die("unable to create %s: %s\n", pack_name, strerror(errno));
mmap_fd = output_fd;
} else {
input_fd = open(pack_name, O_RDONLY);
if (input_fd < 0)
die("cannot open packfile '%s': %s",
pack_name, strerror(errno));
output_fd = -1;
mmap_fd = input_fd;
}
SHA1_Init(&input_ctx);
return pack_name;
}

static void parse_pack_header(void)
@ -101,14 +167,12 @@ static void parse_pack_header(void) @@ -101,14 +167,12 @@ static void parse_pack_header(void)

/* Header consistency check */
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
die("packfile '%s' signature mismatch", pack_name);
die("pack signature mismatch");
if (!pack_version_ok(hdr->hdr_version))
die("packfile '%s' version %d unsupported",
pack_name, ntohl(hdr->hdr_version));
die("pack version %d unsupported", ntohl(hdr->hdr_version));

nr_objects = ntohl(hdr->hdr_entries);
use(sizeof(struct pack_header));
/*fprintf(stderr, "Indexing %d objects\n", nr_objects);*/
}

static void bad_object(unsigned long offset, const char *format,
@ -122,8 +186,7 @@ static void bad_object(unsigned long offset, const char *format, ...) @@ -122,8 +186,7 @@ static void bad_object(unsigned long offset, const char *format, ...)
va_start(params, format);
vsnprintf(buf, sizeof(buf), format, params);
va_end(params);
die("packfile '%s': bad object at offset %lu: %s",
pack_name, offset, buf);
die("pack has bad object at offset %lu: %s", offset, buf);
}

static void *unpack_entry_data(unsigned long offset, unsigned long size)
@ -212,7 +275,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_ @@ -212,7 +275,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
return unpack_entry_data(obj->offset, obj->size);
}

static void * get_data_from_pack(struct object_entry *obj)
static void *get_data_from_pack(struct object_entry *obj)
{
unsigned long from = obj[0].offset + obj[0].hdr_size;
unsigned long len = obj[1].offset - from;
@ -222,9 +285,9 @@ static void * get_data_from_pack(struct object_entry *obj) @@ -222,9 +285,9 @@ static void * get_data_from_pack(struct object_entry *obj)
int st;

map = mmap(NULL, len + pg_offset, PROT_READ, MAP_PRIVATE,
input_fd, from - pg_offset);
mmap_fd, from - pg_offset);
if (map == MAP_FAILED)
die("cannot mmap packfile '%s': %s", pack_name, strerror(errno));
die("cannot mmap pack file: %s", strerror(errno));
data = xmalloc(obj->size);
memset(&stream, 0, sizeof(stream));
stream.next_out = data;
@ -261,8 +324,8 @@ static int find_delta(const union delta_base *base) @@ -261,8 +324,8 @@ static int find_delta(const union delta_base *base)
return -first-1;
}

static int find_delta_childs(const union delta_base *base,
int *first_index, int *last_index)
static int find_delta_children(const union delta_base *base,
int *first_index, int *last_index)
{
int first = find_delta(base);
int last = first;
@ -304,10 +367,9 @@ static void sha1_object(const void *data, unsigned long size, @@ -304,10 +367,9 @@ static void sha1_object(const void *data, unsigned long size,
SHA1_Final(sha1, &ctx);
}

static void resolve_delta(struct delta_entry *delta, void *base_data,
static void resolve_delta(struct object_entry *delta_obj, void *base_data,
unsigned long base_size, enum object_type type)
{
struct object_entry *obj = delta->obj;
void *delta_data;
unsigned long delta_size;
void *result;
@ -315,29 +377,34 @@ static void resolve_delta(struct delta_entry *delta, void *base_data, @@ -315,29 +377,34 @@ static void resolve_delta(struct delta_entry *delta, void *base_data,
union delta_base delta_base;
int j, first, last;

obj->real_type = type;
delta_data = get_data_from_pack(obj);
delta_size = obj->size;
delta_obj->real_type = type;
delta_data = get_data_from_pack(delta_obj);
delta_size = delta_obj->size;
result = patch_delta(base_data, base_size, delta_data, delta_size,
&result_size);
free(delta_data);
if (!result)
bad_object(obj->offset, "failed to apply delta");
sha1_object(result, result_size, type, obj->sha1);

hashcpy(delta_base.sha1, obj->sha1);
if (!find_delta_childs(&delta_base, &first, &last)) {
for (j = first; j <= last; j++)
if (deltas[j].obj->type == OBJ_REF_DELTA)
resolve_delta(&deltas[j], result, result_size, type);
bad_object(delta_obj->offset, "failed to apply delta");
sha1_object(result, result_size, type, delta_obj->sha1);
nr_resolved_deltas++;

hashcpy(delta_base.sha1, delta_obj->sha1);
if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
if (child->real_type == OBJ_REF_DELTA)
resolve_delta(child, result, result_size, type);
}
}

memset(&delta_base, 0, sizeof(delta_base));
delta_base.offset = obj->offset;
if (!find_delta_childs(&delta_base, &first, &last)) {
for (j = first; j <= last; j++)
if (deltas[j].obj->type == OBJ_OFS_DELTA)
resolve_delta(&deltas[j], result, result_size, type);
delta_base.offset = delta_obj->offset;
if (!find_delta_children(&delta_base, &first, &last)) {
for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
if (child->real_type == OBJ_OFS_DELTA)
resolve_delta(child, result, result_size, type);
}
}

free(result);
@ -353,7 +420,7 @@ static int compare_delta_entry(const void *a, const void *b) @@ -353,7 +420,7 @@ static int compare_delta_entry(const void *a, const void *b)
/* Parse all objects and return the pack content SHA1 hash */
static void parse_pack_objects(unsigned char *sha1)
{
int i;
int i, percent = -1;
struct delta_entry *delta = deltas;
void *data;
struct stat st;
@ -362,34 +429,43 @@ static void parse_pack_objects(unsigned char *sha1) @@ -362,34 +429,43 @@ static void parse_pack_objects(unsigned char *sha1)
* First pass:
* - find locations of all objects;
* - calculate SHA1 of all non-delta objects;
* - remember base SHA1 for all deltas.
* - remember base (SHA1 or offset) for all deltas.
*/
if (verbose)
fprintf(stderr, "Indexing %d objects.\n", nr_objects);
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
data = unpack_raw_entry(obj, &delta->base);
obj->real_type = obj->type;
if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA) {
nr_deltas++;
delta->obj = obj;
delta->obj_no = i;
delta++;
} else
sha1_object(data, obj->size, obj->type, obj->sha1);
free(data);
if (verbose)
percent = display_progress(i+1, nr_objects, percent);
}
objects[i].offset = consumed_bytes;
if (verbose)
fputc('\n', stderr);

/* Check pack integrity */
SHA1_Update(&input_ctx, input_buffer, input_offset);
flush();
SHA1_Final(sha1, &input_ctx);
if (hashcmp(fill(20), sha1))
die("packfile '%s' SHA1 mismatch", pack_name);
die("pack is corrupted (SHA1 mismatch)");
use(20);

/* If input_fd is a file, we should have reached its end now. */
if (fstat(input_fd, &st))
die("cannot fstat packfile '%s': %s", pack_name, strerror(errno));
die("cannot fstat packfile: %s", strerror(errno));
if (S_ISREG(st.st_mode) && st.st_size != consumed_bytes)
die("packfile '%s' has junk at the end", pack_name);
die("pack has junk at the end");

if (!nr_deltas)
return;

/* Sort deltas by base SHA1/offset for fast searching */
qsort(deltas, nr_deltas, sizeof(struct delta_entry),
@ -403,6 +479,8 @@ static void parse_pack_objects(unsigned char *sha1) @@ -403,6 +479,8 @@ static void parse_pack_objects(unsigned char *sha1)
* recursively checking if the resulting object is used as a base
* for some more deltas.
*/
if (verbose)
fprintf(stderr, "Resolving %d deltas.\n", nr_deltas);
for (i = 0; i < nr_objects; i++) {
struct object_entry *obj = &objects[i];
union delta_base base;
@ -411,32 +489,179 @@ static void parse_pack_objects(unsigned char *sha1) @@ -411,32 +489,179 @@ static void parse_pack_objects(unsigned char *sha1)
if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
continue;
hashcpy(base.sha1, obj->sha1);
ref = !find_delta_childs(&base, &ref_first, &ref_last);
ref = !find_delta_children(&base, &ref_first, &ref_last);
memset(&base, 0, sizeof(base));
base.offset = obj->offset;
ofs = !find_delta_childs(&base, &ofs_first, &ofs_last);
ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
if (!ref && !ofs)
continue;
data = get_data_from_pack(obj);
if (ref)
for (j = ref_first; j <= ref_last; j++)
if (deltas[j].obj->type == OBJ_REF_DELTA)
resolve_delta(&deltas[j], data,
for (j = ref_first; j <= ref_last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
if (child->real_type == OBJ_REF_DELTA)
resolve_delta(child, data,
obj->size, obj->type);
}
if (ofs)
for (j = ofs_first; j <= ofs_last; j++)
if (deltas[j].obj->type == OBJ_OFS_DELTA)
resolve_delta(&deltas[j], data,
for (j = ofs_first; j <= ofs_last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
if (child->real_type == OBJ_OFS_DELTA)
resolve_delta(child, data,
obj->size, obj->type);
}
free(data);
if (verbose)
percent = display_progress(nr_resolved_deltas,
nr_deltas, percent);
}
if (verbose && nr_resolved_deltas == nr_deltas)
fputc('\n', stderr);
}

static int write_compressed(int fd, void *in, unsigned int size)
{
z_stream stream;
unsigned long maxsize;
void *out;

memset(&stream, 0, sizeof(stream));
deflateInit(&stream, zlib_compression_level);
maxsize = deflateBound(&stream, size);
out = xmalloc(maxsize);

/* Compress it */
stream.next_in = in;
stream.avail_in = size;
stream.next_out = out;
stream.avail_out = maxsize;
while (deflate(&stream, Z_FINISH) == Z_OK);
deflateEnd(&stream);

size = stream.total_out;
write_or_die(fd, out, size);
free(out);
return size;
}

static void append_obj_to_pack(void *buf,
unsigned long size, enum object_type type)
{
struct object_entry *obj = &objects[nr_objects++];
unsigned char header[10];
unsigned long s = size;
int n = 0;
unsigned char c = (type << 4) | (s & 15);
s >>= 4;
while (s) {
header[n++] = c | 0x80;
c = s & 0x7f;
s >>= 7;
}
header[n++] = c;
write_or_die(output_fd, header, n);
obj[1].offset = obj[0].offset + n;
obj[1].offset += write_compressed(output_fd, buf, size);
sha1_object(buf, size, type, obj->sha1);
}

static int delta_pos_compare(const void *_a, const void *_b)
{
struct delta_entry *a = *(struct delta_entry **)_a;
struct delta_entry *b = *(struct delta_entry **)_b;
return a->obj_no - b->obj_no;
}

static void fix_unresolved_deltas(int nr_unresolved)
{
struct delta_entry **sorted_by_pos;
int i, n = 0, percent = -1;

/* Check for unresolved deltas */
/*
* Since many unresolved deltas may well be themselves base objects
* for more unresolved deltas, we really want to include the
* smallest number of base objects that would cover as much delta
* as possible by picking the
* trunc deltas first, allowing for other deltas to resolve without
* additional base objects. Since most base objects are to be found
* before deltas depending on them, a good heuristic is to start
* resolving deltas in the same order as their position in the pack.
*/
sorted_by_pos = xmalloc(nr_unresolved * sizeof(*sorted_by_pos));
for (i = 0; i < nr_deltas; i++) {
if (deltas[i].obj->real_type == OBJ_REF_DELTA ||
deltas[i].obj->real_type == OBJ_OFS_DELTA)
die("packfile '%s' has unresolved deltas", pack_name);
if (objects[deltas[i].obj_no].real_type != OBJ_REF_DELTA)
continue;
sorted_by_pos[n++] = &deltas[i];
}
qsort(sorted_by_pos, n, sizeof(*sorted_by_pos), delta_pos_compare);

for (i = 0; i < n; i++) {
struct delta_entry *d = sorted_by_pos[i];
void *data;
unsigned long size;
char type[10];
enum object_type obj_type;
int j, first, last;

if (objects[d->obj_no].real_type != OBJ_REF_DELTA)
continue;
data = read_sha1_file(d->base.sha1, type, &size);
if (!data)
continue;
if (!strcmp(type, blob_type)) obj_type = OBJ_BLOB;
else if (!strcmp(type, tree_type)) obj_type = OBJ_TREE;
else if (!strcmp(type, commit_type)) obj_type = OBJ_COMMIT;
else if (!strcmp(type, tag_type)) obj_type = OBJ_TAG;
else die("base object %s is of type '%s'",
sha1_to_hex(d->base.sha1), type);

find_delta_children(&d->base, &first, &last);
for (j = first; j <= last; j++) {
struct object_entry *child = objects + deltas[j].obj_no;
if (child->real_type == OBJ_REF_DELTA)
resolve_delta(child, data, size, obj_type);
}

append_obj_to_pack(data, size, obj_type);
free(data);
if (verbose)
percent = display_progress(nr_resolved_deltas,
nr_deltas, percent);
}
free(sorted_by_pos);
if (verbose)
fputc('\n', stderr);
}

static void readjust_pack_header_and_sha1(unsigned char *sha1)
{
struct pack_header hdr;
SHA_CTX ctx;
int size;

/* Rewrite pack header with updated object number */
if (lseek(output_fd, 0, SEEK_SET) != 0)
die("cannot seek back: %s", strerror(errno));
if (xread(output_fd, &hdr, sizeof(hdr)) != sizeof(hdr))
die("cannot read pack header back: %s", strerror(errno));
hdr.hdr_entries = htonl(nr_objects);
if (lseek(output_fd, 0, SEEK_SET) != 0)
die("cannot seek back: %s", strerror(errno));
write_or_die(output_fd, &hdr, sizeof(hdr));
if (lseek(output_fd, 0, SEEK_SET) != 0)
die("cannot seek back: %s", strerror(errno));

/* Recompute and store the new pack's SHA1 */
SHA1_Init(&ctx);
do {
unsigned char *buf[4096];
size = xread(output_fd, buf, sizeof(buf));
if (size < 0)
die("cannot read pack data back: %s", strerror(errno));
SHA1_Update(&ctx, buf, size);
} while (size > 0);
SHA1_Final(sha1, &ctx);
write_or_die(output_fd, sha1, 20);
}

static int sha1_compare(const void *_a, const void *_b)
@ -450,12 +675,12 @@ static int sha1_compare(const void *_a, const void *_b) @@ -450,12 +675,12 @@ static int sha1_compare(const void *_a, const void *_b)
* On entry *sha1 contains the pack content SHA1 hash, on exit it is
* the SHA1 hash of sorted object names.
*/
static void write_index_file(const char *index_name, unsigned char *sha1)
static const char *write_index_file(const char *index_name, unsigned char *sha1)
{
struct sha1file *f;
struct object_entry **sorted_by_sha, **list, **last;
unsigned int array[256];
int i;
int i, fd;
SHA_CTX ctx;

if (nr_objects) {
@ -472,8 +697,19 @@ static void write_index_file(const char *index_name, unsigned char *sha1) @@ -472,8 +697,19 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
else
sorted_by_sha = list = last = NULL;

unlink(index_name);
f = sha1create("%s", index_name);
if (!index_name) {
static char tmpfile[PATH_MAX];
snprintf(tmpfile, sizeof(tmpfile),
"%s/index_XXXXXX", get_object_directory());
fd = mkstemp(tmpfile);
index_name = xstrdup(tmpfile);
} else {
unlink(index_name);
fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
}
if (fd < 0)
die("unable to create %s: %s", index_name, strerror(errno));
f = sha1fd(fd, index_name);

/*
* Write the first-level table (the list is sorted,
@ -513,20 +749,128 @@ static void write_index_file(const char *index_name, unsigned char *sha1) @@ -513,20 +749,128 @@ static void write_index_file(const char *index_name, unsigned char *sha1)
sha1close(f, NULL, 1);
free(sorted_by_sha);
SHA1_Final(sha1, &ctx);
return index_name;
}

static void final(const char *final_pack_name, const char *curr_pack_name,
const char *final_index_name, const char *curr_index_name,
const char *keep_name, const char *keep_msg,
unsigned char *sha1)
{
char *report = "pack";
char name[PATH_MAX];
int err;

if (!from_stdin) {
close(input_fd);
} else {
err = close(output_fd);
if (err)
die("error while closing pack file: %s", strerror(errno));
chmod(curr_pack_name, 0444);
}

if (keep_msg) {
int keep_fd, keep_msg_len = strlen(keep_msg);
if (!keep_name) {
snprintf(name, sizeof(name), "%s/pack/pack-%s.keep",
get_object_directory(), sha1_to_hex(sha1));
keep_name = name;
}
keep_fd = open(keep_name, O_RDWR|O_CREAT|O_EXCL, 0600);
if (keep_fd < 0) {
if (errno != EEXIST)
die("cannot write keep file");
} else {
if (keep_msg_len > 0) {
write_or_die(keep_fd, keep_msg, keep_msg_len);
write_or_die(keep_fd, "\n", 1);
}
close(keep_fd);
report = "keep";
}
}

if (final_pack_name != curr_pack_name) {
if (!final_pack_name) {
snprintf(name, sizeof(name), "%s/pack/pack-%s.pack",
get_object_directory(), sha1_to_hex(sha1));
final_pack_name = name;
}
if (move_temp_to_file(curr_pack_name, final_pack_name))
die("cannot store pack file");
}

chmod(curr_index_name, 0444);
if (final_index_name != curr_index_name) {
if (!final_index_name) {
snprintf(name, sizeof(name), "%s/pack/pack-%s.idx",
get_object_directory(), sha1_to_hex(sha1));
final_index_name = name;
}
if (move_temp_to_file(curr_index_name, final_index_name))
die("cannot store index file");
}

if (!from_stdin) {
printf("%s\n", sha1_to_hex(sha1));
} else {
char buf[48];
int len = snprintf(buf, sizeof(buf), "%s\t%s\n",
report, sha1_to_hex(sha1));
xwrite(1, buf, len);

/*
* Let's just mimic git-unpack-objects here and write
* the last part of the input buffer to stdout.
*/
while (input_len) {
err = xwrite(1, input_buffer + input_offset, input_len);
if (err <= 0)
break;
input_len -= err;
input_offset += err;
}
}
}

int main(int argc, char **argv)
{
int i;
char *index_name = NULL;
char *index_name_buf = NULL;
int i, fix_thin_pack = 0;
const char *curr_pack, *pack_name = NULL;
const char *curr_index, *index_name = NULL;
const char *keep_name = NULL, *keep_msg = NULL;
char *index_name_buf = NULL, *keep_name_buf = NULL;
unsigned char sha1[20];

for (i = 1; i < argc; i++) {
const char *arg = argv[i];

if (*arg == '-') {
if (!strcmp(arg, "-o")) {
if (!strcmp(arg, "--stdin")) {
from_stdin = 1;
} else if (!strcmp(arg, "--fix-thin")) {
fix_thin_pack = 1;
} else if (!strcmp(arg, "--keep")) {
keep_msg = "";
} else if (!strncmp(arg, "--keep=", 7)) {
keep_msg = arg + 7;
} else if (!strncmp(arg, "--pack_header=", 14)) {
struct pack_header *hdr;
char *c;

hdr = (struct pack_header *)input_buffer;
hdr->hdr_signature = htonl(PACK_SIGNATURE);
hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
if (*c != ',')
die("bad %s", arg);
hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
if (*c)
die("bad %s", arg);
input_len = sizeof(*hdr);
} else if (!strcmp(arg, "-v")) {
verbose = 1;
} else if (!strcmp(arg, "-o")) {
if (index_name || (i+1) >= argc)
usage(index_pack_usage);
index_name = argv[++i];
@ -540,9 +884,11 @@ int main(int argc, char **argv) @@ -540,9 +884,11 @@ int main(int argc, char **argv)
pack_name = arg;
}

if (!pack_name)
if (!pack_name && !from_stdin)
usage(index_pack_usage);
if (!index_name) {
if (fix_thin_pack && !from_stdin)
die("--fix-thin cannot be used without --stdin");
if (!index_name && pack_name) {
int len = strlen(pack_name);
if (!has_extension(pack_name, ".pack"))
die("packfile name '%s' does not end with '.pack'",
@ -552,18 +898,55 @@ int main(int argc, char **argv) @@ -552,18 +898,55 @@ int main(int argc, char **argv)
strcpy(index_name_buf + len - 5, ".idx");
index_name = index_name_buf;
}
if (keep_msg && !keep_name && pack_name) {
int len = strlen(pack_name);
if (!has_extension(pack_name, ".pack"))
die("packfile name '%s' does not end with '.pack'",
pack_name);
keep_name_buf = xmalloc(len);
memcpy(keep_name_buf, pack_name, len - 5);
strcpy(keep_name_buf + len - 5, ".keep");
keep_name = keep_name_buf;
}

open_pack_file();
curr_pack = open_pack_file(pack_name);
parse_pack_header();
objects = xcalloc(nr_objects + 1, sizeof(struct object_entry));
deltas = xcalloc(nr_objects, sizeof(struct delta_entry));
objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry));
deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
if (verbose)
setup_progress_signal();
parse_pack_objects(sha1);
if (nr_deltas != nr_resolved_deltas) {
if (fix_thin_pack) {
int nr_unresolved = nr_deltas - nr_resolved_deltas;
int nr_objects_initial = nr_objects;
if (nr_unresolved <= 0)
die("confusion beyond insanity");
objects = xrealloc(objects,
(nr_objects + nr_unresolved + 1)
* sizeof(*objects));
fix_unresolved_deltas(nr_unresolved);
if (verbose)
fprintf(stderr, "%d objects were added to complete this thin pack.\n",
nr_objects - nr_objects_initial);
readjust_pack_header_and_sha1(sha1);
}
if (nr_deltas != nr_resolved_deltas)
die("pack has %d unresolved deltas",
nr_deltas - nr_resolved_deltas);
} else {
/* Flush remaining pack final 20-byte SHA1. */
flush();
}
free(deltas);
write_index_file(index_name, sha1);
curr_index = write_index_file(index_name, sha1);
final(pack_name, curr_pack,
index_name, curr_index,
keep_name, keep_msg,
sha1);
free(objects);
free(index_name_buf);

printf("%s\n", sha1_to_hex(sha1));
free(keep_name_buf);

return 0;
}

157
receive-pack.c

@ -1,15 +1,17 @@ @@ -1,15 +1,17 @@
#include "cache.h"
#include "pack.h"
#include "refs.h"
#include "pkt-line.h"
#include "run-command.h"
#include "exec_cmd.h"
#include "commit.h"
#include "object.h"
#include <sys/wait.h>

static const char receive_pack_usage[] = "git-receive-pack <git-dir>";

static const char *unpacker[] = { "unpack-objects", NULL };

static int deny_non_fast_forwards = 0;
static int unpack_limit = 5000;
static int report_status;

static char capabilities[] = "report-status";
@ -25,6 +27,12 @@ static int receive_pack_config(const char *var, const char *value) @@ -25,6 +27,12 @@ static int receive_pack_config(const char *var, const char *value)
return 0;
}

if (strcmp(var, "receive.unpacklimit") == 0)
{
unpack_limit = git_config_int(var, value);
return 0;
}

return 0;
}

@ -227,29 +235,127 @@ static void read_head_info(void) @@ -227,29 +235,127 @@ static void read_head_info(void)
}
}

static const char *unpack(int *error_code)
static const char *parse_pack_header(struct pack_header *hdr)
{
int code = run_command_v_opt(1, unpacker, RUN_GIT_CMD);
char *c = (char*)hdr;
ssize_t remaining = sizeof(struct pack_header);
do {
ssize_t r = xread(0, c, remaining);
if (r <= 0)
return "eof before pack header was fully read";
remaining -= r;
c += r;
} while (remaining > 0);
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
return "protocol error (pack signature mismatch detected)";
if (!pack_version_ok(hdr->hdr_version))
return "protocol error (pack version unsupported)";
return NULL;
}

*error_code = 0;
switch (code) {
case 0:
return NULL;
case -ERR_RUN_COMMAND_FORK:
return "unpack fork failed";
case -ERR_RUN_COMMAND_EXEC:
return "unpack execute failed";
case -ERR_RUN_COMMAND_WAITPID:
return "waitpid failed";
case -ERR_RUN_COMMAND_WAITPID_WRONG_PID:
return "waitpid is confused";
case -ERR_RUN_COMMAND_WAITPID_SIGNAL:
return "unpacker died of signal";
case -ERR_RUN_COMMAND_WAITPID_NOEXIT:
return "unpacker died strangely";
default:
*error_code = -code;
return "unpacker exited with error code";
static const char *pack_lockfile;

static const char *unpack(void)
{
struct pack_header hdr;
const char *hdr_err;
char hdr_arg[38];

hdr_err = parse_pack_header(&hdr);
if (hdr_err)
return hdr_err;
snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
ntohl(hdr.hdr_version), ntohl(hdr.hdr_entries));

if (ntohl(hdr.hdr_entries) < unpack_limit) {
int code;
const char *unpacker[3];
unpacker[0] = "unpack-objects";
unpacker[1] = hdr_arg;
unpacker[2] = NULL;
code = run_command_v_opt(1, unpacker, RUN_GIT_CMD);
switch (code) {
case 0:
return NULL;
case -ERR_RUN_COMMAND_FORK:
return "unpack fork failed";
case -ERR_RUN_COMMAND_EXEC:
return "unpack execute failed";
case -ERR_RUN_COMMAND_WAITPID:
return "waitpid failed";
case -ERR_RUN_COMMAND_WAITPID_WRONG_PID:
return "waitpid is confused";
case -ERR_RUN_COMMAND_WAITPID_SIGNAL:
return "unpacker died of signal";
case -ERR_RUN_COMMAND_WAITPID_NOEXIT:
return "unpacker died strangely";
default:
return "unpacker exited with error code";
}
} else {
const char *keeper[6];
int fd[2], s, len, status;
pid_t pid;
char keep_arg[256];
char packname[46];

s = sprintf(keep_arg, "--keep=receive-pack %i on ", getpid());
if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
strcpy(keep_arg + s, "localhost");

keeper[0] = "index-pack";
keeper[1] = "--stdin";
keeper[2] = "--fix-thin";
keeper[3] = hdr_arg;
keeper[4] = keep_arg;
keeper[5] = NULL;

if (pipe(fd) < 0)
return "index-pack pipe failed";
pid = fork();
if (pid < 0)
return "index-pack fork failed";
if (!pid) {
dup2(fd[1], 1);
close(fd[1]);
close(fd[0]);
execv_git_cmd(keeper);
die("execv of index-pack failed");
}
close(fd[1]);

/*
* The first thing we expects from index-pack's output
* is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
* %40s is the newly created pack SHA1 name. In the "keep"
* case, we need it to remove the corresponding .keep file
* later on. If we don't get that then tough luck with it.
*/
for (len = 0;
len < 46 && (s = xread(fd[0], packname+len, 46-len)) > 0;
len += s);
close(fd[0]);
if (len == 46 && packname[45] == '\n' &&
memcmp(packname, "keep\t", 5) == 0) {
char path[PATH_MAX];
packname[45] = 0;
snprintf(path, sizeof(path), "%s/pack/pack-%s.keep",
get_object_directory(), packname + 5);
pack_lockfile = xstrdup(path);
}

/* Then wrap our index-pack process. */
while (waitpid(pid, &status, 0) < 0)
if (errno != EINTR)
return "waitpid failed";
if (WIFEXITED(status)) {
int code = WEXITSTATUS(status);
if (code)
return "index-pack exited with error code";
reprepare_packed_git();
return NULL;
}
return "index-pack abnormal exit";
}
}

@ -302,10 +408,11 @@ int main(int argc, char **argv) @@ -302,10 +408,11 @@ int main(int argc, char **argv)

read_head_info();
if (commands) {
int code;
const char *unpack_status = unpack(&code);
const char *unpack_status = unpack();
if (!unpack_status)
execute_commands();
if (pack_lockfile)
unlink(pack_lockfile);
if (report_status)
report(unpack_status);
}

22
sha1_file.c

@ -663,7 +663,7 @@ void prepare_packed_git(void) @@ -663,7 +663,7 @@ void prepare_packed_git(void)
prepare_packed_git_run_once = 1;
}

static void reprepare_packed_git(void)
void reprepare_packed_git(void)
{
prepare_packed_git_run_once = 0;
prepare_packed_git();
@ -1203,6 +1203,24 @@ unsigned long find_pack_entry_one(const unsigned char *sha1, @@ -1203,6 +1203,24 @@ unsigned long find_pack_entry_one(const unsigned char *sha1,
return 0;
}

static int matches_pack_name(struct packed_git *p, const char *ig)
{
const char *last_c, *c;

if (!strcmp(p->pack_name, ig))
return 0;

for (c = p->pack_name, last_c = c; *c;)
if (*c == '/')
last_c = ++c;
else
++c;
if (!strcmp(last_c, ig))
return 0;

return 1;
}

static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
{
struct packed_git *p;
@ -1214,7 +1232,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, cons @@ -1214,7 +1232,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, cons
if (ignore_packed) {
const char **ig;
for (ig = ignore_packed; *ig; ig++)
if (!strcmp(p->pack_name, *ig))
if (!matches_pack_name(p, *ig))
break;
if (*ig)
continue;

Loading…
Cancel
Save