From 85c62395b152f99e8867aaf84cea93dddc03243c Mon Sep 17 00:00:00 2001 From: David Barr Date: Sun, 28 Nov 2010 13:45:01 -0600 Subject: [PATCH] fast-import: let importers retrieve blobs New objects written by fast-import are not available immediately. Until a checkpoint has been started and finishes writing the pack index, any new blobs will not be accessible using standard git tools. So introduce a new way to access them: a "cat-blob" command in the command stream requests for fast-import to print a blob to stdout or a file descriptor specified by the argument to --cat-blob-fd. The value for cat-blob-fd cannot be specified in the stream because that would be a layering violation: the decision of where to direct a stream has to be made when fast-import is started anyway, so we might as well make the stream format is independent of that detail. Output uses the same format as "git cat-file --batch". Thanks to Sverre Rabbelier and Sam Vilain for guidance in designing the protocol. Based-on-patch-by: Jonathan Nieder Signed-off-by: David Barr Acked-by: Ramkumar Ramachandra Signed-off-by: Jonathan Nieder Signed-off-by: Junio C Hamano --- Documentation/git-fast-import.txt | 41 +++++++ fast-import.c | 97 +++++++++++++++ t/t9300-fast-import.sh | 193 +++++++++++++++++++++++++++++- 3 files changed, 329 insertions(+), 2 deletions(-) diff --git a/Documentation/git-fast-import.txt b/Documentation/git-fast-import.txt index ef86763948..5d8f60c790 100644 --- a/Documentation/git-fast-import.txt +++ b/Documentation/git-fast-import.txt @@ -92,6 +92,11 @@ OPTIONS --(no-)-relative-marks= with the --(import|export)-marks= options. +--cat-blob-fd=:: + Specify the file descriptor that will be written to + when the `cat-blob` command is encountered in the stream. + The default behaviour is to write to `stdout`. + --export-pack-edges=:: After creating a packfile, print a line of data to listing the filename of the packfile and the last @@ -320,6 +325,11 @@ and control the current import process. More detailed discussion standard output. This command is optional and is not needed to perform an import. +`cat-blob`:: + Causes fast-import to print a blob in 'cat-file --batch' + format to the file descriptor set with `--cat-blob-fd` or + `stdout` if unspecified. + `feature`:: Require that fast-import supports the specified feature, or abort if it does not. @@ -879,6 +889,29 @@ Placing a `progress` command immediately after a `checkpoint` will inform the reader when the `checkpoint` has been completed and it can safely access the refs that fast-import updated. +`cat-blob` +~~~~~~~~~~ +Causes fast-import to print a blob to a file descriptor previously +arranged with the `--cat-blob-fd` argument. The command otherwise +has no impact on the current import; its main purpose is to +retrieve blobs that may be in fast-import's memory but not +accessible from the target repository. + +.... + 'cat-blob' SP LF +.... + +The `` can be either a mark reference (`:`) +set previously or a full 40-byte SHA-1 of a Git blob, preexisting or +ready to be written. + +output uses the same format as `git cat-file --batch`: + +==== + SP 'blob' SP LF + LF +==== + `feature` ~~~~~~~~~ Require that fast-import supports the specified feature, or abort if @@ -905,6 +938,13 @@ import-marks:: second, an --import-marks= command-line option overrides any "feature import-marks" command in the stream. +cat-blob:: + Ignored. Versions of fast-import not supporting the + "cat-blob" command will exit with a message indicating so. + This lets the import error out early with a clear message, + rather than wasting time on the early part of an import + before the unsupported command is detected. + `option` ~~~~~~~~ Processes the specified option so that git fast-import behaves in a @@ -930,6 +970,7 @@ not be passed as option: * date-format * import-marks * export-marks +* cat-blob-fd * force Crash Reports diff --git a/fast-import.c b/fast-import.c index 4bd9bf7d08..dd58f517b9 100644 --- a/fast-import.c +++ b/fast-import.c @@ -55,6 +55,8 @@ Format of STDIN stream: ('from' sp committish lf)? lf?; + cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf; + checkpoint ::= 'checkpoint' lf lf?; @@ -361,6 +363,9 @@ static uintmax_t next_mark; static struct strbuf new_data = STRBUF_INIT; static int seen_data_command; +/* Where to write output of cat-blob commands */ +static int cat_blob_fd = STDOUT_FILENO; + static void parse_argv(void); static void write_branch_report(FILE *rpt, struct branch *b) @@ -2689,6 +2694,81 @@ static void parse_reset_branch(void) unread_command_buf = 1; } +static void cat_blob_write(const char *buf, unsigned long size) +{ + if (write_in_full(cat_blob_fd, buf, size) != size) + die_errno("Write to frontend failed"); +} + +static void cat_blob(struct object_entry *oe, unsigned char sha1[20]) +{ + struct strbuf line = STRBUF_INIT; + unsigned long size; + enum object_type type = 0; + char *buf; + + if (!oe || oe->pack_id == MAX_PACK_ID) { + buf = read_sha1_file(sha1, &type, &size); + } else { + type = oe->type; + buf = gfi_unpack_entry(oe, &size); + } + + /* + * Output based on batch_one_object() from cat-file.c. + */ + if (type <= 0) { + strbuf_reset(&line); + strbuf_addf(&line, "%s missing\n", sha1_to_hex(sha1)); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + free(buf); + return; + } + if (!buf) + die("Can't read object %s", sha1_to_hex(sha1)); + if (type != OBJ_BLOB) + die("Object %s is a %s but a blob was expected.", + sha1_to_hex(sha1), typename(type)); + strbuf_reset(&line); + strbuf_addf(&line, "%s %s %lu\n", sha1_to_hex(sha1), + typename(type), size); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + cat_blob_write(buf, size); + cat_blob_write("\n", 1); + free(buf); +} + +static void parse_cat_blob(void) +{ + const char *p; + struct object_entry *oe = oe; + unsigned char sha1[20]; + + /* cat-blob SP LF */ + p = command_buf.buf + strlen("cat-blob "); + if (*p == ':') { + char *x; + oe = find_mark(strtoumax(p + 1, &x, 10)); + if (x == p + 1) + die("Invalid mark: %s", command_buf.buf); + if (!oe) + die("Unknown mark: %s", command_buf.buf); + if (*x) + die("Garbage after mark: %s", command_buf.buf); + hashcpy(sha1, oe->idx.sha1); + } else { + if (get_sha1_hex(p, sha1)) + die("Invalid SHA1: %s", command_buf.buf); + if (p[40]) + die("Garbage after SHA1: %s", command_buf.buf); + oe = find_object(sha1); + } + + cat_blob(oe, sha1); +} + static void parse_checkpoint(void) { if (object_count) { @@ -2773,6 +2853,14 @@ static void option_export_marks(const char *marks) safe_create_leading_directories_const(export_marks_file); } +static void option_cat_blob_fd(const char *fd) +{ + unsigned long n = ulong_arg("--cat-blob-fd", fd); + if (n > (unsigned long) INT_MAX) + die("--cat-blob-fd cannot exceed %d", INT_MAX); + cat_blob_fd = (int) n; +} + static void option_export_pack_edges(const char *edges) { if (pack_edges) @@ -2826,6 +2914,8 @@ static int parse_one_feature(const char *feature, int from_stream) option_import_marks(feature + 13, from_stream); } else if (!prefixcmp(feature, "export-marks=")) { option_export_marks(feature + 13); + } else if (!strcmp(feature, "cat-blob")) { + ; /* Don't die - this feature is supported */ } else if (!prefixcmp(feature, "relative-marks")) { relative_marks_paths = 1; } else if (!prefixcmp(feature, "no-relative-marks")) { @@ -2920,6 +3010,11 @@ static void parse_argv(void) if (parse_one_feature(a + 2, 0)) continue; + if (!prefixcmp(a + 2, "cat-blob-fd=")) { + option_cat_blob_fd(a + 2 + strlen("cat-blob-fd=")); + continue; + } + die("unknown option %s", a); } if (i != global_argc) @@ -2971,6 +3066,8 @@ int main(int argc, const char **argv) parse_new_tag(); else if (!prefixcmp(command_buf.buf, "reset ")) parse_reset_branch(); + else if (!prefixcmp(command_buf.buf, "cat-blob ")) + parse_cat_blob(); else if (!strcmp("checkpoint", command_buf.buf)) parse_checkpoint(); else if (!prefixcmp(command_buf.buf, "progress ")) diff --git a/t/t9300-fast-import.sh b/t/t9300-fast-import.sh index c80bb0cf10..a0162b73d9 100755 --- a/t/t9300-fast-import.sh +++ b/t/t9300-fast-import.sh @@ -23,11 +23,18 @@ file5_data='an inline file. file6_data='#!/bin/sh echo "$@"' +>empty + ### ### series A ### test_tick + +test_expect_success 'empty stream succeeds' ' + git fast-import input <expect <<-EOF && + ${blob} blob 11 + yes it can + + EOF + echo "cat-blob $blob" | + git fast-import --cat-blob-fd=6 6>actual && + test_cmp expect actual +' + +test_expect_success 'R: in-stream cat-blob-fd not respected' ' + echo hello >greeting && + blob=$(git hash-object -w greeting) && + cat >expect <<-EOF && + ${blob} blob 6 + hello + + EOF + git fast-import --cat-blob-fd=3 3>actual.3 >actual.1 <<-EOF && + cat-blob $blob + EOF + test_cmp expect actual.3 && + test_cmp empty actual.1 && + git fast-import 3>actual.3 >actual.1 <<-EOF && + option cat-blob-fd=3 + cat-blob $blob + EOF + test_cmp empty actual.3 && + test_cmp expect actual.1 +' + +test_expect_success 'R: print new blob' ' + blob=$(echo "yep yep yep" | git hash-object --stdin) && + cat >expect <<-EOF && + ${blob} blob 12 + yep yep yep + + EOF + git fast-import --cat-blob-fd=6 6>actual <<-\EOF && + blob + mark :1 + data <expect <<-EOF && + ${blob} blob 25 + a new blob named by sha1 + + EOF + git fast-import --cat-blob-fd=6 6>actual <<-EOF && + blob + data <big && + for i in 1 2 3 + do + cat big big big big >bigger && + cat bigger bigger bigger bigger >big || + exit + done + ) +' + +test_expect_success 'R: print two blobs to stdout' ' + blob1=$(git hash-object big) && + blob1_len=$(wc -c expect && + { + cat <<-\END_PART1 && + blob + mark :1 + data <actual && + test_cmp expect actual +' + +test_expect_success 'setup: have pipes?' ' + rm -f frob && + if mkfifo frob + then + test_set_prereq PIPE + fi +' + +test_expect_success PIPE 'R: copy using cat-file' ' + expect_id=$(git hash-object big) && + expect_len=$(wc -c expect.response && + + rm -f blobs && + cat >frontend <<-\FRONTEND_END && + #!/bin/sh + cat <response && + dd if=/dev/stdin of=blob bs=$size count=1 <&3 && + read newline <&3 && + + cat < $GIT_COMMITTER_DATE + data <blobs + ) && + git show copied:file3 >actual && + test_cmp expect.response response && + test_cmp big actual +' + cat >input << EOF option git quiet blob @@ -1640,8 +1831,6 @@ hi EOF -touch empty - test_expect_success 'R: quiet option results in no stats being output' ' cat input | git fast-import 2> output && test_cmp empty output