From 77cef4ee39ee083838e5cf2137b0f344b49afb6a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 29 Jan 2013 10:39:30 +0100 Subject: [PATCH 1/4] Fix bug in sparse file listing List posix archives containing sparse files >8GB correctly and do not fail. This fixes also bug in format of listing for sparse files >8GB - now the real size is printed instead of the effective one (this is not strictly posix format related). * src/list.c: Remove redundant assignment. * src/tar.h: Add new 'real_size' and 'real_size_set' fields in tar_stat_info struct. * src/xheader.c: Correctly handle (especially sparse) file sizes directly in xheader_decode(). --- src/list.c | 1 - src/tar.h | 4 ++++ src/xheader.c | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/list.c b/src/list.c index dd501a9..6db36d1 100644 --- a/src/list.c +++ b/src/list.c @@ -670,7 +670,6 @@ decode_header (union block *header, struct tar_stat_info *stat_info, } } - stat_info->archive_file_size = stat_info->stat.st_size; xheader_decode (stat_info); if (sparse_member_p (stat_info)) diff --git a/src/tar.h b/src/tar.h index b181e58..690c146 100644 --- a/src/tar.h +++ b/src/tar.h @@ -327,6 +327,10 @@ struct tar_stat_info size_t sparse_map_size; /* Size of the sparse map */ struct sp_array *sparse_map; + off_t real_size; /* The real size of sparse file */ + int real_size_set; /* True when GNU.sparse.realsize is set in + archived file */ + size_t xattr_map_size; /* Size of the xattr map */ struct xattr_array *xattr_map; diff --git a/src/xheader.c b/src/xheader.c index be793d4..708aece 100644 --- a/src/xheader.c +++ b/src/xheader.c @@ -764,6 +764,16 @@ xheader_decode (struct tar_stat_info *st) continue; } run_override_list (keyword_override_list, st); + + /* The archived (effective) file size is always set directly in tar header + field, possibly overridden by "size" extended header - in both cases, + result is now decoded in st->stat.st_size */ + st->archive_file_size = st->stat.st_size; + + /* The real file size (given by stat()) may be redefined for sparse + files in "GNU.sparse.realsize" extended header */ + if (st->real_size_set) + st->stat.st_size = st->real_size; } static void @@ -1438,7 +1435,10 @@ sparse_size_decoder (struct tar_stat_info *st, { uintmax_t u; if (decode_num (&u, arg, TYPE_MAXIMUM (off_t), keyword)) - st->stat.st_size = u; + { + st->real_size_set = 1; + st->real_size = u; + } } static void -- 2.13.5 From 198d4621b2a367986c71cd2489c1465ec3be89dc Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Fri, 7 Nov 2014 11:47:44 +0200 Subject: [PATCH 2/4] Add testcase for the previous commit. * tests/sparse05.at: New file. * tests/Makefile.am: Add sparse05.at * tests/testsuite.at: Include sparse05.at --- tests/Makefile.am | 1 + tests/sparse05.at | 46 ++++++++++++++++++++++++++++++++++++++++++++++ tests/testsuite.at | 1 + 3 files changed, 48 insertions(+) create mode 100644 tests/sparse05.at diff --git a/tests/Makefile.am b/tests/Makefile.am index 8e1ef8d..094b71c 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -171,6 +171,7 @@ TESTSUITE_AT = \ sparse02.at\ sparse03.at\ sparse04.at\ + sparse05.at\ sparsemv.at\ sparsemvp.at\ spmvp00.at\ diff --git a/tests/sparse05.at b/tests/sparse05.at new file mode 100644 index 0000000..72f3274 --- /dev/null +++ b/tests/sparse05.at @@ -0,0 +1,46 @@ +# Process this file with autom4te to create testsuite. -*- Autotest -*- +# +# Test suite for GNU tar. +# Copyright 2014 Free Software Foundation, Inc. + +# This file is part of GNU tar. + +# GNU tar is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. + +# GNU tar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +AT_SETUP([listing sparse files bigger than 2^33 B]) +AT_KEYWORDS([sparse sparse05]) + +# Description: If an archive in POSIX.1-2001 archive contained a sparse file +# member whose real size (excluding zero blocks) is bigger than 2^33 bytes, +# tar 1.28 would incorrectly list the real member size. +# Reported by: Pavel Raiskup +# References: <1359119879.15037.4.camel@raiskup>, +# http://lists.gnu.org/archive/html/bug-tar/2013-01/msg00001.html + +AT_TAR_CHECK([ +(echo 0 =2560 +for i in `seq 1 999`; do + echo 10M =2560 +done) | genfile --sparse --file BIGFILE --block-size 4K - || AT_SKIP_TEST +tar -f - -c --sparse --posix BIGFILE | tar tvf - | awk '{ print $3, $(NF) }' +], +[0], +[20961034240 BIGFILE +], +[], +[], +[], +[pax]) + +AT_CLEANUP diff --git a/tests/testsuite.at b/tests/testsuite.at index 3eb0eee..f7f00ee 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -308,6 +308,7 @@ m4_include([sparse01.at]) m4_include([sparse02.at]) m4_include([sparse03.at]) m4_include([sparse04.at]) +m4_include([sparse05.at]) m4_include([sparsemv.at]) m4_include([spmvp00.at]) m4_include([spmvp01.at]) -- 2.13.5 From 8a795036f08bca508c3f3425f41c566562573e5f Mon Sep 17 00:00:00 2001 From: Sergey Poznyakoff Date: Fri, 7 Nov 2014 11:37:33 +0200 Subject: [PATCH 3/4] genfile: improve sparse mode Paxutils: 45af1632aa64a5ba1b108e248920e67c180e8485 * tests/genfile.c (generate_sparse_file): Handle '-' argument (read from stdin); If content strings starts with '=', treat it as fragment size and use default pattern to fill it. * doc/genfile.texi: Document changes to genfile. --- doc/genfile.texi | 31 +++++++++++++++------- tests/genfile.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 19 deletions(-) diff --git a/doc/genfile.texi b/doc/genfile.texi index b37e26e..b993b9b 100644 --- a/doc/genfile.texi +++ b/doc/genfile.texi @@ -124,9 +124,8 @@ the rest of the command line specifies a so-called @dfn{file map}. descriptors}. Each descriptor is composed of two values: a number, specifying fragment offset from the end of the previous fragment or, for the very first fragment, from the beginning of the file, and -@dfn{contents string}, i.e., a string of characters, specifying the -pattern to fill the fragment with. File offset can be suffixed with -the following quantifiers: +@dfn{contents string}, that specifies the pattern to fill the fragment +with. File offset can be suffixed with the following quantifiers: @table @samp @item k @@ -140,17 +139,29 @@ The number is expressed in megabytes. The number is expressed in gigabytes. @end table - For each letter in contents string @command{genfile} will generate -a @dfn{block} of data, filled with this letter and will write it to -the fragment. The size of block is given by @option{--block-size} -option. It defaults to 512. Thus, if the string consists of @var{n} -characters, the resulting file fragment will contain -@code{@var{n}*@var{block-size}} of data. + Contents string can be either a fragment size or a pattern. +Fragment size is a decimal number, prefixed with an equals sign. It +can be suffixed with a quantifier, as discussed above. If fragment +size is given, the fragment of that size will be filled with the +currently selected pattern (@pxref{Generate Mode, --pattern}) and +written to the file. - Last fragment descriptor can have only file offset part. In this + A pattern is a string of arbitrary ASCII characters. For each +of them, @command{genfile} will generate a @dfn{block} of data, +filled with that character and will write it to the fragment. The size +of block is given by @option{--block-size} option. It defaults to 512. +Thus, if pattern consists of @var{n} characters, the resulting file +fragment will contain @code{@var{n}*@var{block-size}} bytes of data. + + The last fragment descriptor can have only file offset part. In this case @command{genfile} will create a hole at the end of the file up to the given offset. + A dash appearing as a fragment descriptor instructs +@command{genfile} to read file map from the standard input. Each line +of input should consist of fragment offset and contents string, +separated by any amount of whitespace. + For example, consider the following invocation: @smallexample diff --git a/tests/genfile.c b/tests/genfile.c index fa480ef..d41336b 100644 --- a/tests/genfile.c +++ b/tests/genfile.c @@ -32,6 +32,7 @@ #include #include #include +#include #define obstack_chunk_alloc malloc #define obstack_chunk_free free #include @@ -506,6 +507,53 @@ mksparse (int fd, off_t displ, char *marks) } } +static int +make_fragment (int fd, char *offstr, char *mapstr) +{ + int i; + off_t displ = get_size (offstr, 1); + + file_length += displ; + + if (!mapstr || !*mapstr) + { + mkhole (fd, displ); + return 1; + } + else if (*mapstr == '=') + { + off_t n = get_size (mapstr + 1, 1); + + switch (pattern) + { + case DEFAULT_PATTERN: + for (i = 0; i < block_size; i++) + buffer[i] = i & 255; + break; + + case ZEROS_PATTERN: + memset (buffer, 0, block_size); + break; + } + + if (lseek (fd, displ, SEEK_CUR) == -1) + error (EXIT_FAILURE, errno, "lseek"); + + for (; n; n--) + { + if (write (fd, buffer, block_size) != block_size) + error (EXIT_FAILURE, errno, "write"); + file_length += block_size; + } + } + else + { + file_length += block_size * strlen (mapstr); + mksparse (fd, displ, mapstr); + } + return 0; +} + static void generate_sparse_file (int argc, char **argv) { @@ -526,20 +574,33 @@ generate_sparse_file (int argc, char **argv) file_length = 0; - for (i = 0; i < argc; i += 2) + while (argc) { - off_t displ = get_size (argv[i], 1); - file_length += displ; + if (argv[0][0] == '-' && argv[0][1] == 0) + { + char buf[256]; + while (fgets (buf, sizeof (buf), stdin)) + { + size_t n = strlen (buf); - if (i == argc-1) - { - mkhole (fd, displ); - break; + while (n > 0 && c_isspace (buf[n-1])) + buf[--n] = 0; + + n = strcspn (buf, " \t"); + buf[n++] = 0; + while (buf[n] && c_isblank (buf[n])) + ++n; + make_fragment (fd, buf, buf + n); + } + ++argv; + --argc; } else { - file_length += block_size * strlen (argv[i+1]); - mksparse (fd, displ, argv[i+1]); + if (make_fragment (fd, argv[0], argv[1])) + break; + argc -= 2; + argv += 2; } } -- 2.13.5 From 29e35df407d6c7b1e1ff57f7ef2030a253132a8a Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Fri, 4 Dec 2015 19:36:14 +0100 Subject: [PATCH 4/4] genfile: remove unused variable paxutils: 58b8ac114790e2de7992db1a387ec14238783f39 * tests/genfile.c (generate_sparse_file): Remove unused 'i'. --- tests/genfile.c | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/genfile.c b/tests/genfile.c index d41336b..4699d21 100644 --- a/tests/genfile.c +++ b/tests/genfile.c @@ -557,7 +557,6 @@ make_fragment (int fd, char *offstr, char *mapstr) static void generate_sparse_file (int argc, char **argv) { - int i; int fd; int flags = O_CREAT | O_RDWR | O_BINARY; -- 2.13.5