You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
279 lines
11 KiB
279 lines
11 KiB
From eea6b49210edf69682b2d0606bee17bbccb3765b Mon Sep 17 00:00:00 2001 |
|
From: Dmitry Monakhov <dmonakhov@openvz.org> |
|
Date: Fri, 30 Oct 2015 22:04:46 +0000 |
|
Subject: [PATCH] copy: fix copying of extents beyond the apparent file size |
|
|
|
fallocate can allocate extents beyond EOF via FALLOC_FL_KEEP_SIZE. |
|
Where there is a gap (hole) between the extents, and EOF is within |
|
that gap, the final hole wasn't reproduced, resulting in silent |
|
data corruption in the copied file (size too small). |
|
|
|
* src/copy.c (extent_copy): Ensure we don't process extents |
|
beyond the apparent file size, since processing and allocating |
|
those is not currently supported. |
|
* tests/cp/fiemap-extents.sh: Renamed from tests/cp/fiemap-empty.sh |
|
and re-enable parts checking the extents at and beyond EOF. |
|
* tests/local.mk: Reference the renamed test. |
|
Fixes http://bugs.gnu.org/21790 |
|
--- |
|
src/copy.c | 19 ++++++++- |
|
tests/cp/fiemap-empty.sh | 102 -------------------------------------------- |
|
tests/cp/fiemap-extents.sh | 81 +++++++++++++++++++++++++++++++++++ |
|
tests/local.mk | 2 +- |
|
5 files changed, 100 insertions(+), 104 deletions(-) |
|
delete mode 100755 tests/cp/fiemap-empty.sh |
|
create mode 100755 tests/cp/fiemap-extents.sh |
|
|
|
diff --git a/src/copy.c b/src/copy.c |
|
index dc1cd29..6771bb5 100644 |
|
--- a/src/copy.c |
|
+++ b/src/copy.c |
|
@@ -432,6 +432,20 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size, |
|
ext_len = 0; |
|
} |
|
|
|
+ /* Truncate extent to EOF. Extents starting after EOF are |
|
+ treated as zero length extents starting right after EOF. |
|
+ Generally this will trigger with an extent starting after |
|
+ src_total_size, and result in creating a hole or zeros until EOF. |
|
+ Though in a file in which extents have changed since src_total_size |
|
+ was determined, we might have an extent spanning that size, |
|
+ in which case we'll only copy data up to that size. */ |
|
+ if (src_total_size < ext_start + ext_len) |
|
+ { |
|
+ if (src_total_size < ext_start) |
|
+ ext_start = src_total_size; |
|
+ ext_len = src_total_size - ext_start; |
|
+ } |
|
+ |
|
hole_size = ext_start - last_ext_start - last_ext_len; |
|
|
|
wrote_hole_at_eof = false; |
|
@@ -495,14 +509,17 @@ extent_copy (int src_fd, int dest_fd, char *buf, size_t buf_size, |
|
off_t n_read; |
|
empty_extent = false; |
|
last_ext_len = ext_len; |
|
+ bool read_hole; |
|
|
|
if ( ! sparse_copy (src_fd, dest_fd, buf, buf_size, |
|
sparse_mode == SPARSE_ALWAYS, |
|
src_name, dst_name, ext_len, &n_read, |
|
- &wrote_hole_at_eof)) |
|
+ &read_hole)) |
|
goto fail; |
|
|
|
dest_pos = ext_start + n_read; |
|
+ if (n_read) |
|
+ wrote_hole_at_eof = read_hole; |
|
} |
|
|
|
/* If the file ends with unwritten extents not accounted for in the |
|
deleted file mode 100755 |
|
index b3b2cd7..0000000 |
|
--- a/tests/cp/fiemap-empty.sh |
|
+++ /dev/null |
|
@@ -1,101 +0,0 @@ |
|
-#!/bin/sh |
|
-# Test cp reads unwritten extents efficiently |
|
- |
|
-# Copyright (C) 2011-2013 Free Software Foundation, Inc. |
|
- |
|
-# This program is free software: you can redistribute it and/or modify |
|
-# it under the terms of the GNU General Public License as published by |
|
-# the Free Software Foundation, either version 3 of the License, or |
|
-# (at your option) any later version. |
|
- |
|
-# This program is distributed in the hope that it will be useful, |
|
-# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
-# GNU General Public License for more details. |
|
- |
|
-# You should have received a copy of the GNU General Public License |
|
-# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
- |
|
-. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src |
|
-print_ver_ cp |
|
- |
|
-# FIXME: enable any part of this test that is still relevant, |
|
-# or, if none are relevant (now that cp does not handle unwritten |
|
-# extents), just remove the test altogether. |
|
-skip_ 'disabled for now' |
|
- |
|
-touch fiemap_chk |
|
-fiemap_capable_ fiemap_chk || |
|
- skip_ 'this file system lacks FIEMAP support' |
|
-rm fiemap_chk |
|
- |
|
-# TODO: rather than requiring $(fallocate), possible add |
|
-# this functionality to truncate --alloc |
|
-fallocate --help >/dev/null || skip_ 'The fallocate utility is required' |
|
-fallocate -l 1 -n falloc.test || |
|
- skip_ 'this file system lacks FALLOCATE support' |
|
-rm falloc.test |
|
- |
|
-# Require more space than we'll actually use, so that |
|
-# tests run in parallel do not run out of space. |
|
-# Otherwise, with inadequate space, simply running the following |
|
-# fallocate command would induce a temporary disk-full condition, |
|
-# which would cause failure of unrelated tests run in parallel. |
|
-require_file_system_bytes_free_ 800000000 |
|
- |
|
-fallocate -l 600MiB space.test || |
|
- skip_ 'this test needs at least 600MiB free space' |
|
- |
|
-# Disable this test on old BTRFS (e.g. Fedora 14) |
|
-# which reports ordinary extents for unwritten ones. |
|
-filefrag space.test || skip_ 'the 'filefrag' utility is missing' |
|
-filefrag -v space.test | grep -F 'unwritten' > /dev/null || |
|
- skip_ 'this file system does not report empty extents as "unwritten"' |
|
- |
|
-rm space.test |
|
- |
|
-# Ensure we read a large empty file quickly |
|
-fallocate -l 300MiB empty.big || framework_failure_ |
|
-timeout 3 cp --sparse=always empty.big cp.test || fail=1 |
|
-test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1 |
|
-rm empty.big cp.test |
|
- |
|
-# Ensure we handle extents beyond file size correctly. |
|
-# Note until we support fallocate, we will not maintain |
|
-# the file allocation. FIXME: amend this test when fallocate is supported. |
|
-fallocate -l 10MiB -n unwritten.withdata || framework_failure_ |
|
-dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unwritten.withdata |
|
-cp unwritten.withdata cp.test || fail=1 |
|
-test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1 |
|
-cmp unwritten.withdata cp.test || fail=1 |
|
-rm unwritten.withdata cp.test |
|
- |
|
-# The following to generate unaccounted extents followed by a hole, is not |
|
-# supported by ext4 at least. The ftruncate discards all extents not |
|
-# accounted for in the size. |
|
-# fallocate -l 10MiB -n unacc.withholes |
|
-# dd count=10 if=/dev/urandom conv=notrunc iflag=fullblock of=unacc.withholes |
|
-# truncate -s20M unacc.withholes |
|
- |
|
-# Ensure we handle a hole after empty extents correctly. |
|
-# Since all extents are accounted for in the size, |
|
-# we can maintain the allocation independently from |
|
-# fallocate() support. |
|
-fallocate -l 10MiB empty.withholes |
|
-truncate -s 20M empty.withholes |
|
-sectors_per_block=$(expr $(stat -c %o .) / 512) |
|
-cp empty.withholes cp.test || fail=1 |
|
-test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1 |
|
-# These are usually equal but can vary by an IO block due to alignment |
|
-alloc_diff=$(expr $(stat -c %b empty.withholes) - $(stat -c %b cp.test)) |
|
-alloc_diff=$(echo $alloc_diff | tr -d -- -) # abs() |
|
-test $alloc_diff -le $sectors_per_block || fail=1 |
|
-# Again with SPARSE_ALWAYS |
|
-cp --sparse=always empty.withholes cp.test || fail=1 |
|
-test $(stat -c %s empty.withholes) = $(stat -c %s cp.test) || fail=1 |
|
-# cp.test should take 0 space, but allowing for some systems |
|
-# that store default extended attributes in data blocks |
|
-test $(stat -c %b cp.test) -le $sectors_per_block || fail=1 |
|
-rm empty.withholes cp.test |
|
- |
|
-Exit $fail |
|
diff --git a/tests/cp/fiemap-extents.sh b/tests/cp/fiemap-extents.sh |
|
new file mode 100755 |
|
index 0000000..55ec5df |
|
--- /dev/null |
|
+++ b/tests/cp/fiemap-extents.sh |
|
@@ -0,0 +1,81 @@ |
|
+#!/bin/sh |
|
+# Test cp handles extents correctly |
|
+ |
|
+# Copyright (C) 2011-2015 Free Software Foundation, Inc. |
|
+ |
|
+# This program is free software: you can redistribute it and/or modify |
|
+# it under the terms of the GNU General Public License as published by |
|
+# the Free Software Foundation, either version 3 of the License, or |
|
+# (at your option) any later version. |
|
+ |
|
+# This program is distributed in the hope that it will be useful, |
|
+# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
+# GNU General Public License for more details. |
|
+ |
|
+# You should have received a copy of the GNU General Public License |
|
+# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
+ |
|
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src |
|
+print_ver_ cp |
|
+ |
|
+require_sparse_support_ |
|
+ |
|
+touch fiemap_chk |
|
+fiemap_capable_ fiemap_chk || |
|
+ skip_ 'this file system lacks FIEMAP support' |
|
+rm fiemap_chk |
|
+ |
|
+fallocate --help >/dev/null || skip_ 'The fallocate utility is required' |
|
+touch falloc.test || framework_failure_ |
|
+fallocate -l 1 -o 0 -n falloc.test || |
|
+ skip_ 'this file system lacks FALLOCATE support' |
|
+rm falloc.test |
|
+ |
|
+# We don't currently handle unwritten extents specially |
|
+if false; then |
|
+# Require more space than we'll actually use, so that |
|
+# tests run in parallel do not run out of space. |
|
+# Otherwise, with inadequate space, simply running the following |
|
+# fallocate command would induce a temporary disk-full condition, |
|
+# which would cause failure of unrelated tests run in parallel. |
|
+require_file_system_bytes_free_ 800000000 |
|
+ |
|
+fallocate -l 600MiB space.test || |
|
+ skip_ 'this test needs at least 600MiB free space' |
|
+ |
|
+# Disable this test on old BTRFS (e.g. Fedora 14) |
|
+# which reports ordinary extents for unwritten ones. |
|
+filefrag space.test || skip_ 'the 'filefrag' utility is missing' |
|
+filefrag -v space.test | grep -F 'unwritten' > /dev/null || |
|
+ skip_ 'this file system does not report empty extents as "unwritten"' |
|
+ |
|
+rm space.test |
|
+ |
|
+# Ensure we read a large empty file quickly |
|
+fallocate -l 300MiB empty.big || framework_failure_ |
|
+timeout 3 cp --sparse=always empty.big cp.test || fail=1 |
|
+test $(stat -c %s empty.big) = $(stat -c %s cp.test) || fail=1 |
|
+rm empty.big cp.test |
|
+fi |
|
+ |
|
+# Ensure we handle extents beyond file size correctly. |
|
+# Note until we support fallocate, we will not maintain |
|
+# the file allocation. FIXME: amend this test if fallocate is supported. |
|
+# Note currently this only uses fiemap logic when the allocation (-l) |
|
+# is smaller than the size, thus identifying the file as sparse. |
|
+# Note the '-l 1' case is an effective noop, and just checks |
|
+# a file with a trailing hole is copied correctly. |
|
+for sparse_mode in always auto never; do |
|
+ for alloc in '-l 4MiB ' '-l 1MiB -o 4MiB' '-l 1'; do |
|
+ dd count=10 if=/dev/urandom iflag=fullblock of=unwritten.withdata |
|
+ truncate -s 2MiB unwritten.withdata || framework_failure_ |
|
+ fallocate $alloc -n unwritten.withdata || framework_failure_ |
|
+ cp --sparse=$sparse_mode unwritten.withdata cp.test || fail=1 |
|
+ test $(stat -c %s unwritten.withdata) = $(stat -c %s cp.test) || fail=1 |
|
+ cmp unwritten.withdata cp.test || fail=1 |
|
+ rm unwritten.withdata cp.test |
|
+ done |
|
+done |
|
+ |
|
+Exit $fail |
|
diff --git a/tests/local.mk b/tests/local.mk |
|
index adf96f0..89fdbb0 100644 |
|
--- a/tests/local.mk |
|
+++ b/tests/local.mk |
|
@@ -446,7 +446,7 @@ all_tests = \ |
|
tests/cp/existing-perm-dir.sh \ |
|
tests/cp/existing-perm-race.sh \ |
|
tests/cp/fail-perm.sh \ |
|
- tests/cp/fiemap-empty.sh \ |
|
+ tests/cp/fiemap-extents.sh \ |
|
tests/cp/fiemap-FMR.sh \ |
|
tests/cp/fiemap-perf.sh \ |
|
tests/cp/fiemap-2.sh \ |
|
-- |
|
1.7.2.5 |
|
|
|
|