From ec76ec82566f0696472d22c3f8c2cd095c0a3e64 Mon Sep 17 00:00:00 2001 From: Dakota Williams Date: Thu, 28 Apr 2022 11:30:37 -0400 Subject: [PATCH 1/3] Add support for non-resident cow files Cow files can now exist off of the disk it is cow-ing. This comes with a few limitations: 1. You cannot unmount a device with an active cow file on it. You would have to destroy or unmount the device being snapshotted by that cow file. DattoBD will hold the file handle to the cow file and will prevent the unmount. 3. Don't create loops with the snapshotted devices. If device A's cow file exists on device B, and device B's cow file is on device A, then you wouldn't be able to unmount either device or shutdown the machine. You'd have to destroy either snapshot to be able to unmount. This commit also adds some fields to the /proc/datto-info device. The `resident` field is a 0/1 false/true value that answers the question of whether the cow file is on the device it is tracking. The `full_cow_path` field represents the full path to the cow device, rather than `cow_file`'s relative-to-mountpoint path. You are encouraged to use the new `full_cow_path` field; `cow_file` will be kept, but deprecated, for backwards compatibility. --- src/bio_helper.c | 8 +++++-- src/bio_helper.h | 2 +- src/proc_seq_file.c | 5 ++++- src/snap_device.h | 7 ++++++- src/system_call_hooking.c | 13 ++++-------- src/tracer.c | 44 ++++++++++++++++++++++++++------------- 6 files changed, 51 insertions(+), 28 deletions(-) diff --git a/src/bio_helper.c b/src/bio_helper.c index 4738bd33..0c33569e 100644 --- a/src/bio_helper.c +++ b/src/bio_helper.c @@ -325,11 +325,15 @@ struct inode *page_get_inode(struct page *pg) return pg->mapping->host; } -int bio_needs_cow(struct bio *bio, struct inode *inode) +int bio_needs_cow(struct bio *bio, struct snap_device *dev) { bio_iter_t iter; bio_iter_bvec_t bvec; + if (!test_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags)) { + return 1; // if the cow is non-resident, then we don't need to check if the bio is for the cow file. + } + #ifdef HAVE_ENUM_REQ_OPF //#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) if (bio_op(bio) == REQ_OP_WRITE_ZEROES) @@ -339,7 +343,7 @@ int bio_needs_cow(struct bio *bio, struct inode *inode) // check the inode of each page return true if it does not match our cow // file bio_for_each_segment (bvec, bio, iter) { - if (page_get_inode(bio_iter_page(bio, iter)) != inode) + if (page_get_inode(bio_iter_page(bio, iter)) != dev->sd_cow_inode) return 1; } diff --git a/src/bio_helper.h b/src/bio_helper.h index 145e70f1..5c0b451b 100644 --- a/src/bio_helper.h +++ b/src/bio_helper.h @@ -148,7 +148,7 @@ void dattobd_bio_op_clear_flag(struct bio *bio, unsigned int flag); struct inode *page_get_inode(struct page *pg); -int bio_needs_cow(struct bio *bio, struct inode *inode); +int bio_needs_cow(struct bio *bio, struct snap_device *dev); void bio_free_clone(struct bio *bio); diff --git a/src/proc_seq_file.c b/src/proc_seq_file.c index b73c25cd..21df7811 100644 --- a/src/proc_seq_file.c +++ b/src/proc_seq_file.c @@ -91,6 +91,8 @@ static int dattobd_proc_show(struct seq_file *m, void *v) seq_printf(m, "\t\t\t\"minor\": %u,\n", dev->sd_minor); seq_printf(m, "\t\t\t\"cow_file\": \"%s\",\n", dev->sd_cow_path); + seq_printf(m, "\t\t\t\"full_cow_path\": \"%s\",\n", + dev->sd_cow_full_path); seq_printf(m, "\t\t\t\"block_device\": \"%s\",\n", dev->sd_bdev_path); seq_printf(m, "\t\t\t\"max_cache\": %lu,\n", @@ -133,7 +135,8 @@ static int dattobd_proc_show(struct seq_file *m, void *v) if (error) seq_printf(m, "\t\t\t\"error\": %d,\n", error); - seq_printf(m, "\t\t\t\"state\": %lu\n", dev->sd_state); + seq_printf(m, "\t\t\t\"state\": %lu,\n", dev->sd_state); + seq_printf(m, "\t\t\t\"resident\": %d\n", test_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags)); seq_printf(m, "\t\t}"); } diff --git a/src/snap_device.h b/src/snap_device.h index e877ed60..14e0076c 100644 --- a/src/snap_device.h +++ b/src/snap_device.h @@ -16,9 +16,13 @@ #define ACTIVE 1 #define UNVERIFIED 2 +// macros for defining the flags of a snap_device (bit offsets) +#define SD_FLAG_COW_RESIDENT 0 // the cow file exists on the backing device + struct snap_device { unsigned int sd_minor; // minor number of the snapshot unsigned long sd_state; // current state of the snapshot + unsigned long sd_flags; // flags unsigned long sd_falloc_size; // space allocated to the cow file (in // megabytes) unsigned long sd_cache_size; // maximum cache size (in bytes) @@ -31,7 +35,8 @@ struct snap_device { struct block_device *sd_base_dev; // device being snapshot char *sd_bdev_path; // base device file path struct cow_manager *sd_cow; // cow manager - char *sd_cow_path; // cow file path + char *sd_cow_path; // cow file path (for resident cow files) + char *sd_cow_full_path; // full cow file path (for non-resident cow files) struct inode *sd_cow_inode; // cow file inode make_request_fn *sd_orig_mrf; // block device's original make request function diff --git a/src/system_call_hooking.c b/src/system_call_hooking.c index aabe9e6a..c715dbcb 100644 --- a/src/system_call_hooking.c +++ b/src/system_call_hooking.c @@ -90,16 +90,11 @@ int __handle_bdev_mount_nowrite(const struct vfsmount *mnt, dev->sd_base_dev != mnt->mnt_sb->s_bdev) continue; - // if we are unmounting the vfsmount we are using go to dormant - // state - if (mnt == dattobd_get_mnt(dev->sd_cow->filp)) { - LOG_DEBUG("block device umount detected for device %d", - i); - auto_transition_dormant(i); + LOG_DEBUG("block device umount detected for device %d", i); + auto_transition_dormant(i); - ret = 0; - goto out; - } + ret = 0; + goto out; } i = 0; ret = -ENODEV; diff --git a/src/tracer.c b/src/tracer.c index 842a56fe..25b770e5 100644 --- a/src/tracer.c +++ b/src/tracer.c @@ -159,7 +159,7 @@ static int snap_trace_bio(struct snap_device *dev, struct bio *bio) unsigned int bytes, pages; // if we don't need to cow this bio just call the real mrf normally - if (!bio_needs_cow(bio, dev->sd_cow_inode)) + if (!bio_needs_cow(bio, dev)) return dattobd_call_mrf(dev->sd_orig_mrf, dattobd_bio_get_queue(bio), bio); @@ -255,6 +255,14 @@ static int inc_trace_bio(struct snap_device *dev, struct bio *bio) bio_iter_t iter; bio_iter_bvec_t bvec; + if (!test_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags)) { + // if the cow is non-resident, then we don't need to check if + // the bio is for the cow file. + ret = inc_make_sset(dev, bio_sector(bio), + bio_size(bio) / SECTOR_SIZE); + goto out; + } + #ifdef HAVE_ENUM_REQ_OPF //#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,10,0) if (bio_op(bio) == REQ_OP_WRITE_ZEROES) { @@ -406,10 +414,7 @@ static int __tracer_destroy_cow(struct snap_device *dev, int close_method) if (dev->sd_cow) { LOG_DEBUG("destroying cow manager"); - if (close_method == 0) { - cow_free(dev->sd_cow); - dev->sd_cow = NULL; - } else if (close_method == 1) { + if (close_method == 0 || close_method == 1) { ret = cow_sync_and_free(dev->sd_cow); dev->sd_cow = NULL; } else if (close_method == 2) { @@ -483,11 +488,8 @@ static int __tracer_setup_cow(struct snap_device *dev, } } - // verify that file is on block device - if (!file_is_on_bdev(dev->sd_cow->filp, bdev)) { - ret = -EINVAL; - LOG_ERROR(ret, "'%s' is not on '%s'", cow_path, bdev_name); - goto error; + if (file_is_on_bdev(dev->sd_cow->filp, bdev)) { + set_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags); } // find the cow file's inode number @@ -627,6 +629,12 @@ static void __tracer_destroy_cow_path(struct snap_device *dev) kfree(dev->sd_cow_path); dev->sd_cow_path = NULL; } + + if (dev->sd_cow_full_path) { + LOG_DEBUG("freeing full cow path"); + kfree(dev->sd_cow_full_path); + dev->sd_cow_full_path = NULL; + } } static int __tracer_setup_cow_path(struct snap_device *dev, @@ -635,6 +643,11 @@ static int __tracer_setup_cow_path(struct snap_device *dev, int ret; // get the pathname of the cow file (relative to the mountpoint) + LOG_DEBUG("getting absolute pathname of cow file"); + ret = file_get_absolute_pathname(cow_file, &dev->sd_cow_full_path, NULL); + if (ret) + goto error; + LOG_DEBUG("getting relative pathname of cow file"); ret = dentry_get_relative_pathname(dattobd_get_dentry(cow_file), &dev->sd_cow_path, NULL); @@ -1579,14 +1592,17 @@ void __tracer_dormant_to_active(struct snap_device *dev, const char __user *user_mount_path) { int ret; + char *resident_cow_path; char *cow_path; // generate the full pathname ret = user_mount_pathname_concat(user_mount_path, dev->sd_cow_path, - &cow_path); + &resident_cow_path); if (ret) goto error; + cow_path = test_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags) ? resident_cow_path : dev->sd_cow_full_path; + // setup the cow manager ret = __tracer_setup_cow_reopen(dev, dev->sd_base_dev, cow_path); if (ret) @@ -1608,13 +1624,13 @@ void __tracer_dormant_to_active(struct snap_device *dev, set_bit(ACTIVE, &dev->sd_state); clear_bit(UNVERIFIED, &dev->sd_state); - kfree(cow_path); + kfree(resident_cow_path); return; error: LOG_ERROR(ret, "error transitioning tracer to active state"); - if (cow_path) - kfree(cow_path); + if (resident_cow_path) + kfree(resident_cow_path); tracer_set_fail_state(dev, ret); } From 65744a3e26e098103f932ea1466df66454479cef Mon Sep 17 00:00:00 2001 From: Dakota Williams Date: Thu, 28 Apr 2022 11:28:24 -0400 Subject: [PATCH 2/3] Add resident cow checking for FUSE mounts FUSE mounts cannot have resident cow files. FUSE mounts the device first, then sets up its ioctl callbacks (open, close, read, write). This becomes a problem on a remount, since we'd try to reopen the cow file during the mount syscall, which will hang since open hasn't been set by FUSE yet. --- src/tracer.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/tracer.c b/src/tracer.c index 25b770e5..38ac4ba3 100644 --- a/src/tracer.c +++ b/src/tracer.c @@ -24,6 +24,9 @@ #include #endif +#define FUSEBLK_MNT_NAME "fuseblk" +#define FUSEBLK_MNT_LEN 7 + #if !defined(HAVE_BDEV_STACK_LIMITS) && !defined(HAVE_BLK_SET_DEFAULT_LIMITS) //#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,31) @@ -435,6 +438,7 @@ static int __tracer_setup_cow(struct snap_device *dev, int ret; uint64_t max_file_size; char bdev_name[BDEVNAME_SIZE]; + char fuseblk_name[] = FUSEBLK_MNT_NAME; bdevname(bdev, bdev_name); @@ -490,6 +494,21 @@ static int __tracer_setup_cow(struct snap_device *dev, if (file_is_on_bdev(dev->sd_cow->filp, bdev)) { set_bit(SD_FLAG_COW_RESIDENT, &dev->sd_flags); + /* Cow file should not be on the same device as a fuse fs. + * This is because the typical setup for a fuse device involves + * calling mount() and /then/ setting its ioctl callbacks (open + * is of specific interest). Dormant- or unverified-to-active + * transitions cause a deadlock because open will be called in + * the mount syscall where the fuse device's callbacks won't + * have been set yet. We prevent this from happening by not + * allowing it in the first place. */ + if (0 == strncmp(bdev->bd_super->s_type->name, fuseblk_name, + FUSEBLK_MNT_LEN)) { + ret = -EDEADLOCK; + LOG_ERROR(ret, "Cow file cannot be on same volume for" + " fuse filesystems"); + return ret; + } } // find the cow file's inode number From 4bb36095e5ed30e29cd0e4fad09c7a193b927dd5 Mon Sep 17 00:00:00 2001 From: Dakota Williams Date: Mon, 23 May 2022 15:45:49 -0400 Subject: [PATCH 3/3] Add synchronization checks for failing tests --- src/ioctl_handlers.c | 1 + src/snap_device.h | 4 ++-- src/snap_ops.c | 2 ++ src/tracer.c | 12 +++++++----- src/tracing_params.c | 4 ++++ tests/test_setup.py | 9 +++++---- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/ioctl_handlers.c b/src/ioctl_handlers.c index cb5a6dae..2f52071a 100644 --- a/src/ioctl_handlers.c +++ b/src/ioctl_handlers.c @@ -59,6 +59,7 @@ int __verify_minor(unsigned int minor, int mode) } // check that the device is not busy if we care + smp_mb(); if (mode == 1 && atomic_read(&snap_devices[minor]->sd_refs)) { LOG_ERROR(-EBUSY, "device specified is busy"); return -EBUSY; diff --git a/src/snap_device.h b/src/snap_device.h index 14e0076c..b64d5a9c 100644 --- a/src/snap_device.h +++ b/src/snap_device.h @@ -35,8 +35,8 @@ struct snap_device { struct block_device *sd_base_dev; // device being snapshot char *sd_bdev_path; // base device file path struct cow_manager *sd_cow; // cow manager - char *sd_cow_path; // cow file path (for resident cow files) - char *sd_cow_full_path; // full cow file path (for non-resident cow files) + char *sd_cow_path; // deprecated: cow file path + char *sd_cow_full_path; // full cow file path struct inode *sd_cow_inode; // cow file inode make_request_fn *sd_orig_mrf; // block device's original make request function diff --git a/src/snap_ops.c b/src/snap_ops.c index 185f317f..e2fff005 100644 --- a/src/snap_ops.c +++ b/src/snap_ops.c @@ -14,7 +14,9 @@ static int __tracer_add_ref(struct snap_device *dev, int ref_cnt) goto error; } + smp_mb(); atomic_add(ref_cnt, &dev->sd_refs); + smp_mb(); error: return ret; diff --git a/src/tracer.c b/src/tracer.c index 38ac4ba3..205d4395 100644 --- a/src/tracer.c +++ b/src/tracer.c @@ -372,7 +372,9 @@ static void minor_range_include(unsigned int minor) static void __tracer_init(struct snap_device *dev) { LOG_DEBUG("initializing tracer"); + smp_mb(); atomic_set(&dev->sd_fail_code, 0); + smp_mb(); bio_queue_init(&dev->sd_cow_bios); bio_queue_init(&dev->sd_orig_bios); sset_queue_init(&dev->sd_pending_ssets); @@ -417,7 +419,10 @@ static int __tracer_destroy_cow(struct snap_device *dev, int close_method) if (dev->sd_cow) { LOG_DEBUG("destroying cow manager"); - if (close_method == 0 || close_method == 1) { + if (close_method == 0) { + cow_free(dev->sd_cow); + dev->sd_cow = NULL; + } else if (close_method == 1) { ret = cow_sync_and_free(dev->sd_cow); dev->sd_cow = NULL; } else if (close_method == 2) { @@ -1624,6 +1629,7 @@ void __tracer_dormant_to_active(struct snap_device *dev, // setup the cow manager ret = __tracer_setup_cow_reopen(dev, dev->sd_base_dev, cow_path); + kfree(resident_cow_path); if (ret) goto error; @@ -1643,13 +1649,9 @@ void __tracer_dormant_to_active(struct snap_device *dev, set_bit(ACTIVE, &dev->sd_state); clear_bit(UNVERIFIED, &dev->sd_state); - kfree(resident_cow_path); - return; error: LOG_ERROR(ret, "error transitioning tracer to active state"); - if (resident_cow_path) - kfree(resident_cow_path); tracer_set_fail_state(dev, ret); } diff --git a/src/tracing_params.c b/src/tracing_params.c index beab8181..9c959036 100644 --- a/src/tracing_params.c +++ b/src/tracing_params.c @@ -28,7 +28,9 @@ int tp_alloc(struct snap_device *dev, struct bio *bio, tp->orig_bio = bio; tp->bio_sects.head = NULL; tp->bio_sects.tail = NULL; + smp_mb(); atomic_set(&tp->refs, 1); + smp_mb(); *tp_out = tp; return 0; @@ -36,7 +38,9 @@ int tp_alloc(struct snap_device *dev, struct bio *bio, void tp_get(struct tracing_params *tp) { + smp_mb(); atomic_inc(&tp->refs); + smp_mb(); } void tp_put(struct tracing_params *tp) diff --git a/tests/test_setup.py b/tests/test_setup.py index f61100b8..b7e52dc4 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -38,10 +38,11 @@ def test_setup_cow_file_path_is_dir(self): self.assertFalse(os.path.exists(self.snap_device)) self.assertIsNone(dattobd.info(self.minor)) - def test_setup_cow_file_on_wrong_device(self): - self.assertEqual(dattobd.setup(self.minor, self.device, "/tmp/{}".format(self.cow_file)), errno.EINVAL) - self.assertFalse(os.path.exists(self.snap_device)) - self.assertIsNone(dattobd.info(self.minor)) + def test_setup_cow_file_on_other_device(self): + self.assertEqual(dattobd.setup(self.minor, self.device, "/tmp/{}".format(self.cow_file)), 0) + self.addCleanup(dattobd.destroy, self.minor) + self.assertTrue(os.path.exists(self.snap_device)) + self.assertIsNotNone(dattobd.info(self.minor)) def test_setup_unmounted_volume(self): util.unmount(self.mount)