Viewing: ext4-add-fstrim-mount-option.patch
commit ad30edf91065d733d7da68de755acf1043429101
Author: Li Dongyang <dongyangli@ddn.com>
AuthorDate: Fri Aug 11 15:32:31 2023 +1000
LU-17980 ldiskfs: add fstrim mount option
Introduce ldiskfs fstrim mount option. to improve online discard
behavior. The '-o fstrim' option overrides '-o discard', instead
of doing online discard, when a block group has more free blocks
than the tunable bg_trimmed_threshold, it will be scheduled for
discard for the whole group with a delay, using ext4_trim_all_free().
When more blocks are freed in the block group, the delay will be
renewed, to aggregate the potential future free blocks, but not
longer than the tunable 60 seconds from the original discard request.
When a block group is scheduled for discard, it will be skipped in
the block allocator for the CR_POWER2_ALIGNED and CR_GOAL_LEN_FAST scan,
to avoid repeat free/allocate on the block group. If we can't find any
suitable block group in the CR_POWER2_ALIGNED and CR_GOAL_LEN_FAST scan,
we will try CR_POWER2_ALIGNED and CR_GOAL_LEN_FAST again
but this time do not skip the block groups.
'-o fstrim' also accepts an optional fstrim_min_blocks parameter
which is also tunable via sysfs, to control the min free blocks we
trim in ext4_trim_all_free(). If the group's average free fragments
is less than fstrim_min_blocks, we skip the group as it's too
fragmented and will result in many small trim commands to the
storage.
There's another sysfs tunable fstrim_avg_frag_no_delay, if the group's
average free fragments is greater than fstrim_avg_frag_no_delay, we
try to trim the group right away without delaying and gathering more
free blocks.
Fix lmd_parse() to pass down the ldiskfs mount options to OST,
when there's mgsnode option the current code cut off options string,
losing the extra ldiskfs mount options.
Signed-off-by: Li Dongyang <dongyangli@ddn.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Reviewed-by: Alex Zhuravlev <bzzz@whamcloud.com>
Tested-by: Eugene Birkine <ebirkine@ddn.com>
Reviewed-by: Jason Huselton <jhuselton@ddn.com>
Reviewed-by: Andreas Dilger <adilger@thelustrecollective.com>
Change-Id: I7f9bc0c10f5caf3ded72b781684579fdd5d796fb
Reviewed-on: https://review.whamcloud.com/60235
---
fs/ext4/ext4.h | 24 ++++++-
fs/ext4/mballoc.c | 179 ++++++++++++++++++++++++++++++++++++++++++++++
fs/ext4/super.c | 22 ++++++
fs/ext4/sysfs.c | 6 ++
4 files changed, 229 insertions(+), 2 deletions(-)
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h
--- a/fs/ext4/ext4.h
+++ b/fs/ext4/ext4.h
@@ -208,6 +208,8 @@ enum criteria {
#define EXT4_MB_USE_RESERVED 0x2000
/* Do strict check for free blocks while retrying block allocation */
#define EXT4_MB_STRICT_CHECK 0x4000
+/* Skip the groups scheduled for fstrim */
+#define EXT4_MB_SKIP_FSTRIM_GROUPS 0x00040000
struct ext4_allocation_request {
/* target inode for block we're allocating */
@@ -1306,6 +1308,7 @@ struct ext4_inode_info {
* scanning in mballoc
*/
#define EXT4_MOUNT2_ABORT 0x00000100 /* Abort filesystem */
+#define EXT4_MOUNT2_FSTRIM 0x00000200 /* fstrim style DISCARD */
#define clear_opt(sb, opt) EXT4_SB(sb)->s_mount_opt &= \
~EXT4_MOUNT_##opt
@@ -1353,7 +1356,11 @@ extern void mb_set_bits(void *bm, int cur, int len);
#define EXT4_LABEL_MAX 16
/* Default min freed blocks which we could clear BG_TRIMMED flag */
-#define EXT4_DEF_BG_TRIMMED_THRESHOLD 128
+#define EXT4_DEF_BG_TRIMMED_THRESHOLD 1024
+
+#define EXT4_DEF_FSTRIM_MIN_BLOCKS 1024
+#define EXT4_DEF_FSTRIM_AVG_FRAG_NO_DELAY 4096
+#define EXT4_DEF_FSTRIM_DELAY_SECS 60
/*
* Structure of the super block
@@ -1650,6 +1657,12 @@ struct ext4_sb_info {
after commit completed */
struct list_head s_discard_list;
struct work_struct s_discard_work;
+ struct list_head s_fstrim_groups_ready;
+ struct list_head s_fstrim_groups_pending;
+ struct list_head s_fstrim_groups_last_free;
+ spinlock_t s_fstrim_groups_lock;
+ struct task_struct *s_fstrim_task;
+ wait_queue_head_t s_fstrim_wq;
atomic_t s_retry_alloc_pending;
struct xarray *s_mb_avg_fragment_size;
struct xarray *s_mb_largest_free_orders;
@@ -1718,6 +1731,10 @@ struct ext4_sb_info {
/* Min freed blocks per group that we could clear BG_TRIMMED on it */
unsigned long s_bg_trimmed_threshold;
+ unsigned int s_fstrim_min_blocks;
+ unsigned int s_fstrim_avg_frag_no_delay;
+ unsigned int s_fstrim_delay_secs;
+
unsigned int s_log_groups_per_flex;
struct flex_groups * __rcu *s_flex_groups;
ext4_group_t s_flex_groups_allocated;
@@ -3740,7 +3757,10 @@ struct ext4_group_info {
void *bb_bitmap;
#endif
struct rw_semaphore alloc_sem;
- ext4_grpblk_t bb_counters[]; /* Nr of free power-of-two-block
+ struct list_head bb_fstrim_list;
+ struct list_head bb_fstrim_last_free;
+ time64_t bb_fstrim_start;
+ time64_t bb_last_free; ext4_grpblk_t bb_counters[]; /* Nr of free power-of-two-block
* regions, index is order.
* bb_counters[3] = 5 means
* 5 free 8-block regions. */
diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
--- a/fs/ext4/mballoc.c
+++ b/fs/ext4/mballoc.c
@@ -432,6 +432,8 @@ static int ext4_try_to_trim_range(struct super_block *sb,
struct ext4_buddy *e4b, ext4_grpblk_t start,
ext4_grpblk_t max, ext4_grpblk_t minblocks);
+static int ext4_mb_fstrim_thread(void *data);
+
/*
* The algorithm using this percpu seq counter goes below:
* 1. We sample the percpu discard_pa_seq counter before trying for block
@@ -2738,6 +2740,11 @@ static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
case CR_POWER2_ALIGNED:
BUG_ON(ac->ac_2order == 0);
+ if (test_opt2(ac->ac_sb, FSTRIM) &&
+ (ac->ac_flags & EXT4_MB_SKIP_FSTRIM_GROUPS) &&
+ !list_empty(&grp->bb_fstrim_list))
+ return false;
+
/* Avoid using the first bg of a flexgroup for data files */
if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&
(flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) &&
@@ -2756,6 +2763,11 @@ static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
return true;
case CR_GOAL_LEN_FAST:
case CR_BEST_AVAIL_LEN:
+ if (test_opt2(ac->ac_sb, FSTRIM) &&
+ (ac->ac_flags & EXT4_MB_SKIP_FSTRIM_GROUPS) &&
+ !list_empty(&grp->bb_fstrim_list))
+ return false;
+
if ((free / fragments) >= ac->ac_g_ex.fe_len)
return true;
break;
@@ -3014,6 +3026,7 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
struct super_block *sb = ac->ac_sb;
struct ext4_sb_info *sbi = EXT4_SB(sb);
struct ext4_buddy e4b;
+ enum criteria cr, orig_cr;
BUG_ON(ac->ac_status == AC_STATUS_FOUND);
@@ -3062,7 +3075,10 @@ ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
ac->ac_criteria = CR_GOAL_LEN_FAST;
if (ac->ac_2order)
ac->ac_criteria = CR_POWER2_ALIGNED;
+ orig_cr = cr = ac->ac_criteria;
+ if (test_opt2(sb, FSTRIM))
+ ac->ac_flags |= EXT4_MB_SKIP_FSTRIM_GROUPS;
ac->ac_e4b = &e4b;
ac->ac_prefetch_ios = 0;
ac->ac_first_err = 0;
@@ -3072,6 +3088,16 @@ repeat:
if (err)
goto out;
+ if (cr != ac->ac_criteria) {
+ if (test_opt2(sb, FSTRIM) &&
+ ac->ac_criteria == CR_GOAL_LEN_SLOW &&
+ orig_cr < CR_GOAL_LEN_SLOW &&
+ (ac->ac_flags & EXT4_MB_SKIP_FSTRIM_GROUPS)) {
+ ac->ac_criteria = orig_cr;
+ ac->ac_flags &= ~EXT4_MB_SKIP_FSTRIM_GROUPS;
+ goto repeat;
+ }
+ }
if (ac->ac_status != AC_STATUS_CONTINUE)
break;
}
@@ -3647,6 +3673,8 @@ int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group,
meta_group_info[i]->bb_avg_fragment_size_order = -1; /* uninit */
meta_group_info[i]->bb_group = group;
meta_group_info[i]->bb_freed_since_trim = 0;
+ INIT_LIST_HEAD(&meta_group_info[i]->bb_fstrim_list);
+ INIT_LIST_HEAD(&meta_group_info[i]->bb_fstrim_last_free);
mb_group_bb_bitmap_alloc(sb, meta_group_info[i], group);
return 0;
@@ -3946,6 +3974,30 @@ int ext4_mb_init(struct super_block *sb)
INIT_LIST_HEAD(&sbi->s_discard_list);
INIT_WORK(&sbi->s_discard_work, ext4_discard_work);
atomic_set(&sbi->s_retry_alloc_pending, 0);
+ INIT_LIST_HEAD(&sbi->s_fstrim_groups_ready);
+ INIT_LIST_HEAD(&sbi->s_fstrim_groups_pending);
+ INIT_LIST_HEAD(&sbi->s_fstrim_groups_last_free);
+ spin_lock_init(&sbi->s_fstrim_groups_lock);
+ init_waitqueue_head(&sbi->s_fstrim_wq);
+ if (test_opt2(sb, FSTRIM)) {
+ if (bdev_max_discard_sectors(sb->s_bdev)) {
+ sbi->s_fstrim_task = kthread_run(ext4_mb_fstrim_thread,
+ sb, "fstrim-%s",
+ sb->s_id);
+ if (IS_ERR(sbi->s_fstrim_task)) {
+ ret = PTR_ERR(sbi->s_fstrim_task);
+ sbi->s_fstrim_task = NULL;
+ ext4_warning(sb, "Failed to create fstrim thread: %d, "
+ "disabling -o fstrim", ret);
+ clear_opt2(sb, FSTRIM);
+ }
+ } else {
+ ext4_warning(sb, "mounting with \"fstrim\" option, "
+ "but the device does not support "
+ "discard, disabling -o fstrim");
+ clear_opt2(sb, FSTRIM);
+ }
+ }
sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN;
sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN;
@@ -4011,6 +4063,9 @@ int ext4_mb_init(struct super_block *sb)
sbi->s_bg_trimmed_threshold = EXT4_DEF_BG_TRIMMED_THRESHOLD;
+ sbi->s_fstrim_avg_frag_no_delay = EXT4_DEF_FSTRIM_AVG_FRAG_NO_DELAY;
+ sbi->s_fstrim_delay_secs = EXT4_DEF_FSTRIM_DELAY_SECS;
+
sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group);
if (sbi->s_locality_groups == NULL) {
ret = -ENOMEM;
@@ -4086,6 +4141,11 @@ void ext4_mb_release(struct super_block *sb)
flush_work(&sbi->s_discard_work);
WARN_ON_ONCE(!list_empty(&sbi->s_discard_list));
+ if (test_opt2(sb, FSTRIM) && sbi->s_fstrim_task != NULL) {
+ kthread_stop(sbi->s_fstrim_task);
+ sbi->s_fstrim_task = NULL;
+ }
+
group_info = rcu_access_pointer(sbi->s_group_info);
if (group_info) {
for (i = 0; i < ngroups; i++) {
@@ -4226,6 +4286,11 @@ void ext4_process_freed_data(struct super_block *sb, tid_t commit_tid)
list_for_each_entry_safe(entry, tmp, &freed_data_list, efd_list)
kmem_cache_free(ext4_free_data_cachep, entry);
}
+
+ if (test_opt2(sb, FSTRIM) &&
+ (!list_empty(&sbi->s_fstrim_groups_ready) ||
+ !list_empty(&sbi->s_fstrim_groups_pending)))
+ wake_up(&sbi->s_fstrim_wq);
}
int __init ext4_init_mballoc(void)
@@ -6936,6 +7001,42 @@ do_more:
mb_free_blocks(inode, &e4b, bit, count_clusters);
}
+ if (test_opt2(sb, FSTRIM) &&
+ grp->bb_freed_since_trim >= sbi->s_bg_trimmed_threshold) {
+ ext4_grpblk_t avg_frag;
+
+ avg_frag = (grp->bb_free == 0 || grp->bb_fragments == 0) ?
+ count : (grp->bb_free / grp->bb_fragments);
+ if (avg_frag >= sbi->s_fstrim_min_blocks) {
+ spin_lock(&sbi->s_fstrim_groups_lock);
+ if (list_empty(&grp->bb_fstrim_list)) {
+ grp->bb_fstrim_start = ktime_get_seconds();
+ grp->bb_last_free = grp->bb_fstrim_start;
+ if (avg_frag >=
+ sbi->s_fstrim_avg_frag_no_delay) {
+ list_add_tail(&grp->bb_fstrim_list,
+ &sbi->s_fstrim_groups_ready);
+ } else {
+ list_add_tail(&grp->bb_fstrim_list,
+ &sbi->s_fstrim_groups_pending);
+ list_add_tail(&grp->bb_fstrim_last_free,
+ &sbi->s_fstrim_groups_last_free);
+ }
+ } else if (!list_empty(&grp->bb_fstrim_last_free)) {
+ grp->bb_last_free = ktime_get_seconds();
+ if (avg_frag >=
+ sbi->s_fstrim_avg_frag_no_delay) {
+ list_move_tail(&grp->bb_fstrim_list,
+ &sbi->s_fstrim_groups_ready);
+ list_del_init(&grp->bb_fstrim_last_free);
+ } else {
+ list_move_tail(&grp->bb_fstrim_last_free,
+ &sbi->s_fstrim_groups_last_free);
+ }
+ }
+ spin_unlock(&sbi->s_fstrim_groups_lock);
+ }
+ }
ext4_unlock_group(sb, block_group);
/*
@@ -7365,6 +7466,84 @@ ext4_trim_all_free(struct super_block *sb, ext4_group_t group,
return ret;
}
+static int ext4_mb_fstrim_thread(void *data)
+{
+ struct super_block *sb = (struct super_block *)data;
+ struct ext4_sb_info *sbi = EXT4_SB(sb);
+ struct ext4_group_info *grp, *tmp;
+ struct list_head fstrim_groups;
+ struct request_queue *q = bdev_get_queue(sb->s_bdev);
+ unsigned int discard_granularity =
+ q->limits.discard_granularity >> sb->s_blocksize_bits;
+ uint64_t minblks;
+ time64_t now;
+ int ret = 0;
+
+ INIT_LIST_HEAD(&fstrim_groups);
+
+ while (!kthread_should_stop()) {
+ wait_event_interruptible(sbi->s_fstrim_wq,
+ !list_empty(&sbi->s_fstrim_groups_ready) ||
+ !list_empty(&sbi->s_fstrim_groups_pending) ||
+ kthread_should_stop());
+ if (kthread_should_stop())
+ break;
+
+ spin_lock(&sbi->s_fstrim_groups_lock);
+ if (list_empty(&sbi->s_fstrim_groups_ready) &&
+ list_empty(&sbi->s_fstrim_groups_pending)) {
+ spin_unlock(&sbi->s_fstrim_groups_lock);
+ continue;
+ }
+
+ list_splice_init(&sbi->s_fstrim_groups_ready, &fstrim_groups);
+
+ now = ktime_get_seconds();
+ list_for_each_entry_safe(grp, tmp,
+ &sbi->s_fstrim_groups_pending,
+ bb_fstrim_list) {
+ if (now - grp->bb_fstrim_start >=
+ sbi->s_fstrim_delay_secs) {
+ list_move_tail(&grp->bb_fstrim_list,
+ &fstrim_groups);
+ list_del_init(&grp->bb_fstrim_last_free);
+ } else {
+ break;
+ }
+ }
+ list_for_each_entry_safe(grp, tmp,
+ &sbi->s_fstrim_groups_last_free,
+ bb_fstrim_last_free) {
+ if (now - grp->bb_last_free >=
+ 2 * sbi->s_commit_interval / HZ) {
+ list_move_tail(&grp->bb_fstrim_list,
+ &fstrim_groups);
+ list_del_init(&grp->bb_fstrim_last_free);
+ } else {
+ break;
+ }
+ }
+ spin_unlock(&sbi->s_fstrim_groups_lock);
+
+ list_for_each_entry_safe(grp, tmp, &fstrim_groups,
+ bb_fstrim_list) {
+ minblks = sbi->s_fstrim_min_blocks;
+ if (minblks < discard_granularity)
+ minblks = discard_granularity;
+ if (ret >= 0 && !atomic_read(&sbi->s_retry_alloc_pending))
+ ret = ext4_trim_all_free(sb, grp->bb_group, 0,
+ EXT4_CLUSTERS_PER_GROUP(sb) - 1,
+ EXT4_NUM_B2C(sbi, minblks));
+ list_del_init(&grp->bb_fstrim_list);
+ cond_resched();
+ if (kthread_should_stop())
+ break;
+ }
+ }
+
+ return 0;
+}
+
/**
* ext4_trim_fs() -- trim ioctl handle function
* @sb: superblock for filesystem
diff --git a/fs/ext4/super.c b/fs/ext4/super.c
--- a/fs/ext4/super.c
+++ b/fs/ext4/super.c
@@ -1718,6 +1718,7 @@ enum {
Opt_max_dir_size_kb, Opt_nojournal_checksum, Opt_nombcache,
Opt_no_prefetch_block_bitmaps, Opt_mb_optimize_scan,
Opt_errors, Opt_data, Opt_data_err, Opt_jqfmt, Opt_dax_type,
+ Opt_fstrim, Opt_nofstrim,
#ifdef CONFIG_EXT4_DEBUG
Opt_fc_debug_max_replay, Opt_fc_debug_force
#endif
@@ -1836,6 +1837,9 @@ static const struct fs_parameter_spec ext4_param_specs[] = {
fsparam_flag ("dioread_lock", Opt_dioread_lock),
fsparam_flag ("discard", Opt_discard),
fsparam_flag ("nodiscard", Opt_nodiscard),
+ fsparam_flag ("fstrim", Opt_fstrim),
+ fsparam_u32 ("fstrim", Opt_fstrim),
+ fsparam_flag ("nofstrim", Opt_nofstrim),
fsparam_u32 ("init_itable", Opt_init_itable),
fsparam_flag ("init_itable", Opt_init_itable),
fsparam_flag ("noinit_itable", Opt_noinit_itable),
@@ -1899,6 +1903,8 @@ static const struct mount_opts {
MOPT_EXT4_ONLY | MOPT_CLEAR},
{Opt_discard, EXT4_MOUNT_DISCARD, MOPT_SET},
{Opt_nodiscard, EXT4_MOUNT_DISCARD, MOPT_CLEAR},
+ {Opt_fstrim, EXT4_MOUNT2_FSTRIM, MOPT_SET | MOPT_2},
+ {Opt_nofstrim, EXT4_MOUNT2_FSTRIM, MOPT_CLEAR | MOPT_2},
{Opt_delalloc, EXT4_MOUNT_DELALLOC,
MOPT_EXT4_ONLY | MOPT_SET | MOPT_EXPLICIT},
{Opt_nodelalloc, EXT4_MOUNT_DELALLOC,
@@ -2000,6 +2006,7 @@ ext4_sb_read_encoding(const struct ext4_super_block *es)
#define EXT4_SPEC_s_fc_debug_max_replay (1 << 17)
#define EXT4_SPEC_s_sb_block (1 << 18)
#define EXT4_SPEC_mb_optimize_scan (1 << 19)
+#define EXT4_SPEC_s_fstrim_min_blocks (1 << 20)
struct ext4_fs_context {
char *s_qf_names[EXT4_MAXQUOTAS];
@@ -2030,6 +2037,7 @@ struct ext4_fs_context {
kuid_t s_resuid;
kgid_t s_resgid;
ext4_fsblk_t s_sb_block;
+ unsigned int s_fstrim_min_blocks;
};
static void ext4_fc_free(struct fs_context *fc)
@@ -2424,6 +2432,13 @@ static int ext4_parse_param(struct fs_context *fc, struct fs_parameter *param)
return -EINVAL;
}
return 0;
+ case Opt_fstrim:
+ ctx_set_mount_opt2(ctx, EXT4_MOUNT2_FSTRIM);
+ ctx->s_fstrim_min_blocks = EXT4_DEF_FSTRIM_MIN_BLOCKS;
+ if (param->type == fs_value_is_string)
+ ctx->s_fstrim_min_blocks = result.uint_32;
+ ctx->spec |= EXT4_SPEC_s_fstrim_min_blocks;
+ return 0;
}
/*
@@ -2882,6 +2897,7 @@ static void ext4_apply_options(struct fs_context *fc, struct super_block *sb)
APPLY(s_li_wait_mult);
APPLY(s_resgid);
APPLY(s_resuid);
+ APPLY(s_fstrim_min_blocks);
#ifdef CONFIG_EXT4_DEBUG
APPLY(s_fc_debug_max_replay);
@@ -5680,6 +5696,12 @@ static int __ext4_fill_super(struct fs_context *fc, struct super_block *sb)
if (err)
goto failed_mount5;
+ if (test_opt(sb, DISCARD) && test_opt2(sb, FSTRIM)) {
+ ext4_msg(sb, KERN_WARNING,
+ "fstrim overrides discard option");
+ clear_opt(sb, DISCARD);
+ }
+
err = ext4_mb_init(sb);
if (err) {
ext4_msg(sb, KERN_ERR, "failed to initialize mballoc (%d)",
diff --git a/fs/ext4/sysfs.c b/fs/ext4/sysfs.c
--- a/fs/ext4/sysfs.c
+++ b/fs/ext4/sysfs.c
@@ -286,6 +286,9 @@ EXT4_RW_ATTR_SBI_UI(mb_large_req, s_mb_large_req);
EXT4_RW_ATTR_SBI_UI(mb_max_linear_groups, s_mb_max_linear_groups);
EXT4_RW_ATTR_SBI_UI(extent_max_zeroout_kb, s_extent_max_zeroout_kb);
EXT4_RW_ATTR_SBI_UI(bg_trimmed_threshold, s_bg_trimmed_threshold);
+EXT4_RW_ATTR_SBI_UI(fstrim_min_blocks, s_fstrim_min_blocks);
+EXT4_RW_ATTR_SBI_UI(fstrim_avg_frag_no_delay, s_fstrim_avg_frag_no_delay);
+EXT4_RW_ATTR_SBI_UI(fstrim_delay_secs, s_fstrim_delay_secs);
EXT4_ATTR(trigger_fs_error, 0200, trigger_test_error);
EXT4_RW_ATTR_SBI_PI(err_ratelimit_interval_ms, s_err_ratelimit_state.interval);
EXT4_RW_ATTR_SBI_PI(err_ratelimit_burst, s_err_ratelimit_state.burst);
@@ -345,6 +348,9 @@ static struct attribute *ext4_attrs[] = {
ATTR_LIST(max_writeback_mb_bump),
ATTR_LIST(extent_max_zeroout_kb),
ATTR_LIST(bg_trimmed_threshold),
+ ATTR_LIST(fstrim_min_blocks),
+ ATTR_LIST(fstrim_avg_frag_no_delay),
+ ATTR_LIST(fstrim_delay_secs),
ATTR_LIST(trigger_fs_error),
ATTR_LIST(err_ratelimit_interval_ms),
ATTR_LIST(err_ratelimit_burst),
--
