| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <e4d9a66512e8c29cc18698f7d33bc9e1cf8b8795.1667822611.git.ritesh.list@gmail.com>
Date: Mon, 7 Nov 2022 17:51:36 +0530
From: "Ritesh Harjani (IBM)" <ritesh.list@...il.com>
To: Theodore Ts'o <tytso@....edu>
Cc: linux-ext4@...r.kernel.org,
Harshad Shirwadkar <harshadshirwadkar@...il.com>,
Wang Shilong <wshilong@....com>,
Andreas Dilger <adilger.kernel@...ger.ca>,
Li Xi <lixi@....com>, Ritesh Harjani <ritesh.list@...il.com>
Subject: [RFCv1 48/72] e2fsck: allow admin specify number of threads
From: Wang Shilong <wshilong@....com>
-m option is introduced to specify number of threads for pfsck.
Signed-off-by: Wang Shilong <wshilong@....com>
Signed-off-by: Ritesh Harjani (IBM) <ritesh.list@...il.com>
---
e2fsck/e2fsck.h | 2 +
e2fsck/pass1.c | 154 ++++++++++++++++----------
e2fsck/unix.c | 17 ++-
tests/f_multithread/script | 2 +-
tests/f_multithread_completion/script | 2 +-
tests/f_multithread_logfile/script | 2 +-
tests/f_multithread_no/script | 2 +-
tests/f_multithread_preen/script | 2 +-
tests/f_multithread_yes/script | 2 +-
9 files changed, 122 insertions(+), 63 deletions(-)
diff --git a/e2fsck/e2fsck.h b/e2fsck/e2fsck.h
index 23e8d2ed..e3276924 100644
--- a/e2fsck/e2fsck.h
+++ b/e2fsck/e2fsck.h
@@ -487,6 +487,7 @@ struct e2fsck_struct {
char *undo_file;
#ifdef HAVE_PTHREAD
+ __u32 fs_num_threads;
/* serialize fix operation for multiple threads */
pthread_mutex_t fs_fix_mutex;
/* protect block_found_map, block_dup_map */
@@ -727,6 +728,7 @@ void check_resize_inode(e2fsck_t ctx);
int check_init_orphan_file(e2fsck_t ctx);
/* util.c */
+#define E2FSCK_MAX_THREADS (65536)
extern void *e2fsck_allocate_memory(e2fsck_t ctx, unsigned long size,
const char *description);
extern int ask(e2fsck_t ctx, const char * string, int def);
diff --git a/e2fsck/pass1.c b/e2fsck/pass1.c
index fdd1f3d6..5bf6980b 100644
--- a/e2fsck/pass1.c
+++ b/e2fsck/pass1.c
@@ -1199,6 +1199,97 @@ static int e2fsck_should_abort(e2fsck_t ctx)
return 0;
}
+static void init_ext2_max_sizes()
+{
+ int i;
+ __u64 max_sizes;
+
+ /*
+ * Init ext2_max_sizes which will be immutable and shared between
+ * threads
+ */
+#define EXT2_BPP(bits) (1ULL << ((bits) - 2))
+
+ for (i = EXT2_MIN_BLOCK_LOG_SIZE; i <= EXT2_MAX_BLOCK_LOG_SIZE; i++) {
+ max_sizes = EXT2_NDIR_BLOCKS + EXT2_BPP(i);
+ max_sizes = max_sizes + EXT2_BPP(i) * EXT2_BPP(i);
+ max_sizes = max_sizes + EXT2_BPP(i) * EXT2_BPP(i) * EXT2_BPP(i);
+ max_sizes = (max_sizes * (1UL << i));
+ ext2_max_sizes[i - EXT2_MIN_BLOCK_LOG_SIZE] = max_sizes;
+ }
+#undef EXT2_BPP
+}
+
+#ifdef HAVE_PTHREAD
+/* TODO: tdb needs to be handled properly for multiple threads*/
+static int multiple_threads_supported(e2fsck_t ctx)
+{
+#ifdef CONFIG_TDB
+ unsigned int threshold;
+ ext2_ino_t num_dirs;
+ errcode_t retval;
+ char *tdb_dir;
+ int enable;
+
+ profile_get_string(ctx->profile, "scratch_files", "directory", 0, 0,
+ &tdb_dir);
+ profile_get_uint(ctx->profile, "scratch_files",
+ "numdirs_threshold", 0, 0, &threshold);
+ profile_get_boolean(ctx->profile, "scratch_files",
+ "icount", 0, 1, &enable);
+
+ retval = ext2fs_get_num_dirs(ctx->fs, &num_dirs);
+ if (retval)
+ num_dirs = 1024; /* Guess */
+
+ /* tdb is unsupported now */
+ if (enable && tdb_dir && !access(tdb_dir, W_OK) &&
+ (!threshold || num_dirs > threshold))
+ return 0;
+#endif
+ return 1;
+}
+
+/**
+ * Even though we could specify number of threads,
+ * but it might be more than the whole filesystem
+ * block groups, correct it here.
+ */
+static void e2fsck_pass1_set_thread_num(e2fsck_t ctx)
+{
+ unsigned flexbg_size = 1;
+ ext2_filsys fs = ctx->fs;
+ int num_threads = ctx->fs_num_threads;
+ int max_threads;
+
+ if (num_threads < 1) {
+ num_threads = 1;
+ goto out;
+ }
+
+ if (!multiple_threads_supported(ctx)) {
+ num_threads = 1;
+ fprintf(stderr, "Fall through single thread for pass1 "
+ "because tdb could not handle properly\n");
+ goto out;
+ }
+
+ if (ext2fs_has_feature_flex_bg(fs->super))
+ flexbg_size = 1 << fs->super->s_log_groups_per_flex;
+ max_threads = fs->group_desc_count / flexbg_size;
+ if (max_threads == 0)
+ max_threads = 1;
+
+ if (num_threads > max_threads) {
+ fprintf(stderr, "Use max possible thread num: %d instead\n",
+ max_threads);
+ num_threads = max_threads;
+ }
+out:
+ ctx->fs_num_threads = num_threads;
+}
+#endif
+
/*
* We need call mark_table_blocks() before multiple
* thread start, since all known system blocks should be
@@ -1209,6 +1300,11 @@ static errcode_t e2fsck_pass1_prepare(e2fsck_t ctx)
struct problem_context pctx;
ext2_filsys fs = ctx->fs;
+ init_ext2_max_sizes();
+#ifdef HAVE_PTHREAD
+ e2fsck_pass1_set_thread_num(ctx);
+#endif
+
clear_problem_context(&pctx);
if (!(ctx->options & E2F_OPT_PREEN))
fix_problem(ctx, PR_1_PASS_HEADER, &pctx);
@@ -2271,27 +2367,6 @@ endit:
ctx->invalid_bitmaps++;
}
-static void init_ext2_max_sizes()
-{
- int i;
- __u64 max_sizes;
-
- /*
- * Init ext2_max_sizes which will be immutable and shared between
- * threads
- */
-#define EXT2_BPP(bits) (1ULL << ((bits) - 2))
-
- for (i = EXT2_MIN_BLOCK_LOG_SIZE; i <= EXT2_MAX_BLOCK_LOG_SIZE; i++) {
- max_sizes = EXT2_NDIR_BLOCKS + EXT2_BPP(i);
- max_sizes = max_sizes + EXT2_BPP(i) * EXT2_BPP(i);
- max_sizes = max_sizes + EXT2_BPP(i) * EXT2_BPP(i) * EXT2_BPP(i);
- max_sizes = (max_sizes * (1UL << i));
- ext2_max_sizes[i - EXT2_MIN_BLOCK_LOG_SIZE] = max_sizes;
- }
-#undef EXT2_BPP
-}
-
#ifdef HAVE_PTHREAD
static errcode_t e2fsck_pass1_merge_bitmap(ext2_filsys fs, ext2fs_generic_bitmap *src,
ext2fs_generic_bitmap *dest)
@@ -3151,7 +3226,7 @@ static int e2fsck_pass1_threads_start(struct e2fsck_thread_info **pinfo,
static void e2fsck_pass1_multithread(e2fsck_t global_ctx)
{
struct e2fsck_thread_info *infos = NULL;
- int num_threads = 1;
+ int num_threads = global_ctx->fs_num_threads;
errcode_t retval;
retval = e2fsck_pass1_threads_start(&infos, num_threads, global_ctx);
@@ -3174,48 +3249,15 @@ out_abort:
}
#endif
-/* TODO: tdb needs to be handled properly for multiple threads*/
-static int multiple_threads_supported(e2fsck_t ctx)
-{
-#ifdef CONFIG_TDB
- unsigned int threshold;
- ext2_ino_t num_dirs;
- errcode_t retval;
- char *tdb_dir;
- int enable;
-
- profile_get_string(ctx->profile, "scratch_files", "directory", 0, 0,
- &tdb_dir);
- profile_get_uint(ctx->profile, "scratch_files",
- "numdirs_threshold", 0, 0, &threshold);
- profile_get_boolean(ctx->profile, "scratch_files",
- "icount", 0, 1, &enable);
-
- retval = ext2fs_get_num_dirs(ctx->fs, &num_dirs);
- if (retval)
- num_dirs = 1024; /* Guess */
-
- /* tdb is unsupported now */
- if (enable && tdb_dir && !access(tdb_dir, W_OK) &&
- (!threshold || num_dirs > threshold)) {
- fprintf(stderr, "Fall through single thread for pass1 "
- "because tdb could not handle properly\n");
- return 0;
- }
- #endif
- return 1;
-}
-
void e2fsck_pass1(e2fsck_t ctx)
{
errcode_t retval;
- init_ext2_max_sizes();
retval = e2fsck_pass1_prepare(ctx);
if (retval)
return;
#ifdef HAVE_PTHREAD
- if (ctx->options & E2F_OPT_MULTITHREAD && multiple_threads_supported(ctx))
+ if (ctx->options & E2F_OPT_MULTITHREAD || ctx->fs_num_threads > 1)
e2fsck_pass1_multithread(ctx);
else
#endif
diff --git a/e2fsck/unix.c b/e2fsck/unix.c
index af2457e3..dfa3f897 100644
--- a/e2fsck/unix.c
+++ b/e2fsck/unix.c
@@ -826,6 +826,10 @@ static errcode_t PRS(int argc, char *argv[], e2fsck_t *ret_ctx)
int res; /* result of sscanf */
#ifdef CONFIG_JBD_DEBUG
char *jbd_debug;
+#endif
+#ifdef HAVE_PTHREAD
+ char *pm;
+ unsigned long thread_num;
#endif
unsigned long long phys_mem_kb, blk;
@@ -859,7 +863,7 @@ static errcode_t PRS(int argc, char *argv[], e2fsck_t *ret_ctx)
ctx->readahead_kb = ~0ULL;
#ifdef HAVE_PTHREAD
- while ((c = getopt(argc, argv, "pamnyrcC:B:dE:fvtFVM:b:I:j:P:l:L:N:SsDkz:")) != EOF)
+ while ((c = getopt(argc, argv, "pam:nyrcC:B:dE:fvtFVM:b:I:j:P:l:L:N:SsDkz:")) != EOF)
#else
while ((c = getopt(argc, argv, "panyrcC:B:dE:fvtFVM:b:I:j:P:l:L:N:SsDkz:")) != EOF)
#endif
@@ -905,7 +909,18 @@ static errcode_t PRS(int argc, char *argv[], e2fsck_t *ret_ctx)
break;
#ifdef HAVE_PTHREAD
case 'm':
+ thread_num = strtoul(optarg, &pm, 0);
+ if (*pm)
+ fatal_error(ctx,
+ _("Invalid multiple thread num.\n"));
+ if (thread_num > E2FSCK_MAX_THREADS) {
+ fprintf(stderr,
+ _("threads %lu too large (max %lu)\n"),
+ thread_num, E2FSCK_MAX_THREADS);
+ fatal_error(ctx, 0);
+ }
ctx->options |= E2F_OPT_MULTITHREAD;
+ ctx->fs_num_threads = thread_num;
break;
#endif
case 'n':
diff --git a/tests/f_multithread/script b/tests/f_multithread/script
index 0fe96cd0..83cd0f03 100644
--- a/tests/f_multithread/script
+++ b/tests/f_multithread/script
@@ -1,4 +1,4 @@
-FSCK_OPT="-fy -m"
+FSCK_OPT="-fy -m1"
SECOND_FSCK_OPT=-yf
. $cmd_dir/run_e2fsck
diff --git a/tests/f_multithread_completion/script b/tests/f_multithread_completion/script
index bf23cd61..0ec13816 100644
--- a/tests/f_multithread_completion/script
+++ b/tests/f_multithread_completion/script
@@ -1,4 +1,4 @@
-FSCK_OPT="-fy -m -C 1"
+FSCK_OPT="-fy -m1 -C 1"
SECOND_FSCK_OPT=-yf
. $cmd_dir/run_e2fsck
diff --git a/tests/f_multithread_logfile/script b/tests/f_multithread_logfile/script
index 4f9ca6f8..dbb65319 100644
--- a/tests/f_multithread_logfile/script
+++ b/tests/f_multithread_logfile/script
@@ -1,5 +1,5 @@
LOG_FNAME="f_multithread_logfile_xxx"
-FSCK_OPT="-fy -m -y -E log_filename=$LOG_FNAME"
+FSCK_OPT="-fy -m1 -y -E log_filename=$LOG_FNAME"
SKIP_VERIFY="true"
ONE_PASS_ONLY="true"
SKIP_CLEANUP="true"
diff --git a/tests/f_multithread_no/script b/tests/f_multithread_no/script
index b93deb3a..db791e11 100644
--- a/tests/f_multithread_no/script
+++ b/tests/f_multithread_no/script
@@ -1,4 +1,4 @@
-FSCK_OPT="-fn -m"
+FSCK_OPT="-fn -m1"
SECOND_FSCK_OPT=-yf
. $cmd_dir/run_e2fsck
diff --git a/tests/f_multithread_preen/script b/tests/f_multithread_preen/script
index ecb79cd6..8965f4a7 100644
--- a/tests/f_multithread_preen/script
+++ b/tests/f_multithread_preen/script
@@ -1,4 +1,4 @@
-FSCK_OPT="-fp -m"
+FSCK_OPT="-fp -m1"
SECOND_FSCK_OPT=-yf
. $cmd_dir/run_e2fsck
diff --git a/tests/f_multithread_yes/script b/tests/f_multithread_yes/script
index 38891f6a..8b4aa9b8 100644
--- a/tests/f_multithread_yes/script
+++ b/tests/f_multithread_yes/script
@@ -1,4 +1,4 @@
-FSCK_OPT="-f -m"
+FSCK_OPT="-f -m1"
SECOND_FSCK_OPT=-yf
. $cmd_dir/run_e2fsck
--
2.37.3
Powered by blists - more mailing lists