Message-ID: <20250422123612.261764-3-lihongbo22@huawei.com>
Date: Tue, 22 Apr 2025 12:36:10 +0000
From: Hongbo Li <lihongbo22@...wei.com>
To: <xiang@...nel.org>, <chao@...nel.org>, <huyue2@...lpad.com>,
	<jefflexu@...ux.alibaba.com>
CC: <linux-erofs@...ts.ozlabs.org>, <linux-kernel@...r.kernel.org>,
	<lihongbo22@...wei.com>
Subject: [PATCH RFC 2/4] erofs-utils: lib: Implement the main logic for --meta_fix option

The option --meta_fix will fix the metadata area, so for the data
area, we first dump them into a temporary file to avoid alloc buffer
from @g_sbi.bmgr. Then we remmap the blkaddr based on the offset in
@g_sbi.bmgr after metadata finishing, and dump it into @g_sbi.bmgr
directly.

Signed-off-by: Hongbo Li <lihongbo22@...wei.com>
---
 include/erofs/blobraw.h |  28 ++++++++++
 lib/Makefile.am         |   3 +-
 lib/blobraw.c           | 121 ++++++++++++++++++++++++++++++++++++++++
 lib/inode.c             |   6 ++
 mkfs/main.c             |  15 +++++
 5 files changed, 172 insertions(+), 1 deletion(-)
 create mode 100755 include/erofs/blobraw.h
 create mode 100755 lib/blobraw.c

diff --git a/include/erofs/blobraw.h b/include/erofs/blobraw.h
new file mode 100755
index 0000000..d56bb46
--- /dev/null
+++ b/include/erofs/blobraw.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */
+/*
+ * erofs-utils/lib/blobraw.h
+ *
+ * Copyright (C) 2025, HUAWEI, Inc.
+ */
+#ifndef __EROFS_BLOBRAW_H
+#define __EROFS_BLOBRAW_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "erofs/internal.h"
+
+void erofs_blobraw_remap_blkaddr(struct erofs_inode *inode);
+int erofs_blobraw_write_file(struct erofs_inode *inode, int fd);
+int erofs_blobraw_init();
+void erofs_blobraw_exit();
+int erofs_mkfs_dump_rawblob(struct erofs_sb_info *sbi, bool drop_blob);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
+
diff --git a/lib/Makefile.am b/lib/Makefile.am
index bdc74ad..c0804b3 100644
--- a/lib/Makefile.am
+++ b/lib/Makefile.am
@@ -3,6 +3,7 @@
 noinst_LTLIBRARIES = liberofs.la
 noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
       $(top_srcdir)/include/erofs/blobchunk.h \
+      $(top_srcdir)/include/erofs/blobraw.h \
       $(top_srcdir)/include/erofs/block_list.h \
       $(top_srcdir)/include/erofs/cache.h \
       $(top_srcdir)/include/erofs/compress.h \
@@ -33,7 +34,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \
 noinst_HEADERS += compressor.h
 liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \
 		      namei.c data.c compress.c compressor.c zmap.c decompress.c \
-		      compress_hints.c hashmap.c sha256.c blobchunk.c dir.c \
+		      compress_hints.c hashmap.c sha256.c blobchunk.c blobraw.c dir.c \
 		      fragments.c dedupe.c uuid_unparse.c uuid.c tar.c \
 		      block_list.c rebuild.c diskbuf.c bitops.c dedupe_ext.c
 
diff --git a/lib/blobraw.c b/lib/blobraw.c
new file mode 100755
index 0000000..248fc30
--- /dev/null
+++ b/lib/blobraw.c
@@ -0,0 +1,121 @@
+// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0
+/*
+ * erofs-utils/lib/blobraw.c
+ *
+ * Copyright (C) 2025, HUAWEI, Inc.
+ */
+#define _GNU_SOURCE
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "erofs/blobraw.h"
+#include "erofs/print.h"
+#include "erofs/cache.h"
+#include "liberofs_private.h"
+
+static int blobfile = -1;		/* tmp fd for blob buffer */
+static erofs_blk_t remapped_base;	/* metadata block end address */
+static erofs_off_t bloboff;		/* current blob file offset */
+
+void erofs_blobraw_remap_blkaddr(struct erofs_inode *inode)
+{
+	if (inode->i_size > 0)
+		inode->u.i_blkaddr += remapped_base;
+}
+
+int erofs_blobraw_write_file(struct erofs_inode *inode, int fd)
+{
+	static u8 zeroed[EROFS_MAX_BLOCK_SIZE];
+	struct erofs_sb_info *sbi = inode->sbi;
+	unsigned int padding;
+	erofs_off_t foff;
+	ssize_t length, ret, isize = inode->i_size;
+	u64 pos_in, pos_out = bloboff;
+
+	pos_in = 0;
+	do {
+		length = min_t(erofs_off_t, isize,  SSIZE_MAX);
+		ret = erofs_copy_file_range(fd, &pos_in,
+				blobfile, &pos_out, length);
+	} while (ret > 0 && (isize -= ret));
+
+	foff = lseek(blobfile, inode->i_size, SEEK_CUR);
+	DBG_BUGON(foff != bloboff + inode->i_size);
+
+	padding = erofs_blkoff(sbi, inode->i_size);
+	if (padding) {
+		padding = erofs_blksiz(sbi) - padding;
+		ret = __erofs_io_write(blobfile, zeroed, padding);
+		if (ret > 0 && ret != padding)
+			return -EIO;
+	}
+
+	inode->datalayout = EROFS_INODE_FLAT_PLAIN;
+	inode->u.i_blkaddr = erofs_blknr(sbi, bloboff);
+	bloboff += round_up(inode->i_size, erofs_blksiz(sbi));
+
+	return 0;
+}
+
+int erofs_mkfs_dump_rawblob(struct erofs_sb_info *sbi, bool drop_blob)
+{
+	struct erofs_buffer_head *bh;
+	erofs_off_t blob_size;
+	ssize_t length, ret = 0;
+	u64 pos_in, pos_out;
+
+	if (blobfile < 0) {
+		erofs_err("dump rawblob failed");
+		return -1;
+	}
+
+	blob_size = bloboff;
+
+	bh = erofs_balloc(sbi->bmgr, DATA, blob_size, 0);
+	if (IS_ERR(bh))
+		return PTR_ERR(bh);
+
+	erofs_mapbh(NULL, bh->block);
+
+	pos_out = erofs_btell(bh, false);
+	remapped_base = erofs_blknr(sbi, pos_out);
+	pos_out += sbi->bdev.offset;
+
+	/* copy the raw blob data */
+	if (!drop_blob) {
+		pos_in = 0;
+		do {
+			length = min_t(erofs_off_t, blob_size,  SSIZE_MAX);
+			ret = erofs_copy_file_range(blobfile, &pos_in,
+					sbi->bdev.fd, &pos_out, length);
+		} while (ret > 0 && (blob_size -= ret));
+
+		if (ret >= 0) {
+			if (blob_size) {
+				erofs_err("failed to append the remaining %llu-byte blob data",
+					  blob_size);
+				ret = -EIO;
+			} else {
+				ret = 0;
+			}
+		}
+	}
+
+	erofs_bdrop(bh, false);
+	return ret;
+}
+
+int erofs_blobraw_init()
+{
+	blobfile = erofs_tmpfile();
+	if (blobfile < 0)
+		return -errno;
+
+	return 0;
+}
+
+void erofs_blobraw_exit()
+{
+	if (blobfile >= 0)
+		close(blobfile);
+}
diff --git a/lib/inode.c b/lib/inode.c
index 108aa9e..39257c7 100644
--- a/lib/inode.c
+++ b/lib/inode.c
@@ -28,6 +28,7 @@
 #include "erofs/block_list.h"
 #include "erofs/compress_hints.h"
 #include "erofs/blobchunk.h"
+#include "erofs/blobraw.h"
 #include "erofs/fragments.h"
 #include "liberofs_private.h"
 
@@ -579,6 +580,8 @@ int erofs_write_unencoded_file(struct erofs_inode *inode, int fd, u64 fpos)
 		return erofs_blob_write_chunked_file(inode, fd, fpos);
 	}
 
+	if (cfg.c_meta_fix)
+		return erofs_blobraw_write_file(inode, fd);
 	/* fallback to all data uncompressed */
 	return write_uncompressed_file_from_fd(inode, fd);
 }
@@ -600,6 +603,9 @@ int erofs_iflush(struct erofs_inode *inode)
 	else
 		off = erofs_iloc(inode);
 
+	if (S_ISREG(inode->i_mode) && cfg.c_meta_fix)
+		erofs_blobraw_remap_blkaddr(inode);
+
 	if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
 	    S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode))
 		u1.rdev = cpu_to_le32(inode->u.i_rdev);
diff --git a/mkfs/main.c b/mkfs/main.c
index 42635c6..82db452 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -26,6 +26,7 @@
 #include "erofs/block_list.h"
 #include "erofs/compress_hints.h"
 #include "erofs/blobchunk.h"
+#include "erofs/blobraw.h"
 #include "erofs/fragments.h"
 #include "erofs/rebuild.h"
 #include "../lib/liberofs_private.h"
@@ -1422,6 +1423,12 @@ int main(int argc, char **argv)
 			return 1;
 	}
 
+	if (cfg.c_meta_fix) {
+		err = erofs_blobraw_init();
+		if (err)
+			return -1;
+	}
+
 	if (((erofstar.index_mode && !erofstar.headeronly_mode) &&
 	    !erofstar.mapfile) || cfg.c_blobdev_path) {
 		err = erofs_mkfs_init_devices(&g_sbi, 1);
@@ -1498,6 +1505,12 @@ int main(int argc, char **argv)
 			goto exit;
 	}
 
+	if (cfg.c_meta_fix) {
+		err = erofs_mkfs_dump_rawblob(&g_sbi, cfg.c_meta_fix);
+		if (err)
+			goto exit;
+	}
+
 	/* flush all buffers except for the superblock */
 	err = erofs_bflush(g_sbi.bmgr, NULL);
 	if (err)
@@ -1537,6 +1550,8 @@ exit:
 	erofs_cleanup_exclude_rules();
 	if (cfg.c_chunkbits)
 		erofs_blob_exit();
+	if (cfg.c_meta_fix)
+		erofs_blobraw_exit();
 	erofs_packedfile_exit(&g_sbi);
 	erofs_xattr_cleanup_name_prefixes();
 	erofs_rebuild_cleanup();
-- 
2.22.0

Powered by blists - more mailing lists