From 534eda7f8678e5b8fc8ca0f5cf0d9d7a932e0d48 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 22 Nov 2021 07:58:40 +0800 Subject: erofs-utils: fix Makefile for fsck.erofs manpage Add the missing dependency for fsck.erofs manpage. Link: https://lore.kernel.org/r/20211121235840.17600-1-xiang@kernel.org Fixes: f44043561491 ("erofs-utils: introduce fsck.erofs") Signed-off-by: Gao Xiang --- man/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/Makefile.am b/man/Makefile.am index 769b557..4628b85 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -dist_man_MANS = mkfs.erofs.1 dump.erofs.1 +dist_man_MANS = mkfs.erofs.1 dump.erofs.1 fsck.erofs.1 EXTRA_DIST = erofsfuse.1 if ENABLE_FUSE -- cgit v1.2.3 From 57519a9cbe00e1563c2171bef493519a910e96ad Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 22 Nov 2021 08:05:14 +0800 Subject: erofs-utils: add fuse/macosx.h to noinst_HEADERS Fix distcheck recompilation. Link: https://lore.kernel.org/r/20211122000514.27680-1-xiang@kernel.org Fixes: 95801d47cbde ("erofs-utils: fix macOS build & functionality") Signed-off-by: Gao Xiang --- fuse/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 7b007f3..8a2d472 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0+ AUTOMAKE_OPTIONS = foreign +noinst_HEADERS = $(top_srcdir)/fuse/macosx.h bin_PROGRAMS = erofsfuse erofsfuse_SOURCES = dir.c main.c erofsfuse_CFLAGS = -Wall -Werror -I$(top_srcdir)/include -- cgit v1.2.3 From 17eb5c417bd56be4b2f7627c2d6879fbad6a86d6 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 22 Nov 2021 07:48:48 +0800 Subject: erofs-utils: dump: fix de->nid issues As David Michael reported, " In file included from main.c:11: main.c: In function 'erofs_checkdirent': ../include/erofs/print.h:68:25: error: format '%llu' expects argument of type 'long long unsigned int', but argument 3 has type '__le64' {aka 'long unsigned int'} [-Werror=format=] 68 | " " PR_FMT_FUNC_LINE(fmt), \ | ^~~~~~ main.c:264:17: note: in expansion of macro 'erofs_err' 264 | erofs_err("invalid file type %llu", de->nid); | ^~~~~~~~~ main.c: In function 'erofs_read_dirent': ../include/erofs/print.h:68:25: error: format '%llu' expects argument of type 'long long unsigned int', but argument 3 has type '__le64' {aka 'long unsigned int'} [-Werror=format=] 68 | " " PR_FMT_FUNC_LINE(fmt), \ | ^~~~~~ main.c:303:25: note: in expansion of macro 'erofs_err' 303 | erofs_err("parse dir nid %llu error occurred\n", | ^~~~~~~~~ cc1: all warnings being treated as errors " Also there are many de->nid lacking of endianness handling. Should fix them together. Link: https://lore.kernel.org/r/20211121234848.12663-1-xiang@kernel.org Fixes: cf8be8a4352a ("erofs-utils: dump: add feature for collecting filesystem statistics") Cc: Wang Qi Cc: Guo Xuenan Reported-by: David Michael Signed-off-by: Gao Xiang --- dump/main.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/dump/main.c b/dump/main.c index b7560ec..f85903b 100644 --- a/dump/main.c +++ b/dump/main.c @@ -242,11 +242,12 @@ static inline int erofs_checkdirent(struct erofs_dirent *de, { int dname_len; unsigned int nameoff = le16_to_cpu(de->nameoff); + erofs_nid_t nid = le64_to_cpu(de->nid); if (nameoff < sizeof(struct erofs_dirent) || nameoff >= PAGE_SIZE) { erofs_err("invalid de[0].nameoff %u @ nid %llu", - nameoff, de->nid | 0ULL); + nameoff, nid | 0ULL); return -EFSCORRUPTED; } @@ -255,13 +256,12 @@ static inline int erofs_checkdirent(struct erofs_dirent *de, /* a corrupted entry is found */ if (nameoff + dname_len > maxsize || dname_len > EROFS_NAME_LEN) { - erofs_err("bogus dirent @ nid %llu", - le64_to_cpu(de->nid) | 0ULL); + erofs_err("bogus dirent @ nid %llu", nid | 0ULL); DBG_BUGON(1); return -EFSCORRUPTED; } if (de->file_type >= EROFS_FT_MAX) { - erofs_err("invalid file type %llu", de->nid); + erofs_err("invalid file type %llu", nid | 0ULL); return -EFSCORRUPTED; } return dname_len; @@ -273,7 +273,7 @@ static int erofs_read_dirent(struct erofs_dirent *de, { int err; erofs_off_t occupied_size = 0; - struct erofs_inode inode = { .nid = de->nid }; + struct erofs_inode inode = { .nid = le64_to_cpu(de->nid) }; stats.files++; stats.file_category_stat[de->file_type]++; @@ -296,12 +296,12 @@ static int erofs_read_dirent(struct erofs_dirent *de, update_file_size_statatics(occupied_size, inode.i_size); } - if ((de->file_type == EROFS_FT_DIR) - && de->nid != nid && de->nid != parent_nid) { - err = erofs_read_dir(de->nid, nid); + if (de->file_type == EROFS_FT_DIR && inode.nid != nid && + inode.nid != parent_nid) { + err = erofs_read_dir(inode.nid, nid); if (err) { erofs_err("parse dir nid %llu error occurred\n", - de->nid); + inode.nid | 0ULL); return err; } } @@ -338,7 +338,8 @@ static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid) int ret; /* skip "." and ".." dentry */ - if (de->nid == nid || de->nid == parent_nid) { + if (le64_to_cpu(de->nid) == nid || + le64_to_cpu(de->nid) == parent_nid) { de++; continue; } @@ -399,18 +400,18 @@ static int erofs_get_pathname(erofs_nid_t nid, erofs_nid_t parent_nid, if (len < 0) return len; - if (de->nid == target) { + if (le64_to_cpu(de->nid) == target) { memcpy(path + pos, dname, len); path[pos + len] = '\0'; return 0; } if (de->file_type == EROFS_FT_DIR && - de->nid != parent_nid && - de->nid != nid) { + le64_to_cpu(de->nid) != parent_nid && + le64_to_cpu(de->nid) != nid) { memcpy(path + pos, dname, len); - err = erofs_get_pathname(de->nid, nid, - target, path, pos + len); + err = erofs_get_pathname(le64_to_cpu(de->nid), + nid, target, path, pos + len); if (!err) return 0; memset(path + pos, 0, len); -- cgit v1.2.3 From db31e3f8f2deb9f9efdf9f91c2e1f428b184238e Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Sat, 20 Nov 2021 21:39:19 -0800 Subject: erofs-utils: mark certain callback function pointers as const Global variables aren't bad, until you start mutating them in multiple places. Link: https://lore.kernel.org/r/20211121053920.2580751-3-zhangkelvin@google.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- include/erofs/cache.h | 8 ++++---- lib/cache.c | 6 +++--- lib/compressor.c | 2 +- lib/compressor.h | 8 ++++---- lib/compressor_liblzma.c | 2 +- lib/compressor_lz4.c | 2 +- lib/compressor_lz4hc.c | 2 +- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/include/erofs/cache.h b/include/erofs/cache.h index b19d54e..87cd51d 100644 --- a/include/erofs/cache.h +++ b/include/erofs/cache.h @@ -32,7 +32,7 @@ struct erofs_buffer_head { struct erofs_buffer_block *block; erofs_off_t off; - struct erofs_bhops *op; + const struct erofs_bhops *op; void *fsprivate; }; @@ -68,9 +68,9 @@ static inline const int get_alignsize(int type, int *type_ret) return -EINVAL; } -extern struct erofs_bhops erofs_drop_directly_bhops; -extern struct erofs_bhops erofs_skip_write_bhops; -extern struct erofs_bhops erofs_buf_write_bhops; +extern const struct erofs_bhops erofs_drop_directly_bhops; +extern const struct erofs_bhops erofs_skip_write_bhops; +extern const struct erofs_bhops erofs_buf_write_bhops; static inline erofs_off_t erofs_btell(struct erofs_buffer_head *bh, bool end) { diff --git a/lib/cache.c b/lib/cache.c index 8016e38..83d591f 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -26,7 +26,7 @@ static bool erofs_bh_flush_drop_directly(struct erofs_buffer_head *bh) return erofs_bh_flush_generic_end(bh); } -struct erofs_bhops erofs_drop_directly_bhops = { +const struct erofs_bhops erofs_drop_directly_bhops = { .flush = erofs_bh_flush_drop_directly, }; @@ -35,7 +35,7 @@ static bool erofs_bh_flush_skip_write(struct erofs_buffer_head *bh) return false; } -struct erofs_bhops erofs_skip_write_bhops = { +const struct erofs_bhops erofs_skip_write_bhops = { .flush = erofs_bh_flush_skip_write, }; @@ -58,7 +58,7 @@ static bool erofs_bh_flush_buf_write(struct erofs_buffer_head *bh) return erofs_bh_flush_generic_end(bh); } -struct erofs_bhops erofs_buf_write_bhops = { +const struct erofs_bhops erofs_buf_write_bhops = { .flush = erofs_bh_flush_buf_write, }; diff --git a/lib/compressor.c b/lib/compressor.c index ad12cdf..6362825 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -10,7 +10,7 @@ #define EROFS_CONFIG_COMPR_DEF_BOUNDARY (128) -static struct erofs_compressor *compressors[] = { +static const struct erofs_compressor *compressors[] = { #if LZ4_ENABLED #if LZ4HC_ENABLED &erofs_compressor_lz4hc, diff --git a/lib/compressor.h b/lib/compressor.h index aa85ae0..1ea2724 100644 --- a/lib/compressor.h +++ b/lib/compressor.h @@ -27,7 +27,7 @@ struct erofs_compressor { }; struct erofs_compress { - struct erofs_compressor *alg; + const struct erofs_compressor *alg; unsigned int compress_threshold; unsigned int compression_level; @@ -41,9 +41,9 @@ struct erofs_compress { }; /* list of compression algorithms */ -extern struct erofs_compressor erofs_compressor_lz4; -extern struct erofs_compressor erofs_compressor_lz4hc; -extern struct erofs_compressor erofs_compressor_lzma; +extern const struct erofs_compressor erofs_compressor_lz4; +extern const struct erofs_compressor erofs_compressor_lz4hc; +extern const struct erofs_compressor erofs_compressor_lzma; int erofs_compress_destsize(struct erofs_compress *c, void *src, unsigned int *srcsize, diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c index 40a05ef..578ba06 100644 --- a/lib/compressor_liblzma.c +++ b/lib/compressor_liblzma.c @@ -96,7 +96,7 @@ static int erofs_compressor_liblzma_init(struct erofs_compress *c) return 0; } -struct erofs_compressor erofs_compressor_lzma = { +const struct erofs_compressor erofs_compressor_lzma = { .name = "lzma", .default_level = LZMA_PRESET_DEFAULT, .best_level = LZMA_PRESET_EXTREME, diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c index f6832be..fc8c23c 100644 --- a/lib/compressor_lz4.c +++ b/lib/compressor_lz4.c @@ -37,7 +37,7 @@ static int compressor_lz4_init(struct erofs_compress *c) return 0; } -struct erofs_compressor erofs_compressor_lz4 = { +const struct erofs_compressor erofs_compressor_lz4 = { .name = "lz4", .default_level = 0, .best_level = 0, diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c index fd801ab..3f68b00 100644 --- a/lib/compressor_lz4hc.c +++ b/lib/compressor_lz4hc.c @@ -59,7 +59,7 @@ static int compressor_lz4hc_setlevel(struct erofs_compress *c, return 0; } -struct erofs_compressor erofs_compressor_lz4hc = { +const struct erofs_compressor erofs_compressor_lz4hc = { .name = "lz4hc", .default_level = LZ4HC_CLEVEL_DEFAULT, .best_level = LZ4HC_CLEVEL_MAX, -- cgit v1.2.3 From c816397a43b050fff84337b4484e67f72cb5501a Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Wed, 1 Dec 2021 16:27:46 +0800 Subject: erofs-utils: update AUTHORS Add myself to the AUTHORS file. Link: https://lore.kernel.org/r/20211201082746.1642-1-huangjianan@oppo.com Acked-by: Chao Yu Acked-by: Gao Xiang Signed-off-by: Huang Jianan Signed-off-by: Gao Xiang --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 812042a..6b41df8 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,6 +1,7 @@ EROFS USERSPACE UTILITIES M: Li Guifu M: Gao Xiang +M: Huang Jianan R: Chao Yu R: Miao Xie R: Fang Wei -- cgit v1.2.3 From 519dbd2368240c4c6d159459ab5614e1d9540c3c Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Wed, 1 Dec 2021 21:53:15 +0800 Subject: erofs-utils: make liberofs more C++ friendly 1. Add extern "C" wrappers to headers, so that they can be included from C++ 2. Add const keywords to certain pointer type params Link: https://lore.kernel.org/r/20211201135315.3732-1-xiang@kernel.org Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- include/erofs/blobchunk.h | 9 +++++++++ include/erofs/block_list.h | 10 ++++++++++ include/erofs/cache.h | 9 +++++++++ include/erofs/compress.h | 9 +++++++++ include/erofs/compress_hints.h | 10 ++++++++++ include/erofs/config.h | 20 +++++++++----------- include/erofs/decompress.h | 9 +++++++++ include/erofs/defs.h | 17 ++++++++++++++++- include/erofs/err.h | 9 +++++++++ include/erofs/exclude.h | 10 ++++++++++ include/erofs/flex-array.h | 9 +++++++++ include/erofs/hashmap.h | 9 +++++++++ include/erofs/hashtable.h | 9 +++++++++ include/erofs/inode.h | 9 +++++++++ include/erofs/internal.h | 9 +++++++++ include/erofs/io.h | 11 +++++++++++ include/erofs/list.h | 10 ++++++++++ include/erofs/print.h | 9 +++++++++ include/erofs/trace.h | 9 +++++++++ include/erofs/xattr.h | 9 +++++++++ lib/Makefile.am | 3 ++- lib/config.c | 1 + lib/inode.c | 1 + lib/liberofs_private.h | 13 +++++++++++++ lib/xattr.c | 1 + mkfs/main.c | 1 + 26 files changed, 212 insertions(+), 13 deletions(-) create mode 100644 lib/liberofs_private.h diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h index 59a4701..4e1ae79 100644 --- a/include/erofs/blobchunk.h +++ b/include/erofs/blobchunk.h @@ -7,6 +7,11 @@ #ifndef __EROFS_BLOBCHUNK_H #define __EROFS_BLOBCHUNK_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "erofs/internal.h" int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, erofs_off_t off); @@ -16,4 +21,8 @@ void erofs_blob_exit(void); int erofs_blob_init(const char *blobfile_path); int erofs_generate_devtable(void); +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h index 40df228..ca8053e 100644 --- a/include/erofs/block_list.h +++ b/include/erofs/block_list.h @@ -6,6 +6,11 @@ #ifndef __EROFS_BLOCK_LIST_H #define __EROFS_BLOCK_LIST_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "internal.h" #ifdef WITH_ANDROID @@ -29,4 +34,9 @@ erofs_droid_blocklist_write_extent(struct erofs_inode *inode, erofs_blk_t blk_start, erofs_blk_t nblocks, bool first_extent, bool last_extent) {} #endif + +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/cache.h b/include/erofs/cache.h index 87cd51d..7957ee5 100644 --- a/include/erofs/cache.h +++ b/include/erofs/cache.h @@ -8,6 +8,11 @@ #ifndef __EROFS_CACHE_H #define __EROFS_CACHE_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "internal.h" struct erofs_buffer_head; @@ -104,4 +109,8 @@ bool erofs_bflush(struct erofs_buffer_block *bb); void erofs_bdrop(struct erofs_buffer_head *bh, bool tryrevoke); +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/compress.h b/include/erofs/compress.h index 4434aaa..fdbf5ff 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -7,6 +7,11 @@ #ifndef __EROFS_COMPRESS_H #define __EROFS_COMPRESS_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "internal.h" /* workaround for an upstream lz4 compression issue, which can crash us */ @@ -21,4 +26,8 @@ int z_erofs_compress_exit(void); const char *z_erofs_list_available_compressors(unsigned int i); +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/compress_hints.h b/include/erofs/compress_hints.h index a5772c7..43f80e1 100644 --- a/include/erofs/compress_hints.h +++ b/include/erofs/compress_hints.h @@ -6,6 +6,11 @@ #ifndef __EROFS_COMPRESS_HINTS_H #define __EROFS_COMPRESS_HINTS_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "erofs/internal.h" #include #include @@ -20,4 +25,9 @@ struct erofs_compress_hints { bool z_erofs_apply_compress_hints(struct erofs_inode *inode); void erofs_cleanup_compress_hints(void); int erofs_load_compress_hints(void); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/config.h b/include/erofs/config.h index 2040dc6..cb064b6 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -7,20 +7,14 @@ #ifndef __EROFS_CONFIG_H #define __EROFS_CONFIG_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "defs.h" #include "err.h" -#ifdef HAVE_LIBSELINUX -#include -#include -#endif - -#ifdef WITH_ANDROID -#include -#include -#include -#include -#endif enum { FORCE_INODE_COMPACT = 1, @@ -96,4 +90,8 @@ static inline int erofs_selabel_open(const char *file_contexts) } #endif +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h index 3d0d963..e649c80 100644 --- a/include/erofs/decompress.h +++ b/include/erofs/decompress.h @@ -6,6 +6,11 @@ #ifndef __EROFS_DECOMPRESS_H #define __EROFS_DECOMPRESS_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "internal.h" struct z_erofs_decompress_req { @@ -25,4 +30,8 @@ struct z_erofs_decompress_req { int z_erofs_decompress(struct z_erofs_decompress_req *rq); +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/defs.h b/include/erofs/defs.h index 6398cbb..4db237f 100644 --- a/include/erofs/defs.h +++ b/include/erofs/defs.h @@ -8,6 +8,11 @@ #ifndef __EROFS_DEFS_H #define __EROFS_DEFS_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include #include #include @@ -82,7 +87,9 @@ typedef int64_t s64; #endif #endif -#ifndef __OPTIMIZE__ +#ifdef __cplusplus +#define BUILD_BUG_ON(condition) static_assert(!(condition)) +#elif !defined(__OPTIMIZE__) #define BUILD_BUG_ON(condition) ((void)sizeof(char[1 - 2 * !!(condition)])) #else #define BUILD_BUG_ON(condition) assert(!(condition)) @@ -110,6 +117,8 @@ typedef int64_t s64; } \ ) +/* Can easily conflict with C++'s std::min */ +#ifndef __cplusplus #define min(x, y) ({ \ typeof(x) _min1 = (x); \ typeof(y) _min2 = (y); \ @@ -121,6 +130,7 @@ typedef int64_t s64; typeof(y) _max2 = (y); \ (void) (&_max1 == &_max2); \ _max1 > _max2 ? _max1 : _max2; }) +#endif /* * ..and if you can't take the strict types, you can specify one yourself. @@ -308,4 +318,9 @@ unsigned long __roundup_pow_of_two(unsigned long n) #define stat64 stat #define lstat64 lstat #endif + +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/err.h b/include/erofs/err.h index a33bdd1..18f152a 100644 --- a/include/erofs/err.h +++ b/include/erofs/err.h @@ -7,6 +7,11 @@ #ifndef __EROFS_ERR_H #define __EROFS_ERR_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include #define MAX_ERRNO (4095) @@ -28,4 +33,8 @@ static inline long PTR_ERR(const void *ptr) return (long) ptr; } +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h index 6930f2b..599f018 100644 --- a/include/erofs/exclude.h +++ b/include/erofs/exclude.h @@ -5,6 +5,11 @@ #ifndef __EROFS_EXCLUDE_H #define __EROFS_EXCLUDE_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include #include @@ -21,4 +26,9 @@ void erofs_cleanup_exclude_rules(void); int erofs_parse_exclude_path(const char *args, bool is_regex); struct erofs_exclude_rule *erofs_is_exclude_path(const char *dir, const char *name); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/flex-array.h b/include/erofs/flex-array.h index 59168d0..9b1642f 100644 --- a/include/erofs/flex-array.h +++ b/include/erofs/flex-array.h @@ -2,6 +2,11 @@ #ifndef __EROFS_FLEX_ARRAY_H #define __EROFS_FLEX_ARRAY_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include #include #include @@ -144,4 +149,8 @@ static inline size_t st_add(size_t a, size_t b) #define FLEXPTR_ALLOC_STR(x, ptrname, str) \ FLEXPTR_ALLOC_MEM((x), ptrname, (str), strlen(str)) +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/hashmap.h b/include/erofs/hashmap.h index 024a14e..3d38578 100644 --- a/include/erofs/hashmap.h +++ b/include/erofs/hashmap.h @@ -2,6 +2,11 @@ #ifndef __EROFS_HASHMAP_H #define __EROFS_HASHMAP_H +#ifdef __cplusplus +extern "C" +{ +#endif + /* Copied from https://github.com/git/git.git */ #include #include @@ -100,4 +105,8 @@ static inline const char *strintern(const char *string) return memintern(string, strlen(string)); } +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/hashtable.h b/include/erofs/hashtable.h index 90eb84e..3c4dfc1 100644 --- a/include/erofs/hashtable.h +++ b/include/erofs/hashtable.h @@ -5,6 +5,11 @@ #ifndef __EROFS_HASHTABLE_H #define __EROFS_HASHTABLE_H +#ifdef __cplusplus +extern "C" +{ +#endif + /* * Fast hashing routine for ints, longs and pointers. * (C) 2002 Nadia Yvette Chambers, IBM @@ -380,4 +385,8 @@ static inline void hash_del(struct hlist_node *node) #define hash_for_each_possible(name, obj, member, key) \ hlist_for_each_entry(obj, &name[hash_min(key, HASH_BITS(name))], member) +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/inode.h b/include/erofs/inode.h index d5343c2..e23d65f 100644 --- a/include/erofs/inode.h +++ b/include/erofs/inode.h @@ -8,6 +8,11 @@ #ifndef __EROFS_INODE_H #define __EROFS_INODE_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "erofs/internal.h" unsigned char erofs_mode_to_ftype(umode_t mode); @@ -17,4 +22,8 @@ erofs_nid_t erofs_lookupnid(struct erofs_inode *inode); struct erofs_inode *erofs_mkfs_build_tree_from_path(struct erofs_inode *parent, const char *path); +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 666d1f2..a68de32 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -7,6 +7,11 @@ #ifndef __EROFS_INTERNAL_H #define __EROFS_INTERNAL_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "list.h" #include "err.h" @@ -331,4 +336,8 @@ static inline u32 erofs_crc32c(u32 crc, const u8 *in, size_t len) return crc; } +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/io.h b/include/erofs/io.h index 10a3681..6f51e06 100644 --- a/include/erofs/io.h +++ b/include/erofs/io.h @@ -7,7 +7,14 @@ #ifndef __EROFS_IO_H #define __EROFS_IO_H +#ifdef __cplusplus +extern "C" +{ +#endif + +#ifndef _GNU_SOURCE #define _GNU_SOURCE +#endif #include #include "internal.h" @@ -47,4 +54,8 @@ static inline int blk_read(int device_id, void *buf, blknr_to_addr(nblocks)); } +#ifdef __cplusplus +} #endif + +#endif // EROFS_IO_H_ diff --git a/include/erofs/list.h b/include/erofs/list.h index d2bc704..fd5358d 100644 --- a/include/erofs/list.h +++ b/include/erofs/list.h @@ -7,6 +7,11 @@ #ifndef __EROFS_LIST_HEAD_H #define __EROFS_LIST_HEAD_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "defs.h" struct list_head { @@ -105,4 +110,9 @@ static inline int list_empty(struct list_head *head) &pos->member != (head); \ pos = n, n = list_next_entry(n, member)) + +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/print.h b/include/erofs/print.h index 91f864b..2213d1d 100644 --- a/include/erofs/print.h +++ b/include/erofs/print.h @@ -7,6 +7,11 @@ #ifndef __EROFS_PRINT_H #define __EROFS_PRINT_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "config.h" #include @@ -72,4 +77,8 @@ enum { #define erofs_dump(fmt, ...) fprintf(stderr, fmt, ##__VA_ARGS__) +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/trace.h b/include/erofs/trace.h index d70d236..893e16c 100644 --- a/include/erofs/trace.h +++ b/include/erofs/trace.h @@ -5,7 +5,16 @@ #ifndef __EROFS_TRACE_H #define __EROFS_TRACE_H +#ifdef __cplusplus +extern "C" +{ +#endif + #define trace_erofs_map_blocks_flatmode_enter(inode, map, flags) ((void)0) #define trace_erofs_map_blocks_flatmode_exit(inode, map, flags, ret) ((void)0) +#ifdef __cplusplus +} +#endif + #endif diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h index f0c4c26..8e68812 100644 --- a/include/erofs/xattr.h +++ b/include/erofs/xattr.h @@ -7,6 +7,11 @@ #ifndef __EROFS_XATTR_H #define __EROFS_XATTR_H +#ifdef __cplusplus +extern "C" +{ +#endif + #include "internal.h" #define EROFS_INODE_XATTR_ICOUNT(_size) ({\ @@ -44,4 +49,8 @@ int erofs_prepare_xattr_ibody(struct erofs_inode *inode); char *erofs_export_xattr_ibody(struct list_head *ixattrs, unsigned int size); int erofs_build_shared_xattrs_from_path(const char *path); +#ifdef __cplusplus +} +#endif + #endif diff --git a/lib/Makefile.am b/lib/Makefile.am index 395c712..67ba798 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -21,7 +21,8 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ $(top_srcdir)/include/erofs/print.h \ $(top_srcdir)/include/erofs/trace.h \ $(top_srcdir)/include/erofs/xattr.h \ - $(top_srcdir)/include/erofs/compress_hints.h + $(top_srcdir)/include/erofs/compress_hints.h \ + $(top_srcdir)/lib/liberofs_private.h noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ diff --git a/lib/config.c b/lib/config.c index 363dcc5..f1c8edf 100644 --- a/lib/config.c +++ b/lib/config.c @@ -8,6 +8,7 @@ #include #include "erofs/print.h" #include "erofs/internal.h" +#include "liberofs_private.h" struct erofs_configure cfg; struct erofs_sb_info sbi; diff --git a/lib/inode.c b/lib/inode.c index 2fa74e2..461c797 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -25,6 +25,7 @@ #include "erofs/block_list.h" #include "erofs/compress_hints.h" #include "erofs/blobchunk.h" +#include "liberofs_private.h" #define S_SHIFT 12 static unsigned char erofs_ftype_by_mode[S_IFMT >> S_SHIFT] = { diff --git a/lib/liberofs_private.h b/lib/liberofs_private.h new file mode 100644 index 0000000..c2312e8 --- /dev/null +++ b/lib/liberofs_private.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only OR Apache-2.0 */ + +#ifdef HAVE_LIBSELINUX +#include +#include +#endif + +#ifdef WITH_ANDROID +#include +#include +#include +#include +#endif diff --git a/lib/xattr.c b/lib/xattr.c index 196133a..00fb963 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -17,6 +17,7 @@ #include "erofs/xattr.h" #include "erofs/cache.h" #include "erofs/io.h" +#include "liberofs_private.h" #define EA_HASHTABLE_BITS 16 diff --git a/mkfs/main.c b/mkfs/main.c index 58a6441..90cedde 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -23,6 +23,7 @@ #include "erofs/block_list.h" #include "erofs/compress_hints.h" #include "erofs/blobchunk.h" +#include "../lib/liberofs_private.h" #ifdef HAVE_LIBUUID #include -- cgit v1.2.3 From 43c52e68d447c50096c9bad9e241791bb24981d8 Mon Sep 17 00:00:00 2001 From: Daeho Jeong Date: Thu, 2 Dec 2021 11:09:23 -0800 Subject: erofs-utils: loosen hash_64 compile restriction We've got the below "unused function" warning for 32bit compilation. So, fixed it. error: unused function 'hash_64' [-Werror,-Wunused-function] Link: https://lore.kernel.org/r/20211202190923.975767-1-daeho43@gmail.com Reviewed-by: Gao Xiang Signed-off-by: Daeho Jeong Signed-off-by: Gao Xiang --- include/erofs/hashtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/erofs/hashtable.h b/include/erofs/hashtable.h index 3c4dfc1..f4eca8d 100644 --- a/include/erofs/hashtable.h +++ b/include/erofs/hashtable.h @@ -256,7 +256,7 @@ static inline u32 hash_32(u32 val, unsigned int bits) return __hash_32(val) >> (32 - bits); } -static __always_inline u32 hash_64(u64 val, unsigned int bits) +static inline u32 hash_64(u64 val, unsigned int bits) { #if BITS_PER_LONG == 64 /* 64x64-bit multiply is efficient on all 64-bit processors */ -- cgit v1.2.3 From 49b344c5531a7819f282b0b1b6eb01682c0e41ae Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Thu, 2 Dec 2021 10:25:21 +0800 Subject: erofs-utils: lib: use GPL-2.0+ OR Apache-2.0 dual license Let's release liberofs under GPL-2.0+ OR Apache-2.0 license for better 3rd-party integration. Link: https://lore.kernel.org/r/20211202022521.4291-1-huangjianan@oppo.com Acked-by: Yue Hu Acked-by: Hu Weiwen Acked-by: Gao Xiang Acked-by: Guo Xuenan Acked-by: Li Guifu Signed-off-by: Huang Jianan Signed-off-by: Gao Xiang --- COPYING | 366 ++--------------------------------------- LICENSES/Apache-2.0 | 186 +++++++++++++++++++++ LICENSES/GPL-2.0 | 359 ++++++++++++++++++++++++++++++++++++++++ include/erofs/blobchunk.h | 2 +- include/erofs/block_list.h | 2 +- include/erofs/cache.h | 2 +- include/erofs/compress.h | 2 +- include/erofs/compress_hints.h | 2 +- include/erofs/config.h | 2 +- include/erofs/decompress.h | 2 +- include/erofs/defs.h | 2 +- include/erofs/err.h | 2 +- include/erofs/exclude.h | 2 +- include/erofs/inode.h | 2 +- include/erofs/internal.h | 2 +- include/erofs/io.h | 2 +- include/erofs/list.h | 2 +- include/erofs/print.h | 2 +- include/erofs/trace.h | 2 +- include/erofs/xattr.h | 2 +- lib/Makefile.am | 2 +- lib/blobchunk.c | 2 +- lib/block_list.c | 2 +- lib/cache.c | 2 +- lib/compress.c | 2 +- lib/compress_hints.c | 2 +- lib/compressor.c | 2 +- lib/compressor.h | 2 +- lib/compressor_liblzma.c | 2 +- lib/compressor_lz4.c | 2 +- lib/compressor_lz4hc.c | 2 +- lib/config.c | 2 +- lib/data.c | 2 +- lib/decompress.c | 2 +- lib/exclude.c | 2 +- lib/inode.c | 2 +- lib/io.c | 2 +- lib/namei.c | 2 +- lib/super.c | 2 +- lib/xattr.c | 2 +- lib/zmap.c | 2 +- 41 files changed, 594 insertions(+), 393 deletions(-) create mode 100644 LICENSES/Apache-2.0 create mode 100644 LICENSES/GPL-2.0 diff --git a/COPYING b/COPYING index b7eaaf4..8767cae 100644 --- a/COPYING +++ b/COPYING @@ -1,359 +1,15 @@ -Valid-License-Identifier: GPL-2.0 -Valid-License-Identifier: GPL-2.0-only -Valid-License-Identifier: GPL-2.0+ -Valid-License-Identifier: GPL-2.0-or-later -SPDX-URL: https://spdx.org/licenses/GPL-2.0.html -Usage-Guide: - To use this license in source code, put one of the following SPDX - tag/value pairs into a comment according to the placement - guidelines in the licensing rules documentation. - For 'GNU General Public License (GPL) version 2 only' use: - SPDX-License-Identifier: GPL-2.0 - or - SPDX-License-Identifier: GPL-2.0-only - For 'GNU General Public License (GPL) version 2 or any later version' use: - SPDX-License-Identifier: GPL-2.0+ - or - SPDX-License-Identifier: GPL-2.0-or-later -License-Text: +erofs-utils uses two different license patterns: - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + - most liberofs files in `lib` and `include` directories + use GPL-2.0+ OR Apache-2.0 dual license; - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. + - all other files use GPL-2.0+ license, unless + explicitly stated otherwise. - Preamble +Relevant licenses can be found in the LICENSES directory. - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General -Public License instead of this License. +This model is selected to emphasize that +files in `lib` and `include` directory are designed to be included into +3rd-party applications, while all other files, are intended to be used +"as is", as part of their intended scenarios, with no intention to +support 3rd-party integration use cases. diff --git a/LICENSES/Apache-2.0 b/LICENSES/Apache-2.0 new file mode 100644 index 0000000..f6c1877 --- /dev/null +++ b/LICENSES/Apache-2.0 @@ -0,0 +1,186 @@ +Valid-License-Identifier: Apache-2.0 +SPDX-URL: https://spdx.org/licenses/Apache-2.0.html +Usage-Guide: + The Apache-2.0 may only be used for dual-licensed files where the other + license is GPL2 compatible. If you end up using this it MUST be used + together with a GPL2 compatible license using "OR". + To use the Apache License version 2.0 put the following SPDX tag/value + pair into a comment according to the placement guidelines in the + licensing rules documentation: + SPDX-License-Identifier: Apache-2.0 +License-Text: + +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the +copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other +entities that control, are controlled by, or are under common control with +that entity. For the purposes of this definition, "control" means (i) the +power, direct or indirect, to cause the direction or management of such +entity, whether by contract or otherwise, or (ii) ownership of fifty +percent (50%) or more of the outstanding shares, or (iii) beneficial +ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation source, +and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation +or translation of a Source form, including but not limited to compiled +object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, +made available under the License, as indicated by a copyright notice that +is included in or attached to the work (an example is provided in the +Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, +that is based on (or derived from) the Work and for which the editorial +revisions, annotations, elaborations, or other modifications represent, as +a whole, an original work of authorship. For the purposes of this License, +Derivative Works shall not include works that remain separable from, or +merely link (or bind by name) to the interfaces of, the Work and Derivative +Works thereof. + +"Contribution" shall mean any work of authorship, including the original +version of the Work and any modifications or additions to that Work or +Derivative Works thereof, that is intentionally submitted to Licensor for +inclusion in the Work by the copyright owner or by an individual or Legal +Entity authorized to submit on behalf of the copyright owner. For the +purposes of this definition, "submitted" means any form of electronic, +verbal, or written communication sent to the Licensor or its +representatives, including but not limited to communication on electronic +mailing lists, source code control systems, and issue tracking systems that +are managed by, or on behalf of, the Licensor for the purpose of discussing +and improving the Work, but excluding communication that is conspicuously +marked or otherwise designated in writing by the copyright owner as "Not a +Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on +behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable copyright license to + reproduce, prepare Derivative Works of, publicly display, publicly + perform, sublicense, and distribute the Work and such Derivative Works + in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this + License, each Contributor hereby grants to You a perpetual, worldwide, + non-exclusive, no-charge, royalty-free, irrevocable (except as stated in + this section) patent license to make, have made, use, offer to sell, + sell, import, and otherwise transfer the Work, where such license + applies only to those patent claims licensable by such Contributor that + are necessarily infringed by their Contribution(s) alone or by + combination of their Contribution(s) with the Work to which such + Contribution(s) was submitted. If You institute patent litigation + against any entity (including a cross-claim or counterclaim in a + lawsuit) alleging that the Work or a Contribution incorporated within + the Work constitutes direct or contributory patent infringement, then + any patent licenses granted to You under this License for that Work + shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or + Derivative Works thereof in any medium, with or without modifications, + and in Source or Object form, provided that You meet the following + conditions: + + a. You must give any other recipients of the Work or Derivative Works a + copy of this License; and + + b. You must cause any modified files to carry prominent notices stating + that You changed the files; and + + c. You must retain, in the Source form of any Derivative Works that You + distribute, all copyright, patent, trademark, and attribution notices + from the Source form of the Work, excluding those notices that do not + pertain to any part of the Derivative Works; and + + d. If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained within + such NOTICE file, excluding those notices that do not pertain to any + part of the Derivative Works, in at least one of the following + places: within a NOTICE text file distributed as part of the + Derivative Works; within the Source form or documentation, if + provided along with the Derivative Works; or, within a display + generated by the Derivative Works, if and wherever such third-party + notices normally appear. The contents of the NOTICE file are for + informational purposes only and do not modify the License. You may + add Your own attribution notices within Derivative Works that You + distribute, alongside or as an addendum to the NOTICE text from the + Work, provided that such additional attribution notices cannot be + construed as modifying the License. + + You may add Your own copyright statement to Your modifications and may + provide additional or different license terms and conditions for use, + reproduction, or distribution of Your modifications, or for any such + Derivative Works as a whole, provided Your use, reproduction, and + distribution of the Work otherwise complies with the conditions stated + in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any + Contribution intentionally submitted for inclusion in the Work by You to + the Licensor shall be under the terms and conditions of this License, + without any additional terms or conditions. Notwithstanding the above, + nothing herein shall supersede or modify the terms of any separate + license agreement you may have executed with Licensor regarding such + Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to + in writing, Licensor provides the Work (and each Contributor provides + its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS + OF ANY KIND, either express or implied, including, without limitation, + any warranties or conditions of TITLE, NON-INFRINGEMENT, + MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely + responsible for determining the appropriateness of using or + redistributing the Work and assume any risks associated with Your + exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether + in tort (including negligence), contract, or otherwise, unless required + by applicable law (such as deliberate and grossly negligent acts) or + agreed to in writing, shall any Contributor be liable to You for + damages, including any direct, indirect, special, incidental, or + consequential damages of any character arising as a result of this + License or out of the use or inability to use the Work (including but + not limited to damages for loss of goodwill, work stoppage, computer + failure or malfunction, or any and all other commercial damages or + losses), even if such Contributor has been advised of the possibility of + such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the + Work or Derivative Works thereof, You may choose to offer, and charge a + fee for, acceptance of support, warranty, indemnity, or other liability + obligations and/or rights consistent with this License. However, in + accepting such obligations, You may act only on Your own behalf and on + Your sole responsibility, not on behalf of any other Contributor, and + only if You agree to indemnify, defend, and hold each Contributor + harmless for any liability incurred by, or claims asserted against, such + Contributor by reason of your accepting any such warranty or additional + liability. + +END OF TERMS AND CONDITIONS diff --git a/LICENSES/GPL-2.0 b/LICENSES/GPL-2.0 new file mode 100644 index 0000000..b7eaaf4 --- /dev/null +++ b/LICENSES/GPL-2.0 @@ -0,0 +1,359 @@ +Valid-License-Identifier: GPL-2.0 +Valid-License-Identifier: GPL-2.0-only +Valid-License-Identifier: GPL-2.0+ +Valid-License-Identifier: GPL-2.0-or-later +SPDX-URL: https://spdx.org/licenses/GPL-2.0.html +Usage-Guide: + To use this license in source code, put one of the following SPDX + tag/value pairs into a comment according to the placement + guidelines in the licensing rules documentation. + For 'GNU General Public License (GPL) version 2 only' use: + SPDX-License-Identifier: GPL-2.0 + or + SPDX-License-Identifier: GPL-2.0-only + For 'GNU General Public License (GPL) version 2 or any later version' use: + SPDX-License-Identifier: GPL-2.0+ + or + SPDX-License-Identifier: GPL-2.0-or-later +License-Text: + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/include/erofs/blobchunk.h b/include/erofs/blobchunk.h index 4e1ae79..49cb7bf 100644 --- a/include/erofs/blobchunk.h +++ b/include/erofs/blobchunk.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * erofs-utils/lib/blobchunk.h * diff --git a/include/erofs/block_list.h b/include/erofs/block_list.h index ca8053e..78fab44 100644 --- a/include/erofs/block_list.h +++ b/include/erofs/block_list.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu diff --git a/include/erofs/cache.h b/include/erofs/cache.h index 7957ee5..de12399 100644 --- a/include/erofs/cache.h +++ b/include/erofs/cache.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/compress.h b/include/erofs/compress.h index fdbf5ff..21eac57 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/compress_hints.h b/include/erofs/compress_hints.h index 43f80e1..659c5b6 100644 --- a/include/erofs/compress_hints.h +++ b/include/erofs/compress_hints.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/include/erofs/config.h b/include/erofs/config.h index cb064b6..e41c079 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/decompress.h b/include/erofs/decompress.h index e649c80..82bf7b8 100644 --- a/include/erofs/decompress.h +++ b/include/erofs/decompress.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/include/erofs/defs.h b/include/erofs/defs.h index 4db237f..e745bc0 100644 --- a/include/erofs/defs.h +++ b/include/erofs/defs.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/err.h b/include/erofs/err.h index 18f152a..08b0bdb 100644 --- a/include/erofs/err.h +++ b/include/erofs/err.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/exclude.h b/include/erofs/exclude.h index 599f018..3f17032 100644 --- a/include/erofs/exclude.h +++ b/include/erofs/exclude.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Created by Li Guifu */ diff --git a/include/erofs/inode.h b/include/erofs/inode.h index e23d65f..79b39b0 100644 --- a/include/erofs/inode.h +++ b/include/erofs/inode.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/internal.h b/include/erofs/internal.h index a68de32..e6beb8c 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/io.h b/include/erofs/io.h index 6f51e06..0f58c70 100644 --- a/include/erofs/io.h +++ b/include/erofs/io.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/list.h b/include/erofs/list.h index fd5358d..2a0e961 100644 --- a/include/erofs/list.h +++ b/include/erofs/list.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/print.h b/include/erofs/print.h index 2213d1d..f188a6b 100644 --- a/include/erofs/print.h +++ b/include/erofs/print.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/include/erofs/trace.h b/include/erofs/trace.h index 893e16c..398e331 100644 --- a/include/erofs/trace.h +++ b/include/erofs/trace.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2020 Gao Xiang */ diff --git a/include/erofs/xattr.h b/include/erofs/xattr.h index 8e68812..226e984 100644 --- a/include/erofs/xattr.h +++ b/include/erofs/xattr.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Originally contributed by an anonymous person, * heavily changed by Li Guifu diff --git a/lib/Makefile.am b/lib/Makefile.am index 67ba798..c745e49 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0+ +# SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 noinst_LTLIBRARIES = liberofs.la noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ diff --git a/lib/blobchunk.c b/lib/blobchunk.c index b605b0b..5e9a88a 100644 --- a/lib/blobchunk.c +++ b/lib/blobchunk.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * erofs-utils/lib/blobchunk.c * diff --git a/lib/block_list.c b/lib/block_list.c index 87609a9..896fb01 100644 --- a/lib/block_list.c +++ b/lib/block_list.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2021, Coolpad Group Limited. * Created by Yue Hu diff --git a/lib/cache.c b/lib/cache.c index 83d591f..f820c0b 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/compress.c b/lib/compress.c index 98be7a2..6f16e0c 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/compress_hints.c b/lib/compress_hints.c index 81a8ac9..25adf35 100644 --- a/lib/compress_hints.c +++ b/lib/compress_hints.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2008-2021, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/lib/compressor.c b/lib/compressor.c index 6362825..a4d696e 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/compressor.h b/lib/compressor.h index 1ea2724..fd28dfe 100644 --- a/lib/compressor.h +++ b/lib/compressor.h @@ -1,4 +1,4 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c index 578ba06..42f3ac6 100644 --- a/lib/compressor_liblzma.c +++ b/lib/compressor_liblzma.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * erofs-utils/lib/compressor_liblzma.c * diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c index fc8c23c..0936d03 100644 --- a/lib/compressor_lz4.c +++ b/lib/compressor_lz4.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c index 3f68b00..8f2d25c 100644 --- a/lib/compressor_lz4hc.c +++ b/lib/compressor_lz4hc.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/config.c b/lib/config.c index f1c8edf..d3298da 100644 --- a/lib/config.c +++ b/lib/config.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/data.c b/lib/data.c index 27710f9..e57707e 100644 --- a/lib/data.c +++ b/lib/data.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2020 Gao Xiang * Compression support by Huang Jianan diff --git a/lib/decompress.c b/lib/decompress.c index f313e41..359dae7 100644 --- a/lib/decompress.c +++ b/lib/decompress.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C), 2008-2020, OPPO Mobile Comm Corp., Ltd. * Created by Huang Jianan diff --git a/lib/exclude.c b/lib/exclude.c index 2f980a3..e3c4ed5 100644 --- a/lib/exclude.c +++ b/lib/exclude.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu */ diff --git a/lib/inode.c b/lib/inode.c index 461c797..b6d3092 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018-2019 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/io.c b/lib/io.c index a0d366a..5bc3432 100644 --- a/lib/io.c +++ b/lib/io.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Copyright (C) 2018 HUAWEI, Inc. * http://www.huawei.com/ diff --git a/lib/namei.c b/lib/namei.c index 7377e74..4124170 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu */ diff --git a/lib/super.c b/lib/super.c index 3ccc551..69522bd 100644 --- a/lib/super.c +++ b/lib/super.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Created by Li Guifu */ diff --git a/lib/xattr.c b/lib/xattr.c index 00fb963..02f93f2 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * Originally contributed by an anonymous person, * heavily changed by Li Guifu diff --git a/lib/zmap.c b/lib/zmap.c index 3715c47..09d5d35 100644 --- a/lib/zmap.c +++ b/lib/zmap.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0+ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* * (a large amount of code was adapted from Linux kernel. ) * -- cgit v1.2.3 From 1ca453ee89894b1669ac78d7f216bda172783e8d Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Mon, 6 Dec 2021 20:14:03 +0100 Subject: fsck/main.c: add missing include Otherwise musl C library builds fail with missing S_IFMT/S_IFDIR definitions. Link: https://lore.kernel.org/r/20211206191403.1435229-1-alex@linutronix.de Reviewed-by: Gao Xiang Signed-off-by: Alexander Kanavin Signed-off-by: Gao Xiang --- fsck/main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/fsck/main.c b/fsck/main.c index aefa881..ad48e35 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "erofs/print.h" #include "erofs/io.h" #include "erofs/decompress.h" -- cgit v1.2.3 From 80dbfd1e2622c7185b3ec743a33fe8531e340ef1 Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Tue, 7 Dec 2021 22:05:28 +0800 Subject: erofs-utils: add an option to dump an inode by path This is useful when playing around with EROFS images locally. For example, it allows us to inspect block mappings of a filepath. Link: https://lore.kernel.org/r/20211207140528.185442-1-hsiangkao@linux.alibaba.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- dump/main.c | 25 +++++++++++++++++++++---- man/dump.erofs.1 | 3 +++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/dump/main.c b/dump/main.c index f85903b..072d726 100644 --- a/dump/main.c +++ b/dump/main.c @@ -23,6 +23,7 @@ struct erofsdump_cfg { bool show_superblock; bool show_statistics; erofs_nid_t nid; + const char *inode_path; }; static struct erofsdump_cfg dumpcfg; @@ -71,6 +72,7 @@ static struct option long_options[] = { {"help", no_argument, NULL, 1}, {"nid", required_argument, NULL, 2}, {"device", required_argument, NULL, 3}, + {"path", required_argument, NULL, 4}, {0, 0, 0, 0}, }; @@ -103,6 +105,7 @@ static void usage(void) " -s show information about superblock\n" " --device=X specify an extra device to be used together\n" " --nid=# show the target inode info of nid #\n" + " --path=X show the target inode info of path X\n" " --help display this help and exit.\n", stderr); } @@ -148,6 +151,11 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) return err; ++sbi.extra_devices; break; + case 4: + dumpcfg.inode_path = optarg; + dumpcfg.show_inode = true; + ++dumpcfg.totalshow; + break; default: return -EINVAL; } @@ -450,10 +458,19 @@ static void erofsdump_show_fileinfo(bool show_extent) .m_la = 0, }; - err = erofs_read_inode_from_disk(&inode); - if (err) { - erofs_err("read inode failed @ nid %llu", inode.nid | 0ULL); - return; + if (dumpcfg.inode_path) { + err = erofs_ilookup(dumpcfg.inode_path, &inode); + if (err) { + erofs_err("read inode failed @ %s", dumpcfg.inode_path); + return; + } + } else { + err = erofs_read_inode_from_disk(&inode); + if (err) { + erofs_err("read inode failed @ nid %llu", + inode.nid | 0ULL); + return; + } } err = erofs_get_occupied_size(&inode, &size); diff --git a/man/dump.erofs.1 b/man/dump.erofs.1 index 8efb161..fd437cf 100644 --- a/man/dump.erofs.1 +++ b/man/dump.erofs.1 @@ -22,6 +22,9 @@ You may give multiple `--device' options in the correct order. .BI "\-\-nid=" NID Specify an inode NID in order to print its file information. .TP +.BI "\-\-path=" path +Specify an inode path in order to print its file information. +.TP .BI \-e Show the file extent information. The option depends on option --nid to specify NID. .TP -- cgit v1.2.3 From 3d5921eba3cec0ec60bf088334bd61ab2b75a80a Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 14 Dec 2021 09:33:22 +0800 Subject: erofs-utils: clear compacted_2b if compacted_4b_initial > totalidx Keep in sync with the latest kernel commit c40dd3ca2a45 ("erofs: clear compacted_2b if compacted_4b_initial > totalidx") Link: https://lore.kernel.org/r/20211214013322.4638-1-xiang@kernel.org Signed-off-by: Gao Xiang --- lib/zmap.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/zmap.c b/lib/zmap.c index 09d5d35..abc8bab 100644 --- a/lib/zmap.c +++ b/lib/zmap.c @@ -315,7 +315,8 @@ static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m, if (compacted_4b_initial == 32 / 4) compacted_4b_initial = 0; - if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) + if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && + compacted_4b_initial < totalidx) compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); else compacted_2b = 0; -- cgit v1.2.3 From 51cc6632a237f56c93dc7d3dc3d6fc1d26e1a73e Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Tue, 14 Dec 2021 09:49:53 -0800 Subject: erofs-utils: lib: mark some compressor parameters as const Link: https://lore.kernel.org/r/20211214174953.1947261-1-zhangkelvin@google.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- lib/compressor.c | 4 ++-- lib/compressor.h | 8 ++++---- lib/compressor_liblzma.c | 4 ++-- lib/compressor_lz4.c | 4 ++-- lib/compressor_lz4hc.c | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/compressor.c b/lib/compressor.c index a4d696e..3e04a3a 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -22,8 +22,8 @@ static const struct erofs_compressor *compressors[] = { #endif }; -int erofs_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +int erofs_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { unsigned int uncompressed_size; diff --git a/lib/compressor.h b/lib/compressor.h index fd28dfe..728dd0b 100644 --- a/lib/compressor.h +++ b/lib/compressor.h @@ -21,8 +21,8 @@ struct erofs_compressor { int (*exit)(struct erofs_compress *c); int (*setlevel)(struct erofs_compress *c, int compression_level); - int (*compress_destsize)(struct erofs_compress *c, - void *src, unsigned int *srcsize, + int (*compress_destsize)(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize); }; @@ -45,8 +45,8 @@ extern const struct erofs_compressor erofs_compressor_lz4; extern const struct erofs_compressor erofs_compressor_lz4hc; extern const struct erofs_compressor erofs_compressor_lzma; -int erofs_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +int erofs_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize); int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level); diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c index 42f3ac6..acf442f 100644 --- a/lib/compressor_liblzma.c +++ b/lib/compressor_liblzma.c @@ -18,8 +18,8 @@ struct erofs_liblzma_context { lzma_stream strm; }; -static int erofs_liblzma_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int erofs_liblzma_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { struct erofs_liblzma_context *ctx = c->private_data; diff --git a/lib/compressor_lz4.c b/lib/compressor_lz4.c index 0936d03..b6f6e7e 100644 --- a/lib/compressor_lz4.c +++ b/lib/compressor_lz4.c @@ -12,8 +12,8 @@ #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif -static int lz4_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int lz4_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { int srcSize = (int)*srcsize; diff --git a/lib/compressor_lz4hc.c b/lib/compressor_lz4hc.c index 8f2d25c..eec1c84 100644 --- a/lib/compressor_lz4hc.c +++ b/lib/compressor_lz4hc.c @@ -13,8 +13,8 @@ #define LZ4_DISTANCE_MAX 65535 /* set to maximum value by default */ #endif -static int lz4hc_compress_destsize(struct erofs_compress *c, - void *src, unsigned int *srcsize, +static int lz4hc_compress_destsize(const struct erofs_compress *c, + const void *src, unsigned int *srcsize, void *dst, unsigned int dstsize) { int srcSize = (int)*srcsize; -- cgit v1.2.3 From 097410dbfd1e62122a982dcb3011700c4e9a6e64 Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Thu, 16 Dec 2021 10:09:25 +0800 Subject: erofs-utils: sort shared xattr Sort shared xattr before writing to disk to ensure the consistency of reproducible builds. Link: https://lore.kernel.org/r/20211216020925.24963-1-huangjianan@oppo.com Reviewed-by: Gao Xiang Signed-off-by: Huang Jianan Signed-off-by: Gao Xiang --- lib/xattr.c | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/lib/xattr.c b/lib/xattr.c index 02f93f2..71ffe3e 100644 --- a/lib/xattr.c +++ b/lib/xattr.c @@ -563,13 +563,31 @@ static struct erofs_bhops erofs_write_shared_xattrs_bhops = { .flush = erofs_bh_flush_write_shared_xattrs, }; +static int comp_xattr_item(const void *a, const void *b) +{ + const struct xattr_item *ia, *ib; + unsigned int la, lb; + int ret; + + ia = (*((const struct inode_xattr_node **)a))->item; + ib = (*((const struct inode_xattr_node **)b))->item; + la = ia->len[0] + ia->len[1]; + lb = ib->len[0] + ib->len[1]; + + ret = strncmp(ia->kvbuf, ib->kvbuf, min(la, lb)); + if (ret != 0) + return ret; + + return la > lb; +} + int erofs_build_shared_xattrs_from_path(const char *path) { int ret; struct erofs_buffer_head *bh; - struct inode_xattr_node *node, *n; + struct inode_xattr_node *node, *n, **sorted_n; char *buf; - unsigned int p; + unsigned int p, i; erofs_off_t off; /* check if xattr or shared xattr is disabled */ @@ -607,16 +625,26 @@ int erofs_build_shared_xattrs_from_path(const char *path) off %= EROFS_BLKSIZ; p = 0; + sorted_n = malloc(shared_xattrs_count * sizeof(n)); + if (!sorted_n) + return -ENOMEM; + i = 0; list_for_each_entry_safe(node, n, &shared_xattrs_list, list) { - struct xattr_item *const item = node->item; + list_del(&node->list); + sorted_n[i++] = node; + } + DBG_BUGON(i != shared_xattrs_count); + qsort(sorted_n, shared_xattrs_count, sizeof(n), comp_xattr_item); + + for (i = 0; i < shared_xattrs_count; i++) { + struct inode_xattr_node *const tnode = sorted_n[i]; + struct xattr_item *const item = tnode->item; const struct erofs_xattr_entry entry = { .e_name_index = item->prefix, .e_name_len = item->len[0], .e_value_size = cpu_to_le16(item->len[1]) }; - list_del(&node->list); - item->shared_xattr_id = (off + p) / sizeof(struct erofs_xattr_entry); @@ -624,8 +652,10 @@ int erofs_build_shared_xattrs_from_path(const char *path) p += sizeof(struct erofs_xattr_entry); memcpy(buf + p, item->kvbuf, item->len[0] + item->len[1]); p = EROFS_XATTR_ALIGN(p + item->len[0] + item->len[1]); - free(node); + free(tnode); } + + free(sorted_n); bh->fsprivate = buf; bh->op = &erofs_write_shared_xattrs_bhops; out: -- cgit v1.2.3 From 564adb0a852b38a1790db20516862fc31bca314d Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Thu, 16 Dec 2021 09:24:35 +0800 Subject: erofs-utils: lib: add API to iterate dirs in EROFS This introduces erofs_iterate_dir() to iterate all dirents in a directory inode and convert erofsfuse to use the API. Note that it doesn't recursively walk into sub-directories. If it's really needed, users should handle this in the callback. Link: https://lore.kernel.org/r/20211216012436.10111-1-xiang@kernel.org Signed-off-by: Gao Xiang --- fuse/Makefile.am | 2 +- fuse/dir.c | 100 ---------------------------- fuse/main.c | 52 ++++++++++++++- include/erofs/dir.h | 60 +++++++++++++++++ include/erofs/internal.h | 8 +++ lib/Makefile.am | 2 +- lib/dir.c | 168 +++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 104 deletions(-) delete mode 100644 fuse/dir.c create mode 100644 include/erofs/dir.h create mode 100644 lib/dir.c diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 8a2d472..5aa5ac0 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -3,7 +3,7 @@ AUTOMAKE_OPTIONS = foreign noinst_HEADERS = $(top_srcdir)/fuse/macosx.h bin_PROGRAMS = erofsfuse -erofsfuse_SOURCES = dir.c main.c +erofsfuse_SOURCES = main.c erofsfuse_CFLAGS = -Wall -Werror -I$(top_srcdir)/include erofsfuse_CFLAGS += -DFUSE_USE_VERSION=26 ${libfuse_CFLAGS} ${libselinux_CFLAGS} erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse_LIBS} ${liblz4_LIBS} \ diff --git a/fuse/dir.c b/fuse/dir.c deleted file mode 100644 index bc8735b..0000000 --- a/fuse/dir.c +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Created by Li Guifu - */ -#include -#include -#include "macosx.h" -#include "erofs/internal.h" -#include "erofs/print.h" - -static int erofs_fill_dentries(struct erofs_inode *dir, - fuse_fill_dir_t filler, void *buf, - void *dblk, unsigned int nameoff, - unsigned int maxsize) -{ - struct erofs_dirent *de = dblk; - const struct erofs_dirent *end = dblk + nameoff; - char namebuf[EROFS_NAME_LEN + 1]; - - while (de < end) { - const char *de_name; - unsigned int de_namelen; - - nameoff = le16_to_cpu(de->nameoff); - de_name = (char *)dblk + nameoff; - - /* the last dirent in the block? */ - if (de + 1 >= end) - de_namelen = strnlen(de_name, maxsize - nameoff); - else - de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; - - /* a corrupted entry is found */ - if (nameoff + de_namelen > maxsize || - de_namelen > EROFS_NAME_LEN) { - erofs_err("bogus dirent @ nid %llu", dir->nid | 0ULL); - DBG_BUGON(1); - return -EFSCORRUPTED; - } - - memcpy(namebuf, de_name, de_namelen); - namebuf[de_namelen] = '\0'; - - filler(buf, namebuf, NULL, 0); - ++de; - } - return 0; -} - -int erofsfuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi) -{ - int ret; - struct erofs_inode dir; - char dblk[EROFS_BLKSIZ]; - erofs_off_t pos; - - erofs_dbg("readdir:%s offset=%llu", path, (long long)offset); - - ret = erofs_ilookup(path, &dir); - if (ret) - return ret; - - erofs_dbg("path=%s nid = %llu", path, dir.nid | 0ULL); - - if (!S_ISDIR(dir.i_mode)) - return -ENOTDIR; - - if (!dir.i_size) - return 0; - - pos = 0; - while (pos < dir.i_size) { - unsigned int nameoff, maxsize; - struct erofs_dirent *de; - - maxsize = min_t(unsigned int, EROFS_BLKSIZ, - dir.i_size - pos); - ret = erofs_pread(&dir, dblk, maxsize, pos); - if (ret) - return ret; - - de = (struct erofs_dirent *)dblk; - nameoff = le16_to_cpu(de->nameoff); - if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { - erofs_err("invalid de[0].nameoff %u @ nid %llu", - nameoff, dir.nid | 0ULL); - ret = -EFSCORRUPTED; - break; - } - - ret = erofs_fill_dentries(&dir, filler, buf, - dblk, nameoff, maxsize); - if (ret) - break; - pos += maxsize; - } - return 0; -} diff --git a/fuse/main.c b/fuse/main.c index 255965e..2549d8a 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -12,9 +12,57 @@ #include "erofs/config.h" #include "erofs/print.h" #include "erofs/io.h" +#include "erofs/dir.h" -int erofsfuse_readdir(const char *path, void *buffer, fuse_fill_dir_t filler, - off_t offset, struct fuse_file_info *fi); +struct erofsfuse_dir_context { + struct erofs_dir_context ctx; + fuse_fill_dir_t filler; + struct fuse_file_info *fi; + void *buf; +}; + +static int erofsfuse_fill_dentries(struct erofs_dir_context *ctx) +{ + struct erofsfuse_dir_context *fusectx = (void *)ctx; + char dname[EROFS_NAME_LEN + 1]; + + strncpy(dname, ctx->dname, ctx->de_namelen); + dname[ctx->de_namelen] = '\0'; + fusectx->filler(fusectx->buf, dname, NULL, 0); + return 0; +} + +int erofsfuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + int ret; + struct erofs_inode dir; + struct erofsfuse_dir_context ctx = { + .ctx.dir = &dir, + .ctx.cb = erofsfuse_fill_dentries, + .filler = filler, + .fi = fi, + .buf = buf, + }; + erofs_dbg("readdir:%s offset=%llu", path, (long long)offset); + + ret = erofs_ilookup(path, &dir); + if (ret) + return ret; + + erofs_dbg("path=%s nid = %llu", path, dir.nid | 0ULL); + if (!S_ISDIR(dir.i_mode)) + return -ENOTDIR; + + if (!dir.i_size) + return 0; +#ifdef NDEBUG + return erofs_iterate_dir(&ctx.ctx, false); +#else + return erofs_iterate_dir(&ctx.ctx, true); +#endif + +} static void *erofsfuse_init(struct fuse_conn_info *info) { diff --git a/include/erofs/dir.h b/include/erofs/dir.h new file mode 100644 index 0000000..77656ca --- /dev/null +++ b/include/erofs/dir.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 */ +#ifndef __EROFS_DIR_H +#define __EROFS_DIR_H + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "internal.h" + +#define EROFS_READDIR_VALID_PNID 0x0001 +#define EROFS_READDIR_DOTDOT_FOUND 0x0002 +#define EROFS_READDIR_DOT_FOUND 0x0004 + +#define EROFS_READDIR_ALL_SPECIAL_FOUND \ + (EROFS_READDIR_DOTDOT_FOUND | EROFS_READDIR_DOT_FOUND) + +struct erofs_dir_context; + +/* callback function for iterating over inodes of EROFS */ +typedef int (*erofs_readdir_cb)(struct erofs_dir_context *); + +/* + * Callers could use a wrapper to contain extra information. + * + * Note that callback can reuse `struct erofs_dir_context' with care + * to avoid stack overflow due to deep recursion: + * - if fsck is true, |pnid|, |flags|, (optional)|cb| SHOULD be saved + * to ensure the original state; + * - if fsck is false, EROFS_READDIR_VALID_PNID SHOULD NOT be + * set if |pnid| is inaccurate. + * + * Another way is to allocate a `struct erofs_dir_context' wraper + * with `struct inode' on heap, and chain them together for + * multi-level traversal to completely avoid recursion. + * + * |dname| may be WITHOUT the trailing '\0' and it's ONLY valid in + * the callback context. |de_namelen| is the exact dirent name length. + */ +struct erofs_dir_context { + struct erofs_inode *dir; + erofs_readdir_cb cb; + erofs_nid_t pnid; /* optional */ + + /* [OUT] the dirent which is under processing */ + const char *dname; /* please see the comment above */ + erofs_nid_t de_nid; + u8 de_namelen, de_ftype, flags; + bool dot_dotdot; +}; + +/* Iterate over inodes that are in directory */ +int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/erofs/internal.h b/include/erofs/internal.h index e6beb8c..d2adf57 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -235,6 +235,14 @@ struct erofs_dentry { }; }; +static inline bool is_dot_dotdot_len(const char *name, unsigned int len) +{ + if (len >= 1 && name[0] != '.') + return false; + + return len == 1 || (len == 2 && name[1] == '.'); +} + static inline bool is_dot_dotdot(const char *name) { if (name[0] != '.') diff --git a/lib/Makefile.am b/lib/Makefile.am index c745e49..4a25013 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -27,7 +27,7 @@ noinst_HEADERS = $(top_srcdir)/include/erofs_fs.h \ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ - compress_hints.c hashmap.c sha256.c blobchunk.c + compress_hints.c hashmap.c sha256.c blobchunk.c dir.c liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS} diff --git a/lib/dir.c b/lib/dir.c new file mode 100644 index 0000000..e53c897 --- /dev/null +++ b/lib/dir.c @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 +#include "erofs/print.h" +#include "erofs/dir.h" +#include + +static int traverse_dirents(struct erofs_dir_context *ctx, + void *dentry_blk, unsigned int lblk, + unsigned int next_nameoff, unsigned int maxsize, + bool fsck) +{ + struct erofs_dirent *de = dentry_blk; + const struct erofs_dirent *end = dentry_blk + next_nameoff; + const char *prev_name = NULL; + const char *errmsg; + unsigned int prev_namelen = 0; + int ret = 0; + bool silent = false; + + while (de < end) { + const char *de_name; + unsigned int de_namelen; + unsigned int nameoff; + + nameoff = le16_to_cpu(de->nameoff); + de_name = (char *)dentry_blk + nameoff; + + /* the last dirent check */ + if (de + 1 >= end) + de_namelen = strnlen(de_name, maxsize - nameoff); + else + de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; + + ctx->de_nid = le64_to_cpu(de->nid); + erofs_dbg("traversed nid (%llu)", ctx->de_nid | 0ULL); + + ret = -EFSCORRUPTED; + /* corrupted entry check */ + if (nameoff != next_nameoff) { + errmsg = "bogus dirent nameoff"; + break; + } + + if (nameoff + de_namelen > maxsize || + de_namelen > EROFS_NAME_LEN) { + errmsg = "bogus dirent namelen"; + break; + } + + if (fsck && prev_name) { + int cmp = strncmp(prev_name, de_name, + min(prev_namelen, de_namelen)); + + if (cmp > 0 || (cmp == 0 && + prev_namelen >= de_namelen)) { + errmsg = "wrong dirent name order"; + break; + } + } + + if (fsck && de->file_type >= EROFS_FT_MAX) { + errmsg = "invalid file type %u"; + break; + } + + ctx->dname = de_name; + ctx->de_namelen = de_namelen; + ctx->de_ftype = de->file_type; + ctx->dot_dotdot = is_dot_dotdot_len(de_name, de_namelen); + if (ctx->dot_dotdot) { + switch (de_namelen) { + case 2: + if (fsck && + (ctx->flags & EROFS_READDIR_DOTDOT_FOUND)) { + errmsg = "duplicated `..' dirent"; + goto out; + } + ctx->flags |= EROFS_READDIR_DOTDOT_FOUND; + if (sbi.root_nid == ctx->dir->nid) { + ctx->pnid = sbi.root_nid; + ctx->flags |= EROFS_READDIR_VALID_PNID; + } + if (fsck && + (ctx->flags & EROFS_READDIR_VALID_PNID) && + ctx->de_nid != ctx->pnid) { + errmsg = "corrupted `..' dirent"; + goto out; + } + break; + case 1: + if (fsck && + (ctx->flags & EROFS_READDIR_DOT_FOUND)) { + errmsg = "duplicated `.' dirent"; + goto out; + } + + ctx->flags |= EROFS_READDIR_DOT_FOUND; + if (fsck && ctx->de_nid != ctx->dir->nid) { + errmsg = "corrupted `.' dirent"; + goto out; + } + break; + } + } + ret = ctx->cb(ctx); + if (ret) { + silent = true; + break; + } + prev_name = de_name; + prev_namelen = de_namelen; + next_nameoff += de_namelen; + ++de; + } +out: + if (ret && !silent) + erofs_err("%s @ nid %llu, lblk %u, index %lu", + errmsg, ctx->dir->nid | 0ULL, lblk, + (de - (struct erofs_dirent *)dentry_blk) | 0UL); + return ret; +} + +int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) +{ + struct erofs_inode *dir = ctx->dir; + int err = 0; + erofs_off_t pos; + char buf[EROFS_BLKSIZ]; + + if ((dir->i_mode & S_IFMT) != S_IFDIR) + return -ENOTDIR; + + ctx->flags &= ~EROFS_READDIR_ALL_SPECIAL_FOUND; + pos = 0; + while (pos < dir->i_size) { + erofs_blk_t lblk = erofs_blknr(pos); + erofs_off_t maxsize = min_t(erofs_off_t, + dir->i_size - pos, EROFS_BLKSIZ); + const struct erofs_dirent *de = (const void *)buf; + unsigned int nameoff; + + err = erofs_pread(dir, buf, maxsize, pos); + if (err) { + erofs_err("I/O error occurred when reading dirents @ nid %llu, lblk %u: %d", + dir->nid | 0ULL, lblk, err); + return err; + } + + nameoff = le16_to_cpu(de->nameoff); + if (nameoff < sizeof(struct erofs_dirent) || + nameoff >= PAGE_SIZE) { + erofs_err("invalid de[0].nameoff %u @ nid %llu, lblk %u", + nameoff, dir->nid | 0ULL, lblk); + return -EFSCORRUPTED; + } + err = traverse_dirents(ctx, buf, lblk, nameoff, maxsize, fsck); + if (err) + break; + pos += maxsize; + } + + if (fsck && (ctx->flags & EROFS_READDIR_ALL_SPECIAL_FOUND) != + EROFS_READDIR_ALL_SPECIAL_FOUND) { + erofs_err("`.' or `..' dirent is missing @ nid %llu", + dir->nid | 0ULL); + return -EFSCORRUPTED; + } + return err; +} -- cgit v1.2.3 From b11f84f593f99dfd32b449bbbde5a3279ee154df Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Thu, 16 Dec 2021 09:24:36 +0800 Subject: erofs-utils: fsck: convert to use erofs_iterate_dir() No need to open code after erofs_iterate_dir() is finalized in liberofs. However, there are still some TODOs for fsck: - Avoid too deep recursive traversal, sceptically forming a loop; - Check link counts at runtime to keep consistency; - Check if any ftype / i_mode mismatches. Link: https://lore.kernel.org/r/20211216012436.10111-2-xiang@kernel.org Signed-off-by: Gao Xiang --- fsck/main.c | 178 ++++++++---------------------------------------------------- 1 file changed, 23 insertions(+), 155 deletions(-) diff --git a/fsck/main.c b/fsck/main.c index ad48e35..30d0a1b 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -10,8 +10,9 @@ #include "erofs/print.h" #include "erofs/io.h" #include "erofs/decompress.h" +#include "erofs/dir.h" -static void erofs_check_inode(erofs_nid_t pnid, erofs_nid_t nid); +static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid); struct erofsfsck_cfg { bool corrupted; @@ -126,121 +127,6 @@ static int erofs_check_sb_chksum(void) return 0; } -static bool check_special_dentry(struct erofs_dirent *de, - unsigned int de_namelen, erofs_nid_t nid, - erofs_nid_t pnid) -{ - if (de_namelen == 2 && de->nid != pnid) { - erofs_err("wrong parent dir nid(%llu): pnid(%llu) @ nid(%llu)", - de->nid | 0ULL, pnid | 0ULL, nid | 0ULL); - return false; - } - - if (de_namelen == 1 && de->nid != nid) { - erofs_err("wrong current dir nid(%llu) @ nid(%llu)", - de->nid | 0ULL, nid | 0ULL); - return false; - } - return true; -} - -static int traverse_dirents(erofs_nid_t pnid, erofs_nid_t nid, - void *dentry_blk, erofs_blk_t block, - unsigned int next_nameoff, unsigned int maxsize) -{ - struct erofs_dirent *de = dentry_blk; - const struct erofs_dirent *end = dentry_blk + next_nameoff; - unsigned int idx = 0; - char *prev_name = NULL, *cur_name = NULL; - int ret = 0; - - erofs_dbg("traversing pnid(%llu), nid(%llu)", pnid | 0ULL, nid | 0ULL); - - if (!block && (next_nameoff < 2 * sizeof(struct erofs_dirent))) { - erofs_err("too small dirents of size(%d) in nid(%llu)", - next_nameoff, nid | 0ULL); - return -EFSCORRUPTED; - } - - while (de < end) { - const char *de_name; - unsigned int de_namelen; - unsigned int nameoff; - - nameoff = le16_to_cpu(de->nameoff); - de_name = (char *)dentry_blk + nameoff; - - /* the last dirent check */ - if (de + 1 >= end) - de_namelen = strnlen(de_name, maxsize - nameoff); - else - de_namelen = le16_to_cpu(de[1].nameoff) - nameoff; - - if (prev_name) - free(prev_name); - prev_name = cur_name; - cur_name = strndup(de_name, de_namelen); - if (!cur_name) { - ret = -ENOMEM; - goto out; - } - - erofs_dbg("traversed filename(%s)", cur_name); - - /* corrupted entry check */ - if (nameoff != next_nameoff) { - erofs_err("bogus dirent with nameoff(%u): expected(%d) @ nid %llu, block %u, idx %u", - nameoff, next_nameoff, nid | 0ULL, - block, idx); - ret = -EFSCORRUPTED; - goto out; - } - - if (nameoff + de_namelen > maxsize || - de_namelen > EROFS_NAME_LEN) { - erofs_err("bogus dirent with namelen(%u) @ nid %llu, block %u, idx %u", - de_namelen, nid | 0ULL, block, idx); - ret = -EFSCORRUPTED; - goto out; - } - - if (prev_name && (strcmp(prev_name, cur_name) >= 0)) { - erofs_err("wrong dirent name order @ nid %llu block %u idx %u: prev(%s), cur(%s)", - nid | 0ULL, block, idx, - prev_name, cur_name); - ret = -EFSCORRUPTED; - goto out; - } - - if (is_dot_dotdot(cur_name)) { - if (!check_special_dentry(de, de_namelen, nid, pnid)) { - ret = -EFSCORRUPTED; - goto out; - } - } else { - erofs_check_inode(nid, de->nid); - } - - if (fsckcfg.corrupted) { - ret = -EFSCORRUPTED; - goto out; - } - - next_nameoff += de_namelen; - ++de; - ++idx; - } - -out: - if (prev_name) - free(prev_name); - if (cur_name) - free(cur_name); - - erofs_dbg("traversing ... done nid(%llu)", nid | 0ULL); - return ret; -} - static int verify_uncompressed_inode(struct erofs_inode *inode) { struct erofs_map_blocks map = { @@ -479,12 +365,18 @@ static int erofs_verify_inode_data(struct erofs_inode *inode) return ret; } -static void erofs_check_inode(erofs_nid_t pnid, erofs_nid_t nid) +static int erofsfsck_dirent_iter(struct erofs_dir_context *ctx) +{ + if (ctx->dot_dotdot) + return 0; + + return erofsfsck_check_inode(ctx->dir->nid, ctx->de_nid); +} + +static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) { int ret; struct erofs_inode inode; - char buf[EROFS_BLKSIZ]; - erofs_off_t offset; erofs_dbg("check inode: nid(%llu)", nid | 0ULL); @@ -507,44 +399,21 @@ static void erofs_check_inode(erofs_nid_t pnid, erofs_nid_t nid) if (ret) goto out; - if ((inode.i_mode & S_IFMT) != S_IFDIR) - goto out; - - offset = 0; - while (offset < inode.i_size) { - erofs_blk_t block = erofs_blknr(offset); - erofs_off_t maxsize = min_t(erofs_off_t, - inode.i_size - offset, EROFS_BLKSIZ); - struct erofs_dirent *de = (void *)buf; - - unsigned int nameoff; - - ret = erofs_pread(&inode, buf, maxsize, offset); - if (ret) { - erofs_err("I/O error occurred when reading dirents @ nid %llu, block %u: %d", - nid | 0ULL, block, ret); - goto out; - } - - nameoff = le16_to_cpu(de->nameoff); - if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { - erofs_err("invalid de[0].nameoff %u @ nid %llu block %u: %d", - nameoff, nid | 0ULL, block, ret); - ret = -EFSCORRUPTED; - goto out; - } - - ret = traverse_dirents(pnid, nid, buf, block, - nameoff, maxsize); - if (ret) - goto out; + /* XXXX: the dir depth should be restricted in order to avoid loops */ + if (S_ISDIR(inode.i_mode)) { + struct erofs_dir_context ctx = { + .flags = EROFS_READDIR_VALID_PNID, + .pnid = pnid, + .dir = &inode, + .cb = erofsfsck_dirent_iter, + }; - offset += maxsize; + ret = erofs_iterate_dir(&ctx, true); } out: if (ret && ret != -EIO) fsckcfg.corrupted = true; + return ret; } int main(int argc, char **argv) @@ -583,12 +452,11 @@ int main(int argc, char **argv) goto exit_dev_close; } - erofs_check_inode(sbi.root_nid, sbi.root_nid); - + err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid); if (fsckcfg.corrupted) { erofs_err("Found some filesystem corruption"); err = -EFSCORRUPTED; - } else { + } else if (!err) { erofs_info("No error found"); if (fsckcfg.print_comp_ratio) { double comp_ratio = -- cgit v1.2.3 From 5a9ac8e057cf744eb844da840ea08468c1be1f30 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Fri, 17 Dec 2021 00:02:54 +0800 Subject: erofs-utils: dump: convert readdir to use erofs_iterate_dir() No need to open code after erofs_iterate_dir() is finalized in liberofs. Note that `erofs_get_pathname' isn't touched in this commit. Link: https://lore.kernel.org/r/20211216160254.28166-1-xiang@kernel.org Signed-off-by: Gao Xiang --- configure.ac | 5 ++ dump/main.c | 182 +++++++++++++++++++++---------------------------- lib/liberofs_private.h | 12 ++++ 3 files changed, 93 insertions(+), 106 deletions(-) diff --git a/configure.ac b/configure.ac index 6fdb0e4..a5de291 100644 --- a/configure.ac +++ b/configure.ac @@ -167,6 +167,11 @@ AC_CHECK_DECL(lseek64,[AC_DEFINE(HAVE_LSEEK64_PROTOTYPE, 1, #define _LARGEFILE64_SOURCE #include ]) +AC_CHECK_DECL(memrchr,[AC_DEFINE(HAVE_MEMRCHR, 1, + [Define to 1 if memrchr declared in string.h])],, + [#define _GNU_SOURCE + #include ]) + # Checks for library functions. AC_CHECK_FUNCS(m4_flatten([ backtrace diff --git a/dump/main.c b/dump/main.c index 072d726..7f3f743 100644 --- a/dump/main.c +++ b/dump/main.c @@ -5,12 +5,16 @@ * Created by Wang Qi * Guo Xuenan */ +#define _GNU_SOURCE #include #include #include +#include #include "erofs/print.h" #include "erofs/inode.h" #include "erofs/io.h" +#include "erofs/dir.h" +#include "../lib/liberofs_private.h" #ifdef HAVE_LIBUUID #include @@ -90,10 +94,7 @@ static struct erofsdump_feature feature_lists[] = { { false, EROFS_FEATURE_INCOMPAT_DEVICE_TABLE, "device_table" }, }; -static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid); -static inline int erofs_checkdirent(struct erofs_dirent *de, - struct erofs_dirent *last_de, - u32 maxsize, const char *dname); +static int erofsdump_readdir(struct erofs_dir_context *ctx); static void usage(void) { @@ -198,18 +199,20 @@ static int erofs_get_occupied_size(struct erofs_inode *inode, return 0; } -static int erofs_getfile_extension(const char *filename) +static void inc_file_extension_count(const char *dname, unsigned int len) { - char *postfix = strrchr(filename, '.'); - int type = 0; + char *postfix = memrchr(dname, '.', len); + int type; - if (!postfix) - return OTHERFILETYPE - 1; - for (type = 0; type < OTHERFILETYPE - 1; ++type) { - if (strcmp(postfix, file_types[type]) == 0) - break; + if (!postfix) { + type = OTHERFILETYPE - 1; + } else { + for (type = 0; type < OTHERFILETYPE - 1; ++type) + if (!strncmp(postfix, file_types[type], + len - (postfix - dname))) + break; } - return type; + ++stats.file_type_stat[type]; } static void update_file_size_statatics(erofs_off_t occupied_size, @@ -244,6 +247,56 @@ static void update_file_size_statatics(erofs_off_t occupied_size, stats.file_comp_size[occupied_size_mark]++; } +static int erofsdump_dirent_iter(struct erofs_dir_context *ctx) +{ + /* skip "." and ".." dentry */ + if (ctx->dot_dotdot) + return 0; + + return erofsdump_readdir(ctx); +} + +static int erofsdump_readdir(struct erofs_dir_context *ctx) +{ + int err; + erofs_off_t occupied_size = 0; + struct erofs_inode vi = { .nid = ctx->de_nid }; + + err = erofs_read_inode_from_disk(&vi); + if (err) { + erofs_err("failed to read file inode from disk"); + return err; + } + stats.files++; + stats.file_category_stat[erofs_mode_to_ftype(vi.i_mode)]++; + + err = erofs_get_occupied_size(&vi, &occupied_size); + if (err) { + erofs_err("get file size failed"); + return err; + } + + if (S_ISREG(vi.i_mode)) { + stats.files_total_origin_size += vi.i_size; + inc_file_extension_count(ctx->dname, ctx->de_namelen); + stats.files_total_size += occupied_size; + update_file_size_statatics(occupied_size, vi.i_size); + } + + /* XXXX: the dir depth should be restricted in order to avoid loops */ + if (S_ISDIR(vi.i_mode)) { + struct erofs_dir_context nctx = { + .flags = ctx->dir ? EROFS_READDIR_VALID_PNID : 0, + .pnid = ctx->dir ? ctx->dir->nid : 0, + .dir = &vi, + .cb = erofsdump_dirent_iter, + }; + + return erofs_iterate_dir(&nctx, false); + } + return 0; +} + static inline int erofs_checkdirent(struct erofs_dirent *de, struct erofs_dirent *last_de, u32 maxsize, const char *dname) @@ -275,97 +328,6 @@ static inline int erofs_checkdirent(struct erofs_dirent *de, return dname_len; } -static int erofs_read_dirent(struct erofs_dirent *de, - erofs_nid_t nid, erofs_nid_t parent_nid, - const char *dname) -{ - int err; - erofs_off_t occupied_size = 0; - struct erofs_inode inode = { .nid = le64_to_cpu(de->nid) }; - - stats.files++; - stats.file_category_stat[de->file_type]++; - err = erofs_read_inode_from_disk(&inode); - if (err) { - erofs_err("read file inode from disk failed!"); - return err; - } - - err = erofs_get_occupied_size(&inode, &occupied_size); - if (err) { - erofs_err("get file size failed\n"); - return err; - } - - if (de->file_type == EROFS_FT_REG_FILE) { - stats.files_total_origin_size += inode.i_size; - stats.file_type_stat[erofs_getfile_extension(dname)]++; - stats.files_total_size += occupied_size; - update_file_size_statatics(occupied_size, inode.i_size); - } - - if (de->file_type == EROFS_FT_DIR && inode.nid != nid && - inode.nid != parent_nid) { - err = erofs_read_dir(inode.nid, nid); - if (err) { - erofs_err("parse dir nid %llu error occurred\n", - inode.nid | 0ULL); - return err; - } - } - return 0; -} - -static int erofs_read_dir(erofs_nid_t nid, erofs_nid_t parent_nid) -{ - int err; - erofs_off_t offset; - char buf[EROFS_BLKSIZ]; - struct erofs_inode vi = { .nid = nid }; - - err = erofs_read_inode_from_disk(&vi); - if (err) - return err; - - offset = 0; - while (offset < vi.i_size) { - erofs_off_t maxsize = min_t(erofs_off_t, - vi.i_size - offset, EROFS_BLKSIZ); - struct erofs_dirent *de = (void *)buf; - struct erofs_dirent *end; - unsigned int nameoff; - - err = erofs_pread(&vi, buf, maxsize, offset); - if (err) - return err; - - nameoff = le16_to_cpu(de->nameoff); - end = (void *)buf + nameoff; - while (de < end) { - const char *dname; - int ret; - - /* skip "." and ".." dentry */ - if (le64_to_cpu(de->nid) == nid || - le64_to_cpu(de->nid) == parent_nid) { - de++; - continue; - } - - dname = (char *)buf + nameoff; - ret = erofs_checkdirent(de, end, maxsize, dname); - if (ret < 0) - return ret; - ret = erofs_read_dirent(de, nid, parent_nid, dname); - if (ret < 0) - return ret; - ++de; - } - offset += maxsize; - } - return 0; -} - static int erofs_get_pathname(erofs_nid_t nid, erofs_nid_t parent_nid, erofs_nid_t target, char *path, unsigned int pos) { @@ -629,13 +591,21 @@ static void erofsdump_file_statistic(void) static void erofsdump_print_statistic(void) { int err; + struct erofs_dir_context ctx = { + .flags = 0, + .pnid = 0, + .dir = NULL, + .cb = erofsdump_dirent_iter, + .de_nid = sbi.root_nid, + .dname = "", + .de_namelen = 0 + }; - err = erofs_read_dir(sbi.root_nid, sbi.root_nid); + err = erofsdump_readdir(&ctx); if (err) { erofs_err("read dir failed"); return; } - erofsdump_file_statistic(); erofsdump_filesize_distribution("Original", stats.file_original_size, diff --git a/lib/liberofs_private.h b/lib/liberofs_private.h index c2312e8..0eeca3c 100644 --- a/lib/liberofs_private.h +++ b/lib/liberofs_private.h @@ -11,3 +11,15 @@ #include #include #endif + +#ifndef HAVE_MEMRCHR +static inline void *memrchr(const void *s, int c, size_t n) +{ + const unsigned char *p = (const unsigned char *)s; + + for (p += n; n > 0; n--) + if (*--p == c) + return (void*)p; + return NULL; +} +#endif -- cgit v1.2.3 From e6082718f7434628af5687e103090f3981ed6fea Mon Sep 17 00:00:00 2001 From: Igor Ostapenko Date: Fri, 17 Dec 2021 14:08:23 +0200 Subject: erofs-utils: lib: add API to get pathname of EROFS inode * General-purpose erofs_get_pathname function utilizing erofs_iterate_dir, with recursion and a reused context to avoid overflowing the stack. Recommended buffer size is PATH_MAX. Zero-filling the buffer is not necessary. * dump: PATH_MAX+1 is not required since the definition of PATH_MAX is "chars in a path name including nul". * Return err from erofs_iterate_dir instead of hardcoded 0, to allow breaking the iteration by the callback using a non-zero return code. Link: https://lore.kernel.org/r/CABjEcnE84FNBgiHFk6Q+V3d-4L-93bUFDkdfN4ftPX19kpC=ww@mail.gmail.com Signed-off-by: Igor Ostapenko [ Gao Xiang: let's use NID instead of (struct erofs_inode *). ] Signed-off-by: Gao Xiang --- dump/main.c | 72 ++--------------------------------- include/erofs/dir.h | 2 + lib/dir.c | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 71 deletions(-) diff --git a/dump/main.c b/dump/main.c index 7f3f743..997460c 100644 --- a/dump/main.c +++ b/dump/main.c @@ -328,71 +328,6 @@ static inline int erofs_checkdirent(struct erofs_dirent *de, return dname_len; } -static int erofs_get_pathname(erofs_nid_t nid, erofs_nid_t parent_nid, - erofs_nid_t target, char *path, unsigned int pos) -{ - int err; - erofs_off_t offset; - char buf[EROFS_BLKSIZ]; - struct erofs_inode inode = { .nid = nid }; - - path[pos++] = '/'; - if (target == sbi.root_nid) - return 0; - - err = erofs_read_inode_from_disk(&inode); - if (err) { - erofs_err("read inode failed @ nid %llu", nid | 0ULL); - return err; - } - - offset = 0; - while (offset < inode.i_size) { - erofs_off_t maxsize = min_t(erofs_off_t, - inode.i_size - offset, EROFS_BLKSIZ); - struct erofs_dirent *de = (void *)buf; - struct erofs_dirent *end; - unsigned int nameoff; - - err = erofs_pread(&inode, buf, maxsize, offset); - if (err) - return err; - - nameoff = le16_to_cpu(de->nameoff); - end = (void *)buf + nameoff; - while (de < end) { - const char *dname; - int len; - - nameoff = le16_to_cpu(de->nameoff); - dname = (char *)buf + nameoff; - len = erofs_checkdirent(de, end, maxsize, dname); - if (len < 0) - return len; - - if (le64_to_cpu(de->nid) == target) { - memcpy(path + pos, dname, len); - path[pos + len] = '\0'; - return 0; - } - - if (de->file_type == EROFS_FT_DIR && - le64_to_cpu(de->nid) != parent_nid && - le64_to_cpu(de->nid) != nid) { - memcpy(path + pos, dname, len); - err = erofs_get_pathname(le64_to_cpu(de->nid), - nid, target, path, pos + len); - if (!err) - return 0; - memset(path + pos, 0, len); - } - ++de; - } - offset += maxsize; - } - return -1; -} - static int erofsdump_map_blocks(struct erofs_inode *inode, struct erofs_map_blocks *map, int flags) { @@ -411,7 +346,7 @@ static void erofsdump_show_fileinfo(bool show_extent) erofs_off_t size; u16 access_mode; struct erofs_inode inode = { .nid = dumpcfg.nid }; - char path[PATH_MAX + 1] = {0}; + char path[PATH_MAX]; char access_mode_str[] = "rwxrwxrwx"; char timebuf[128] = {0}; unsigned int extent_count = 0; @@ -441,8 +376,7 @@ static void erofsdump_show_fileinfo(bool show_extent) return; } - err = erofs_get_pathname(sbi.root_nid, sbi.root_nid, - inode.nid, path, 0); + err = erofs_get_pathname(dumpcfg.nid, path, sizeof(path)); if (err < 0) { erofs_err("file path not found @ nid %llu", inode.nid | 0ULL); return; @@ -598,7 +532,7 @@ static void erofsdump_print_statistic(void) .cb = erofsdump_dirent_iter, .de_nid = sbi.root_nid, .dname = "", - .de_namelen = 0 + .de_namelen = 0, }; err = erofsdump_readdir(&ctx); diff --git a/include/erofs/dir.h b/include/erofs/dir.h index 77656ca..336fa9b 100644 --- a/include/erofs/dir.h +++ b/include/erofs/dir.h @@ -52,6 +52,8 @@ struct erofs_dir_context { /* Iterate over inodes that are in directory */ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck); +/* Get a full pathname of the inode NID */ +int erofs_get_pathname(erofs_nid_t nid, char *buf, size_t size); #ifdef __cplusplus } diff --git a/lib/dir.c b/lib/dir.c index e53c897..8955931 100644 --- a/lib/dir.c +++ b/lib/dir.c @@ -1,7 +1,8 @@ // SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 +#include +#include #include "erofs/print.h" #include "erofs/dir.h" -#include static int traverse_dirents(struct erofs_dir_context *ctx, void *dentry_blk, unsigned int lblk, @@ -126,7 +127,7 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) erofs_off_t pos; char buf[EROFS_BLKSIZ]; - if ((dir->i_mode & S_IFMT) != S_IFDIR) + if (!S_ISDIR(dir->i_mode)) return -ENOTDIR; ctx->flags &= ~EROFS_READDIR_ALL_SPECIAL_FOUND; @@ -166,3 +167,104 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) } return err; } + +#define EROFS_PATHNAME_FOUND 1 + +struct erofs_get_pathname_context { + struct erofs_dir_context ctx; + erofs_nid_t target_nid; + char *buf; + size_t size; + size_t pos; +}; + +static int erofs_get_pathname_iter(struct erofs_dir_context *ctx) +{ + int ret; + struct erofs_get_pathname_context *pathctx = (void *)ctx; + const char *dname = ctx->dname; + size_t len = ctx->de_namelen; + size_t pos = pathctx->pos; + + if (ctx->dot_dotdot) + return 0; + + if (ctx->de_nid == pathctx->target_nid) { + if (pos + len + 2 > pathctx->size) { + erofs_err("get_pathname buffer not large enough: len %zd, size %zd", + pos + len + 2, pathctx->size); + return -ERANGE; + } + + pathctx->buf[pos++] = '/'; + strncpy(pathctx->buf + pos, dname, len); + pathctx->buf[pos + len] = '\0'; + return EROFS_PATHNAME_FOUND; + } + + if (ctx->de_ftype == EROFS_FT_DIR || ctx->de_ftype == EROFS_FT_UNKNOWN) { + struct erofs_inode dir = { .nid = ctx->de_nid }; + + ret = erofs_read_inode_from_disk(&dir); + if (ret) { + erofs_err("read inode failed @ nid %llu", dir.nid | 0ULL); + return ret; + } + + if (S_ISDIR(dir.i_mode)) { + ctx->dir = &dir; + pathctx->pos = pos + len + 1; + ret = erofs_iterate_dir(ctx, false); + pathctx->pos = pos; + if (ret == EROFS_PATHNAME_FOUND) { + pathctx->buf[pos++] = '/'; + strncpy(pathctx->buf + pos, dname, len); + } + return ret; + } else if (ctx->de_ftype == EROFS_FT_DIR) { + erofs_err("i_mode and file_type are inconsistent @ nid %llu", + dir.nid | 0ULL); + } + } + return 0; +} + +int erofs_get_pathname(erofs_nid_t nid, char *buf, size_t size) +{ + int ret; + struct erofs_inode root = { .nid = sbi.root_nid }; + struct erofs_get_pathname_context pathctx = { + .ctx.flags = 0, + .ctx.dir = &root, + .ctx.cb = erofs_get_pathname_iter, + .target_nid = nid, + .buf = buf, + .size = size, + .pos = 0, + }; + + if (nid == root.nid) { + if (size < 2) { + erofs_err("get_pathname buffer not large enough: len 2, size %zd", + size); + return -ERANGE; + } + + buf[0] = '/'; + buf[1] = '\0'; + return 0; + } + + ret = erofs_read_inode_from_disk(&root); + if (ret) { + erofs_err("read inode failed @ nid %llu", root.nid | 0ULL); + return ret; + } + + ret = erofs_iterate_dir(&pathctx.ctx, false); + if (ret == EROFS_PATHNAME_FOUND) + return 0; + if (!ret) + return -ENOENT; + return ret; +} -- cgit v1.2.3 From a9eb5f72cc1dfaa735348fd65d91f27d5fc33810 Mon Sep 17 00:00:00 2001 From: Igor Ostapenko Date: Fri, 17 Dec 2021 16:16:54 +0200 Subject: erofs-utils: dump: remove unused function erofs_checkdirent is no longer needed. Link: https://lore.kernel.org/r/CABjEcnEnYhCSXvbhtas4J-vgBker-U1+FqjJ=Hvz-MOp--kJrw@mail.gmail.com Signed-off-by: Igor Ostapenko Signed-off-by: Gao Xiang --- dump/main.c | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/dump/main.c b/dump/main.c index 997460c..97c8750 100644 --- a/dump/main.c +++ b/dump/main.c @@ -297,37 +297,6 @@ static int erofsdump_readdir(struct erofs_dir_context *ctx) return 0; } -static inline int erofs_checkdirent(struct erofs_dirent *de, - struct erofs_dirent *last_de, - u32 maxsize, const char *dname) -{ - int dname_len; - unsigned int nameoff = le16_to_cpu(de->nameoff); - erofs_nid_t nid = le64_to_cpu(de->nid); - - if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { - erofs_err("invalid de[0].nameoff %u @ nid %llu", - nameoff, nid | 0ULL); - return -EFSCORRUPTED; - } - - dname_len = (de + 1 >= last_de) ? strnlen(dname, maxsize - nameoff) : - le16_to_cpu(de[1].nameoff) - nameoff; - /* a corrupted entry is found */ - if (nameoff + dname_len > maxsize || - dname_len > EROFS_NAME_LEN) { - erofs_err("bogus dirent @ nid %llu", nid | 0ULL); - DBG_BUGON(1); - return -EFSCORRUPTED; - } - if (de->file_type >= EROFS_FT_MAX) { - erofs_err("invalid file type %llu", nid | 0ULL); - return -EFSCORRUPTED; - } - return dname_len; -} - static int erofsdump_map_blocks(struct erofs_inode *inode, struct erofs_map_blocks *map, int flags) { -- cgit v1.2.3 From 2be280dc28ace9c3840aa5e6ca7ff90ef4212cd1 Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Tue, 21 Dec 2021 17:54:18 -0800 Subject: erofs-utils: lib: Add some comments about const-ness around iterate API The new iterate dir API has non-trivial const correctness requirements. Document them in comment. Link: https://lore.kernel.org/r/20211222015419.268586-1-zhangkelvin@google.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- include/erofs/dir.h | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/erofs/dir.h b/include/erofs/dir.h index 336fa9b..1627d3d 100644 --- a/include/erofs/dir.h +++ b/include/erofs/dir.h @@ -39,6 +39,15 @@ typedef int (*erofs_readdir_cb)(struct erofs_dir_context *); * the callback context. |de_namelen| is the exact dirent name length. */ struct erofs_dir_context { + /* + * During execution of |erofs_iterate_dir|, the function needs to + * read the values inside |erofs_inode* dir|. So it is important + * that the callback function does not modify stuct pointed by + * |dir|. It is OK to repoint |dir| to other objects. + * Unfortunately, it's not possible to enforce this restriction + * with const keyword, as |erofs_iterate_dir| needs to modify + * struct pointed by |dir|. + */ struct erofs_inode *dir; erofs_readdir_cb cb; erofs_nid_t pnid; /* optional */ -- cgit v1.2.3 From 3f9dd773c321c3cb8c8ff8c9548171ca152ff1a1 Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Wed, 22 Dec 2021 10:08:22 +0800 Subject: erofs-utils: lib: Add API to get on disk size of an inode Marginally improve code re-use. It's quite common for users to query for compressed size of an inode. Link: https://lore.kernel.org/r/20211222020822.14156-1-xiang@kernel.org Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- dump/main.c | 6 +++--- include/erofs/internal.h | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/dump/main.c b/dump/main.c index 97c8750..9d05d89 100644 --- a/dump/main.c +++ b/dump/main.c @@ -176,7 +176,7 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) return 0; } -static int erofs_get_occupied_size(struct erofs_inode *inode, +static int erofsdump_get_occupied_size(struct erofs_inode *inode, erofs_off_t *size) { *size = 0; @@ -194,7 +194,7 @@ static int erofs_get_occupied_size(struct erofs_inode *inode, break; default: erofs_err("unknown datalayout"); - return -1; + return -ENOTSUP; } return 0; } @@ -270,7 +270,7 @@ static int erofsdump_readdir(struct erofs_dir_context *ctx) stats.files++; stats.file_category_stat[erofs_mode_to_ftype(vi.i_mode)]++; - err = erofs_get_occupied_size(&vi, &occupied_size); + err = erofsdump_get_occupied_size(&vi, &occupied_size); if (err) { erofs_err("get file size failed"); return err; diff --git a/include/erofs/internal.h b/include/erofs/internal.h index d2adf57..2c7b611 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -320,6 +320,27 @@ int erofs_pread(struct erofs_inode *inode, char *buf, int erofs_map_blocks(struct erofs_inode *inode, struct erofs_map_blocks *map, int flags); int erofs_map_dev(struct erofs_sb_info *sbi, struct erofs_map_dev *map); + +static inline int erofs_get_occupied_size(const struct erofs_inode *inode, + erofs_off_t *size) +{ + *size = 0; + switch (inode->datalayout) { + case EROFS_INODE_FLAT_INLINE: + case EROFS_INODE_FLAT_PLAIN: + case EROFS_INODE_CHUNK_BASED: + *size = inode->i_size; + break; + case EROFS_INODE_FLAT_COMPRESSION_LEGACY: + case EROFS_INODE_FLAT_COMPRESSION: + *size = inode->u.i_blocks * EROFS_BLKSIZ; + break; + default: + return -ENOTSUP; + } + return 0; +} + /* zmap.c */ int z_erofs_fill_inode(struct erofs_inode *vi); int z_erofs_map_blocks_iter(struct erofs_inode *vi, -- cgit v1.2.3 From f86bc462eb4b79f8180f98fef5211cbe764b6e97 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Thu, 23 Dec 2021 16:26:13 +0800 Subject: erofs-utils: dump: fix --path after converting erofs_get_pathname() Otherwise it reports: $ dump.erofs --path=/ foo.erofs.img erofs: file path not found @ nid 36 Link: https://lore.kernel.org/r/20211223082614.74875-1-hsiangkao@linux.alibaba.com Fixes: e6082718f743 ("erofs-utils: lib: add API to get pathname of EROFS inode") Signed-off-by: Gao Xiang --- dump/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dump/main.c b/dump/main.c index 9d05d89..0616113 100644 --- a/dump/main.c +++ b/dump/main.c @@ -345,7 +345,7 @@ static void erofsdump_show_fileinfo(bool show_extent) return; } - err = erofs_get_pathname(dumpcfg.nid, path, sizeof(path)); + err = erofs_get_pathname(inode.nid, path, sizeof(path)); if (err < 0) { erofs_err("file path not found @ nid %llu", inode.nid | 0ULL); return; -- cgit v1.2.3 From eb255afa101b52096bd3e5e48f990576190f03bd Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Thu, 23 Dec 2021 16:49:07 +0800 Subject: erofs-utils: lib: fix --blobdev without -Eforce-chunk-indexes blockmap is used by default, chunk indexes should be switched instead if --blobdev is specified. Link: https://lore.kernel.org/r/20211223084907.93020-1-hsiangkao@linux.alibaba.com Fixes: 016bd812be1e ("erofs-utils: mkfs: enable block map chunk format") Signed-off-by: Gao Xiang --- lib/blobchunk.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/blobchunk.c b/lib/blobchunk.c index 5e9a88a..a145be9 100644 --- a/lib/blobchunk.c +++ b/lib/blobchunk.c @@ -113,7 +113,7 @@ int erofs_blob_write_chunk_indexes(struct erofs_inode *inode, if (multidev) { idx.device_id = 1; - inode->u.chunkformat |= EROFS_CHUNK_FORMAT_INDEXES; + DBG_BUGON(!(inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES)); } else { base_blkaddr = remapped_base; } @@ -171,6 +171,8 @@ int erofs_blob_write_chunked_file(struct erofs_inode *inode) int fd, ret; inode->u.chunkformat |= inode->u.chunkbits - LOG_BLOCK_SIZE; + if (multidev) + inode->u.chunkformat |= EROFS_CHUNK_FORMAT_INDEXES; if (inode->u.chunkformat & EROFS_CHUNK_FORMAT_INDEXES) unit = sizeof(struct erofs_inode_chunk_index); -- cgit v1.2.3 From cb2f110e73680ab91857914c239abf99c4933852 Mon Sep 17 00:00:00 2001 From: Yue Hu Date: Fri, 14 Jan 2022 19:50:52 +0800 Subject: erofs-utils: fuse: support tail-packing inline compressed data Add tail-packing inline compressed data support for erofsfuse. Link: https://lore.kernel.org/r/20220114115053.28824-2-hsiangkao@linux.alibaba.com Signed-off-by: Yue Hu Signed-off-by: Gao Xiang --- include/erofs/internal.h | 5 +++ include/erofs_fs.h | 10 ++++- lib/decompress.c | 5 ++- lib/namei.c | 2 +- lib/zmap.c | 99 ++++++++++++++++++++++++++++++++++++------------ 5 files changed, 92 insertions(+), 29 deletions(-) diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 2c7b611..68cab50 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -131,6 +131,7 @@ EROFS_FEATURE_FUNCS(compr_cfgs, incompat, INCOMPAT_COMPR_CFGS) EROFS_FEATURE_FUNCS(big_pcluster, incompat, INCOMPAT_BIG_PCLUSTER) EROFS_FEATURE_FUNCS(chunked_file, incompat, INCOMPAT_CHUNKED_FILE) EROFS_FEATURE_FUNCS(device_table, incompat, INCOMPAT_DEVICE_TABLE) +EROFS_FEATURE_FUNCS(ztailpacking, incompat, INCOMPAT_ZTAILPACKING) EROFS_FEATURE_FUNCS(sb_chksum, compat, COMPAT_SB_CHKSUM) #define EROFS_I_EA_INITED (1 << 0) @@ -192,6 +193,9 @@ struct erofs_inode { uint8_t z_algorithmtype[2]; uint8_t z_logical_clusterbits; uint8_t z_physical_clusterblks; + uint64_t z_tailextent_headlcn; + unsigned int z_idataoff; +#define z_idata_size idata_size }; }; #ifdef WITH_ANDROID @@ -295,6 +299,7 @@ struct erofs_map_blocks { * approach instead if possible since it's more metadata lightweight.) */ #define EROFS_GET_BLOCKS_FIEMAP 0x0002 +#define EROFS_GET_BLOCKS_FINDTAIL 0x0008 enum { Z_EROFS_COMPRESSION_SHIFTED = Z_EROFS_COMPRESSION_MAX, diff --git a/include/erofs_fs.h b/include/erofs_fs.h index 9a91877..59d9bbb 100644 --- a/include/erofs_fs.h +++ b/include/erofs_fs.h @@ -23,12 +23,14 @@ #define EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER 0x00000002 #define EROFS_FEATURE_INCOMPAT_CHUNKED_FILE 0x00000004 #define EROFS_FEATURE_INCOMPAT_DEVICE_TABLE 0x00000008 +#define EROFS_FEATURE_INCOMPAT_ZTAILPACKING 0x00000010 #define EROFS_ALL_FEATURE_INCOMPAT \ (EROFS_FEATURE_INCOMPAT_LZ4_0PADDING | \ EROFS_FEATURE_INCOMPAT_COMPR_CFGS | \ EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER | \ EROFS_FEATURE_INCOMPAT_CHUNKED_FILE | \ - EROFS_FEATURE_INCOMPAT_DEVICE_TABLE) + EROFS_FEATURE_INCOMPAT_DEVICE_TABLE | \ + EROFS_FEATURE_INCOMPAT_ZTAILPACKING) #define EROFS_SB_EXTSLOT_SIZE 16 @@ -290,13 +292,17 @@ struct z_erofs_lzma_cfgs { * (4B) + 2B + (4B) if compacted 2B is on. * bit 1 : HEAD1 big pcluster (0 - off; 1 - on) * bit 2 : HEAD2 big pcluster (0 - off; 1 - on) + * bit 3 : tailpacking inline pcluster (0 - off; 1 - on) */ #define Z_EROFS_ADVISE_COMPACTED_2B 0x0001 #define Z_EROFS_ADVISE_BIG_PCLUSTER_1 0x0002 #define Z_EROFS_ADVISE_BIG_PCLUSTER_2 0x0004 +#define Z_EROFS_ADVISE_INLINE_PCLUSTER 0x0008 struct z_erofs_map_header { - __le32 h_reserved1; + __le16 h_reserved1; + /* record the size of tailpacking data */ + __le16 h_idata_size; __le16 h_advise; /* * bit 0-3 : algorithm type of head 1 (logical cluster type 01); diff --git a/lib/decompress.c b/lib/decompress.c index 359dae7..1661f91 100644 --- a/lib/decompress.c +++ b/lib/decompress.c @@ -110,6 +110,9 @@ static int z_erofs_decompress_lz4(struct z_erofs_decompress_req *rq) rq->decodedlength); if (ret != (int)rq->decodedlength) { + erofs_err("failed to %s decompress %d in[%u, %u] out[%u]", + rq->partial_decoding ? "partial" : "full", + ret, rq->inputsize, inputmargin, rq->decodedlength); ret = -EIO; goto out; } @@ -129,7 +132,7 @@ out: int z_erofs_decompress(struct z_erofs_decompress_req *rq) { if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED) { - if (rq->inputsize != EROFS_BLKSIZ) + if (rq->inputsize > EROFS_BLKSIZ) return -EFSCORRUPTED; DBG_BUGON(rq->decodedlength > EROFS_BLKSIZ); diff --git a/lib/namei.c b/lib/namei.c index 4124170..97f0f80 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -137,7 +137,7 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->u.chunkbits = LOG_BLOCK_SIZE + (vi->u.chunkformat & EROFS_CHUNK_FORMAT_BLKBITS_MASK); } else if (erofs_inode_is_data_compressed(vi->datalayout)) - z_erofs_fill_inode(vi); + return z_erofs_fill_inode(vi); return 0; bogusimode: erofs_err("bogus i_mode (%o) @ nid %llu", vi->i_mode, vi->nid | 0ULL); diff --git a/lib/zmap.c b/lib/zmap.c index abc8bab..95745c5 100644 --- a/lib/zmap.c +++ b/lib/zmap.c @@ -10,9 +10,14 @@ #include "erofs/io.h" #include "erofs/print.h" +static int z_erofs_do_map_blocks(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags); + int z_erofs_fill_inode(struct erofs_inode *vi) { if (!erofs_sb_has_big_pcluster() && + !erofs_sb_has_ztailpacking() && vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { vi->z_advise = 0; vi->z_algorithmtype[0] = 0; @@ -35,6 +40,7 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi) return 0; DBG_BUGON(!erofs_sb_has_big_pcluster() && + !erofs_sb_has_ztailpacking() && vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY); pos = round_up(iloc(vi->nid) + vi->inode_isize + vi->xattr_isize, 8); @@ -61,6 +67,22 @@ static int z_erofs_fill_inode_lazy(struct erofs_inode *vi) vi->nid * 1ULL); return -EFSCORRUPTED; } + + if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) { + struct erofs_map_blocks map = { .index = UINT_MAX }; + + vi->idata_size = le16_to_cpu(h->h_idata_size); + ret = z_erofs_do_map_blocks(vi, &map, + EROFS_GET_BLOCKS_FINDTAIL); + if (!map.m_plen || + erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) { + erofs_err("invalid tail-packing pclustersize %llu", + map.m_plen | 0ULL); + return -EFSCORRUPTED; + } + if (ret < 0) + return ret; + } vi->flags |= EROFS_I_Z_INITED; return 0; } @@ -76,6 +98,7 @@ struct z_erofs_maprecorder { u16 clusterofs; u16 delta[2]; erofs_blk_t pblk, compressedlcs; + erofs_off_t nextpackoff; }; static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m, @@ -114,6 +137,7 @@ static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m, if (err) return err; + m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index); m->lcn = lcn; di = m->kaddr + erofs_blkoff(pos); @@ -186,12 +210,12 @@ static int get_compacted_la_distance(unsigned int lclusterbits, static int unpack_compacted_index(struct z_erofs_maprecorder *m, unsigned int amortizedshift, - unsigned int eofs, bool lookahead) + erofs_off_t pos, bool lookahead) { struct erofs_inode *const vi = m->inode; const unsigned int lclusterbits = vi->z_logical_clusterbits; const unsigned int lomask = (1 << lclusterbits) - 1; - unsigned int vcnt, base, lo, encodebits, nblk; + unsigned int vcnt, base, lo, encodebits, nblk, eofs; int i; u8 *in, type; bool big_pcluster; @@ -203,8 +227,12 @@ static int unpack_compacted_index(struct z_erofs_maprecorder *m, else return -EOPNOTSUPP; + /* it doesn't equal to round_up(..) */ + m->nextpackoff = round_down(pos, vcnt << amortizedshift) + + (vcnt << amortizedshift); big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; + eofs = erofs_blkoff(pos); base = round_down(eofs, vcnt << amortizedshift); in = m->kaddr + base; @@ -341,8 +369,7 @@ out: err = z_erofs_reload_indexes(m, erofs_blknr(pos)); if (err) return err; - return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos), - lookahead); + return unpack_compacted_index(m, amortizedshift, pos, lookahead); } static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m, @@ -415,6 +442,7 @@ static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN && m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD); + if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN || !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) { map->m_plen = 1 << lclusterbits; @@ -513,10 +541,11 @@ static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) return 0; } -int z_erofs_map_blocks_iter(struct erofs_inode *vi, - struct erofs_map_blocks *map, - int flags) +static int z_erofs_do_map_blocks(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags) { + bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER; struct z_erofs_maprecorder m = { .inode = vi, .map = map, @@ -527,20 +556,8 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, unsigned long initial_lcn; unsigned long long ofs, end; - /* when trying to read beyond EOF, leave it unmapped */ - if (map->m_la >= vi->i_size) { - map->m_llen = map->m_la + 1 - vi->i_size; - map->m_la = vi->i_size; - map->m_flags = 0; - goto out; - } - - err = z_erofs_fill_inode_lazy(vi); - if (err) - goto out; - lclusterbits = vi->z_logical_clusterbits; - ofs = map->m_la; + ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? vi->i_size - 1 : map->m_la; initial_lcn = ofs >> lclusterbits; endoff = ofs & ((1 << lclusterbits) - 1); @@ -548,6 +565,9 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, if (err) goto out; + if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL)) + vi->z_idataoff = m.nextpackoff; + map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED; end = (m.lcn + 1ULL) << lclusterbits; switch (m.type) { @@ -583,11 +603,18 @@ int z_erofs_map_blocks_iter(struct erofs_inode *vi, } map->m_llen = end - map->m_la; - map->m_pa = blknr_to_addr(m.pblk); - - err = z_erofs_get_extent_compressedlen(&m, initial_lcn); - if (err) - goto out; + if (flags & EROFS_GET_BLOCKS_FINDTAIL) + vi->z_tailextent_headlcn = m.lcn; + if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { + map->m_flags |= EROFS_MAP_META; + map->m_pa = vi->z_idataoff; + map->m_plen = vi->z_idata_size; + } else { + map->m_pa = blknr_to_addr(m.pblk); + err = z_erofs_get_extent_compressedlen(&m, initial_lcn); + if (err) + goto out; + } if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; @@ -604,7 +631,29 @@ out: erofs_dbg("m_la %" PRIu64 " m_pa %" PRIu64 " m_llen %" PRIu64 " m_plen %" PRIu64 " m_flags 0%o", map->m_la, map->m_pa, map->m_llen, map->m_plen, map->m_flags); + return err; +} +int z_erofs_map_blocks_iter(struct erofs_inode *vi, + struct erofs_map_blocks *map, + int flags) +{ + int err = 0; + + /* when trying to read beyond EOF, leave it unmapped */ + if (map->m_la >= vi->i_size) { + map->m_llen = map->m_la + 1 - vi->i_size; + map->m_la = vi->i_size; + map->m_flags = 0; + goto out; + } + + err = z_erofs_fill_inode_lazy(vi); + if (err) + goto out; + + err = z_erofs_do_map_blocks(vi, map, flags); +out: DBG_BUGON(err < 0 && err != -ENOMEM); return err; } -- cgit v1.2.3 From ddea76ad592f2a05d53fb229dff19b65f44753a0 Mon Sep 17 00:00:00 2001 From: Yue Hu Date: Fri, 14 Jan 2022 19:50:53 +0800 Subject: erofs-utils: mkfs: support tail-packing inline compressed data Currently, we only support tail-packing inline data for uncompressed files, let's support it as well for compressed files. Link: https://lore.kernel.org/r/20220114115053.28824-3-hsiangkao@linux.alibaba.com Signed-off-by: Yue Hu Signed-off-by: Gao Xiang --- include/erofs/config.h | 1 + include/erofs/internal.h | 1 + lib/compress.c | 122 +++++++++++++++++++++++++++++++++++++---------- lib/compressor.c | 16 +++++-- lib/compressor.h | 2 +- lib/inode.c | 57 +++++++++++++++------- mkfs/main.c | 8 ++++ 7 files changed, 159 insertions(+), 48 deletions(-) diff --git a/include/erofs/config.h b/include/erofs/config.h index e41c079..a3f9bac 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -43,6 +43,7 @@ struct erofs_configure { char c_timeinherit; char c_chunkbits; bool c_noinline_data; + bool c_ztailpacking; #ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd; diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 68cab50..3615297 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -175,6 +175,7 @@ struct erofs_inode { unsigned char inode_isize; /* inline tail-end packing size */ unsigned short idata_size; + bool compressed_idata; unsigned int xattr_isize; unsigned int extent_isize; diff --git a/lib/compress.c b/lib/compress.c index 6f16e0c..e81c070 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -72,7 +72,10 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, /* whether the tail-end uncompressed block or not */ if (!d1) { - /* TODO: tail-packing inline compressed data */ + /* + * A lcluster cannot have there parts with the middle one which + * is well-compressed. Such tail pcluster cannot be inlined. + */ DBG_BUGON(!raw); type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN; advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT); @@ -162,6 +165,21 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode) return cfg.c_pclusterblks_def; } +static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data, + unsigned int len, bool raw) +{ + inode->idata_size = len; + inode->compressed_idata = !raw; + + inode->idata = malloc(inode->idata_size); + if (!inode->idata) + return -ENOMEM; + erofs_dbg("Recording %u %scompressed inline data", + inode->idata_size, raw ? "un" : ""); + memcpy(inode->idata, data, inode->idata_size); + return len; +} + static int vle_compress_one(struct erofs_inode *inode, struct z_erofs_vle_compress_ctx *ctx, bool final) @@ -172,16 +190,23 @@ static int vle_compress_one(struct erofs_inode *inode, int ret; static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ]; char *const dst = dstbuf + EROFS_BLKSIZ; + bool trailing = false, tailpcluster = false; while (len) { - const unsigned int pclustersize = + unsigned int pclustersize = z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ; bool raw; + DBG_BUGON(tailpcluster); if (len <= pclustersize) { if (final) { - if (len <= EROFS_BLKSIZ) + /* TODO: compress with 2 pclusters instead */ + if (cfg.c_ztailpacking) { + trailing = true; + pclustersize = EROFS_BLKSIZ; + } else if (len <= EROFS_BLKSIZ) { goto nocompression; + } } else { break; } @@ -189,20 +214,54 @@ static int vle_compress_one(struct erofs_inode *inode, count = min(len, cfg.c_max_decompressed_extent_bytes); ret = erofs_compress_destsize(h, ctx->queue + ctx->head, - &count, dst, pclustersize); - if (ret <= 0) { - if (ret != -EAGAIN) { + &count, dst, pclustersize, + !(final && len == count)); + + /* XXX: need to be polished, yet do it correctly first. */ + if (cfg.c_ztailpacking && final) { + if (ret <= 0 && len < EROFS_BLKSIZ) { + DBG_BUGON(!trailing); + tailpcluster = true; + } else if (ret > 0 && len == count && + /* less than 1 compressed block */ + ret < EROFS_BLKSIZ) { + tailpcluster = true; + } + } + + if (ret <= 0 || (tailpcluster && + ctx->clusterofs + len < EROFS_BLKSIZ)) { + if (ret <= 0 && ret != -EAGAIN) { erofs_err("failed to compress %s: %s", inode->i_srcpath, erofs_strerror(ret)); } + + if (tailpcluster && len < EROFS_BLKSIZ) + ret = z_erofs_fill_inline_data(inode, + ctx->queue + ctx->head, + len, true); + else nocompression: - ret = write_uncompressed_extent(ctx, &len, dst); + ret = write_uncompressed_extent(ctx, &len, dst); + if (ret < 0) return ret; count = ret; + + /* + * XXX: For now, we have to leave `ctx->compressedblks + * = 1' since there is no way to generate compressed + * indexes after the time that ztailpacking is decided. + */ ctx->compressedblks = 1; raw = true; + } else if (tailpcluster && ret < EROFS_BLKSIZ) { + ret = z_erofs_fill_inline_data(inode, dst, ret, false); + if (ret < 0) + return ret; + ctx->compressedblks = 1; + raw = false; } else { const unsigned int tailused = ret & (EROFS_BLKSIZ - 1); const unsigned int padding = @@ -449,6 +508,7 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, { struct z_erofs_map_header h = { .h_advise = cpu_to_le16(inode->z_advise), + .h_idata_size = cpu_to_le16(inode->idata_size), .h_algorithmtype = inode->z_algorithmtype[1] << 4 | inode->z_algorithmtype[0], /* lclustersize */ @@ -476,7 +536,7 @@ int erofs_write_compressed_file(struct erofs_inode *inode) fd = open(inode->i_srcpath, O_RDONLY | O_BINARY); if (fd < 0) { ret = -errno; - goto err_free; + goto err_free_meta; } /* allocate main data buffer */ @@ -504,8 +564,6 @@ int erofs_write_compressed_file(struct erofs_inode *inode) inode->z_algorithmtype[1] = algorithmtype[1]; inode->z_logical_clusterbits = LOG_BLOCK_SIZE; - z_erofs_write_mapheader(inode, compressmeta); - blkaddr = erofs_mapbh(bh->block); /* start_blkaddr */ ctx.blkaddr = blkaddr; ctx.metacur = compressmeta + Z_EROFS_LEGACY_MAP_HEADER_SIZE; @@ -534,35 +592,42 @@ int erofs_write_compressed_file(struct erofs_inode *inode) /* do the final round */ ret = vle_compress_one(inode, &ctx, true); if (ret) - goto err_bdrop; + goto err_free_idata; /* fall back to no compression mode */ compressed_blocks = ctx.blkaddr - blkaddr; - if (compressed_blocks >= BLK_ROUND_UP(inode->i_size)) { - ret = -ENOSPC; - goto err_bdrop; - } + DBG_BUGON(compressed_blocks < !!inode->idata_size); + compressed_blocks -= !!inode->idata_size; vle_write_indexes_final(&ctx); + legacymetasize = ctx.metacur - compressmeta; + /* estimate if data compression saves space or not */ + if (compressed_blocks * EROFS_BLKSIZ + inode->idata_size + + legacymetasize >= inode->i_size) { + ret = -ENOSPC; + goto err_free_idata; + } + z_erofs_write_mapheader(inode, compressmeta); close(fd); - DBG_BUGON(!compressed_blocks); - ret = erofs_bh_balloon(bh, blknr_to_addr(compressed_blocks)); - DBG_BUGON(ret != EROFS_BLKSIZ); + if (compressed_blocks) { + ret = erofs_bh_balloon(bh, blknr_to_addr(compressed_blocks)); + DBG_BUGON(ret != EROFS_BLKSIZ); + } else { + DBG_BUGON(!inode->idata_size); + } erofs_info("compressed %s (%llu bytes) into %u blocks", inode->i_srcpath, (unsigned long long)inode->i_size, compressed_blocks); - /* - * TODO: need to move erofs_bdrop to erofs_write_tail_end - * when both mkfs & kernel support compression inline. - */ - erofs_bdrop(bh, false); - inode->idata_size = 0; + if (inode->idata_size) + inode->bh_data = bh; + else + erofs_bdrop(bh, false); + inode->u.i_blocks = compressed_blocks; - legacymetasize = ctx.metacur - compressmeta; if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { inode->extent_isize = legacymetasize; } else { @@ -575,11 +640,16 @@ int erofs_write_compressed_file(struct erofs_inode *inode) erofs_droid_blocklist_write(inode, blkaddr, compressed_blocks); return 0; +err_free_idata: + if (inode->idata) { + free(inode->idata); + inode->idata = NULL; + } err_bdrop: erofs_bdrop(bh, true); /* revoke buffer */ err_close: close(fd); -err_free: +err_free_meta: free(compressmeta); return ret; } diff --git a/lib/compressor.c b/lib/compressor.c index 3e04a3a..3666496 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -24,23 +24,29 @@ static const struct erofs_compressor *compressors[] = { int erofs_compress_destsize(const struct erofs_compress *c, const void *src, unsigned int *srcsize, - void *dst, unsigned int dstsize) + void *dst, unsigned int dstsize, bool inblocks) { - unsigned int uncompressed_size; + unsigned int uncompressed_capacity, compressed_size; int ret; DBG_BUGON(!c->alg); if (!c->alg->compress_destsize) return -ENOTSUP; + uncompressed_capacity = *srcsize; ret = c->alg->compress_destsize(c, src, srcsize, dst, dstsize); if (ret < 0) return ret; + /* XXX: ret >= EROFS_BLKSIZ is a temporary hack for ztailpacking */ + if (inblocks || ret >= EROFS_BLKSIZ || + uncompressed_capacity != *srcsize) + compressed_size = roundup(ret, EROFS_BLKSIZ); + else + compressed_size = ret; + DBG_BUGON(c->compress_threshold < 100); /* check if there is enough gains to compress */ - uncompressed_size = *srcsize; - if (roundup(ret, EROFS_BLKSIZ) >= uncompressed_size * - c->compress_threshold / 100) + if (*srcsize <= compressed_size * c->compress_threshold / 100) return -EAGAIN; return ret; } diff --git a/lib/compressor.h b/lib/compressor.h index 728dd0b..cf063f1 100644 --- a/lib/compressor.h +++ b/lib/compressor.h @@ -47,7 +47,7 @@ extern const struct erofs_compressor erofs_compressor_lzma; int erofs_compress_destsize(const struct erofs_compress *c, const void *src, unsigned int *srcsize, - void *dst, unsigned int dstsize); + void *dst, unsigned int dstsize, bool inblocks); int erofs_compressor_setlevel(struct erofs_compress *c, int compression_level); int erofs_compressor_init(struct erofs_compress *c, char *alg_name); diff --git a/lib/inode.c b/lib/inode.c index b6d3092..3fd4b69 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -593,28 +593,29 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode) inodesize = Z_EROFS_VLE_EXTENT_ALIGN(inodesize) + inode->extent_isize; - if (is_inode_layout_compression(inode)) - goto noinline; + /* TODO: tailpacking inline of chunk-based format isn't finalized */ if (inode->datalayout == EROFS_INODE_CHUNK_BASED) goto noinline; - if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) { - inode->datalayout = EROFS_INODE_FLAT_PLAIN; - goto noinline; + if (!is_inode_layout_compression(inode)) { + if (cfg.c_noinline_data && S_ISREG(inode->i_mode)) { + inode->datalayout = EROFS_INODE_FLAT_PLAIN; + goto noinline; + } + /* + * If the file sizes of uncompressed files are block-aligned, + * should use the EROFS_INODE_FLAT_PLAIN data layout. + */ + if (!inode->idata_size) + inode->datalayout = EROFS_INODE_FLAT_PLAIN; } - /* - * if the file size is block-aligned for uncompressed files, - * should use EROFS_INODE_FLAT_PLAIN data mapping mode. - */ - if (!inode->idata_size) - inode->datalayout = EROFS_INODE_FLAT_PLAIN; - bh = erofs_balloc(INODE, inodesize, 0, inode->idata_size); if (bh == ERR_PTR(-ENOSPC)) { int ret; - inode->datalayout = EROFS_INODE_FLAT_PLAIN; + if (!is_inode_layout_compression(inode)) + inode->datalayout = EROFS_INODE_FLAT_PLAIN; noinline: /* expend an extra block for tail-end data */ ret = erofs_prepare_tail_block(inode); @@ -627,7 +628,20 @@ noinline: } else if (IS_ERR(bh)) { return PTR_ERR(bh); } else if (inode->idata_size) { - inode->datalayout = EROFS_INODE_FLAT_INLINE; + if (is_inode_layout_compression(inode)) { + struct z_erofs_map_header *h = inode->compressmeta; + + DBG_BUGON(!cfg.c_ztailpacking); + h->h_advise |= Z_EROFS_ADVISE_INLINE_PCLUSTER; + erofs_dbg("Inline %scompressed data (%u bytes) to %s", + inode->compressed_idata ? "" : "un", + inode->idata_size, inode->i_srcpath); + erofs_sb_set_ztailpacking(); + } else { + inode->datalayout = EROFS_INODE_FLAT_INLINE; + erofs_dbg("Inline tail-end data (%u bytes) to %s", + inode->idata_size, inode->i_srcpath); + } /* allocate inline buffer */ ibh = erofs_battach(bh, META, inode->idata_size); @@ -685,15 +699,26 @@ static int erofs_write_tail_end(struct erofs_inode *inode) erofs_droid_blocklist_write_tail_end(inode, NULL_ADDR); } else { int ret; - erofs_off_t pos; + erofs_off_t pos, zero_pos; erofs_mapbh(bh->block); pos = erofs_btell(bh, true) - EROFS_BLKSIZ; + + /* 0'ed data should be padded at head for 0padding conversion */ + if (erofs_sb_has_lz4_0padding() && inode->compressed_idata) { + zero_pos = pos; + pos += EROFS_BLKSIZ - inode->idata_size; + } else { + /* pad 0'ed data for the other cases */ + zero_pos = pos + inode->idata_size; + } ret = dev_write(inode->idata, pos, inode->idata_size); if (ret) return ret; + + DBG_BUGON(inode->idata_size > EROFS_BLKSIZ); if (inode->idata_size < EROFS_BLKSIZ) { - ret = dev_fillzero(pos + inode->idata_size, + ret = dev_fillzero(zero_pos, EROFS_BLKSIZ - inode->idata_size, false); if (ret) diff --git a/mkfs/main.c b/mkfs/main.c index 90cedde..9eb696a 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -187,6 +187,12 @@ static int parse_extended_opts(const char *opts) return -EINVAL; cfg.c_force_chunkformat = FORCE_INODE_CHUNK_INDEXES; } + + if (MATCH_EXTENTED_OPT("ztailpacking", token, keylen)) { + if (vallen) + return -EINVAL; + cfg.c_ztailpacking = true; + } } return 0; } @@ -625,6 +631,8 @@ int main(int argc, char **argv) erofs_show_config(); if (erofs_sb_has_chunked_file()) erofs_warn("EXPERIMENTAL chunked file feature in use. Use at your own risk!"); + if (cfg.c_ztailpacking) + erofs_warn("EXPERIMENTAL compressed inline data feature in use. Use at your own risk!"); erofs_set_fs_root(cfg.c_src_path); #ifndef NDEBUG if (cfg.c_random_pclusterblks) -- cgit v1.2.3 From 412c8f908132a45e7e096f6e8bac57bf58772d78 Mon Sep 17 00:00:00 2001 From: Igor Ostapenko Date: Thu, 20 Jan 2022 09:32:03 +0800 Subject: erofs-utils: fsck: add --extract=X support to extract to path X Add support to extract directories, regular files and symlinks. Allocation for extract_path is done only once, then the buffer is reused. Raw and compressed data chunks are handled with a unified function to avoid code duplication, compressed data is verified linearly (with EROFS_GET_BLOCKS_FIEMAP) instead of lookback, as it's problematic to extract data when looking backwards. Link: https://lore.kernel.org/r/20220120013203.25990-1-xiang@kernel.org Signed-off-by: Igor Ostapenko Signed-off-by: Gao Xiang --- configure.ac | 3 +- fsck/main.c | 577 +++++++++++++++++++++++++++++++++++++++++++---------------- mkfs/main.c | 2 +- 3 files changed, 426 insertions(+), 156 deletions(-) diff --git a/configure.ac b/configure.ac index a5de291..fa917e6 100644 --- a/configure.ac +++ b/configure.ac @@ -188,7 +188,8 @@ AC_CHECK_FUNCS(m4_flatten([ strerror strrchr strtoull - tmpfile64])) + tmpfile64 + utimensat])) # Configure debug mode AS_IF([test "x$enable_debug" != "xno"], [], [ diff --git a/fsck/main.c b/fsck/main.c index 30d0a1b..14534b9 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -6,9 +6,12 @@ #include #include #include +#include +#include #include #include "erofs/print.h" #include "erofs/io.h" +#include "erofs/compress.h" #include "erofs/decompress.h" #include "erofs/dir.h" @@ -18,6 +21,10 @@ struct erofsfsck_cfg { bool corrupted; bool print_comp_ratio; bool check_decomp; + char *extract_path; + size_t extract_pos; + bool overwrite, preserve_owner, preserve_perms; + mode_t umask; u64 physical_blocks; u64 logical_blocks; }; @@ -25,22 +32,47 @@ static struct erofsfsck_cfg fsckcfg; static struct option long_options[] = { {"help", no_argument, 0, 1}, - {"extract", no_argument, 0, 2}, + {"extract", optional_argument, 0, 2}, {"device", required_argument, 0, 3}, + {"no-same-owner", no_argument, 0, 4}, + {"no-same-permissions", no_argument, 0, 5}, + {"same-owner", no_argument, 0, 6}, + {"same-permissions", no_argument, 0, 7}, + {"overwrite", no_argument, 0, 8}, {0, 0, 0, 0}, }; +static void print_available_decompressors(FILE *f, const char *delim) +{ + unsigned int i = 0; + const char *s; + + while ((s = z_erofs_list_available_compressors(i)) != NULL) { + if (i++) + fputs(delim, f); + fputs(s, f); + } + fputc('\n', f); +} + static void usage(void) { fputs("usage: [options] IMAGE\n\n" - "Check erofs filesystem integrity of IMAGE, and [options] are:\n" - " -V print the version number of fsck.erofs and exit.\n" - " -d# set output message level to # (maximum 9)\n" - " -p print total compression ratio of all files\n" - " --device=X specify an extra device to be used together\n" - " --extract check if all files are well encoded\n" - " --help display this help and exit.\n", - stderr); + "Check erofs filesystem compatibility and integrity of IMAGE, and [options] are:\n" + " -V print the version number of fsck.erofs and exit\n" + " -d# set output message level to # (maximum 9)\n" + " -p print total compression ratio of all files\n" + " --device=X specify an extra device to be used together\n" + " --extract[=X] check if all files are well encoded, optionally extract to X\n" + " --help display this help and exit\n" + "\nExtraction options (--extract=X is required):\n" + " --no-same-owner extract files as yourself\n" + " --no-same-permissions apply the user's umask when extracting permission\n" + " --same-permissions extract information about file permissions\n" + " --same-owner extract files about the file ownerships\n" + " --overwrite if file already exists then overwrite\n" + "\nSupported algorithms are: ", stderr); + print_available_decompressors(stderr, ", "); } static void erofsfsck_print_version(void) @@ -74,6 +106,23 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) exit(0); case 2: fsckcfg.check_decomp = true; + if (optarg) { + size_t len = strlen(optarg); + + if (len == 0) + return -EINVAL; + + /* remove trailing slashes except root */ + while (len > 1 && optarg[len - 1] == '/') + len--; + + fsckcfg.extract_path = malloc(PATH_MAX); + if (!fsckcfg.extract_path) + return -ENOMEM; + strncpy(fsckcfg.extract_path, optarg, len); + fsckcfg.extract_path[len] = '\0'; + fsckcfg.extract_pos = len; + } break; case 3: ret = blob_open_ro(optarg); @@ -81,6 +130,21 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) return ret; ++sbi.extra_devices; break; + case 4: + fsckcfg.preserve_owner = false; + break; + case 5: + fsckcfg.preserve_perms = false; + break; + case 6: + fsckcfg.preserve_owner = true; + break; + case 7: + fsckcfg.preserve_perms = true; + break; + case 8: + fsckcfg.overwrite = true; + break; default: return -EINVAL; } @@ -89,6 +153,11 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) if (optind >= argc) return -EINVAL; + if (!fsckcfg.extract_path) + if (fsckcfg.overwrite || fsckcfg.preserve_owner || + fsckcfg.preserve_perms) + return -EINVAL; + cfg.c_img_path = strdup(argv[optind++]); if (!cfg.c_img_path) return -ENOMEM; @@ -100,6 +169,41 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) return 0; } +static void erofsfsck_set_attributes(struct erofs_inode *inode, char *path) +{ + int ret; + +#ifdef HAVE_UTIMENSAT + const struct timespec times[2] = { + [0] = { .tv_sec = inode->i_ctime, + .tv_nsec = inode->i_ctime_nsec }, + [1] = { .tv_sec = inode->i_ctime, + .tv_nsec = inode->i_ctime_nsec }, + }; + + if (utimensat(AT_FDCWD, path, times, AT_SYMLINK_NOFOLLOW) < 0) +#else + if (utime(path, &((struct utimbuf){.actime = inode->i_ctime, + .modtime = inode->i_ctime})) < 0) +#endif + erofs_warn("failed to set times: %s", path); + + if (!S_ISLNK(inode->i_mode)) { + if (fsckcfg.preserve_perms) + ret = chmod(path, inode->i_mode); + else + ret = chmod(path, inode->i_mode & ~fsckcfg.umask); + if (ret < 0) + erofs_warn("failed to set permissions: %s", path); + } + + if (fsckcfg.preserve_owner) { + ret = lchown(path, inode->i_uid, inode->i_gid); + if (ret < 0) + erofs_warn("failed to change ownership: %s", path); + } +} + static int erofs_check_sb_chksum(void) { int ret; @@ -127,137 +231,6 @@ static int erofs_check_sb_chksum(void) return 0; } -static int verify_uncompressed_inode(struct erofs_inode *inode) -{ - struct erofs_map_blocks map = { - .index = UINT_MAX, - }; - int ret; - erofs_off_t ptr = 0; - u64 i_blocks = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); - - while (ptr < inode->i_size) { - map.m_la = ptr; - ret = erofs_map_blocks(inode, &map, 0); - if (ret) - return ret; - - if (map.m_plen != map.m_llen || ptr != map.m_la) { - erofs_err("broken data chunk layout m_la %" PRIu64 " ptr %" PRIu64 " m_llen %" PRIu64 " m_plen %" PRIu64, - map.m_la, ptr, map.m_llen, map.m_plen); - return -EFSCORRUPTED; - } - - if (!(map.m_flags & EROFS_MAP_MAPPED) && !map.m_llen) { - /* reached EOF */ - ptr = inode->i_size; - continue; - } - - ptr += map.m_llen; - } - - if (fsckcfg.print_comp_ratio) { - fsckcfg.logical_blocks += i_blocks; - fsckcfg.physical_blocks += i_blocks; - } - - return 0; -} - -static int verify_compressed_inode(struct erofs_inode *inode) -{ - struct erofs_map_blocks map = { - .index = UINT_MAX, - }; - struct erofs_map_dev mdev; - int ret = 0; - u64 pchunk_len = 0; - erofs_off_t end = inode->i_size; - unsigned int raw_size = 0, buffer_size = 0; - char *raw = NULL, *buffer = NULL; - - while (end > 0) { - map.m_la = end - 1; - - ret = z_erofs_map_blocks_iter(inode, &map, 0); - if (ret) - goto out; - - if (end > map.m_la + map.m_llen) { - erofs_err("broken compressed chunk layout m_la %" PRIu64 " m_llen %" PRIu64 " end %" PRIu64, - map.m_la, map.m_llen, end); - ret = -EFSCORRUPTED; - goto out; - } - - pchunk_len += map.m_plen; - end = map.m_la; - - if (!fsckcfg.check_decomp || !(map.m_flags & EROFS_MAP_MAPPED)) - continue; - - if (map.m_plen > raw_size) { - raw_size = map.m_plen; - raw = realloc(raw, raw_size); - BUG_ON(!raw); - } - - if (map.m_llen > buffer_size) { - buffer_size = map.m_llen; - buffer = realloc(buffer, buffer_size); - BUG_ON(!buffer); - } - - mdev = (struct erofs_map_dev) { - .m_deviceid = map.m_deviceid, - .m_pa = map.m_pa, - }; - ret = erofs_map_dev(&sbi, &mdev); - if (ret) { - erofs_err("failed to map device of m_pa %" PRIu64 ", m_deviceid %u @ nid %llu: %d", - map.m_pa, map.m_deviceid, inode->nid | 0ULL, ret); - goto out; - } - - ret = dev_read(mdev.m_deviceid, raw, mdev.m_pa, map.m_plen); - if (ret < 0) { - erofs_err("failed to read compressed data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d", - mdev.m_pa, map.m_plen, inode->nid | 0ULL, ret); - goto out; - } - - ret = z_erofs_decompress(&(struct z_erofs_decompress_req) { - .in = raw, - .out = buffer, - .decodedskip = 0, - .inputsize = map.m_plen, - .decodedlength = map.m_llen, - .alg = map.m_algorithmformat, - .partial_decoding = 0 - }); - - if (ret < 0) { - erofs_err("failed to decompress data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d", - mdev.m_pa, map.m_plen, inode->nid | 0ULL, ret); - goto out; - } - } - - if (fsckcfg.print_comp_ratio) { - fsckcfg.logical_blocks += - DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); - fsckcfg.physical_blocks += - DIV_ROUND_UP(pchunk_len, EROFS_BLKSIZ); - } -out: - if (raw) - free(raw); - if (buffer) - free(buffer); - return ret < 0 ? ret : 0; -} - static int erofs_verify_xattr(struct erofs_inode *inode) { unsigned int xattr_hdr_size = sizeof(struct erofs_xattr_ibody_header); @@ -336,9 +309,18 @@ out: return ret; } -static int erofs_verify_inode_data(struct erofs_inode *inode) +static int erofs_verify_inode_data(struct erofs_inode *inode, int outfd) { - int ret; + struct erofs_map_blocks map = { + .index = UINT_MAX, + }; + struct erofs_map_dev mdev; + int ret = 0; + bool compressed; + erofs_off_t pos = 0; + u64 pchunk_len = 0; + unsigned int raw_size = 0, buffer_size = 0; + char *raw = NULL, *buffer = NULL; erofs_dbg("verify data chunk of nid(%llu): type(%d)", inode->nid | 0ULL, inode->datalayout); @@ -347,30 +329,284 @@ static int erofs_verify_inode_data(struct erofs_inode *inode) case EROFS_INODE_FLAT_PLAIN: case EROFS_INODE_FLAT_INLINE: case EROFS_INODE_CHUNK_BASED: - ret = verify_uncompressed_inode(inode); + compressed = false; break; case EROFS_INODE_FLAT_COMPRESSION_LEGACY: case EROFS_INODE_FLAT_COMPRESSION: - ret = verify_compressed_inode(inode); + compressed = true; break; default: - ret = -EINVAL; - break; + erofs_err("unknown datalayout"); + return -EINVAL; } - if (ret == -EIO) - erofs_err("I/O error occurred when verifying data chunk of nid(%llu)", - inode->nid | 0ULL); + while (pos < inode->i_size) { + map.m_la = pos; + if (compressed) + ret = z_erofs_map_blocks_iter(inode, &map, + EROFS_GET_BLOCKS_FIEMAP); + else + ret = erofs_map_blocks(inode, &map, + EROFS_GET_BLOCKS_FIEMAP); + if (ret) + goto out; + + if (!compressed && map.m_llen != map.m_plen) { + erofs_err("broken chunk length m_la %" PRIu64 " m_llen %" PRIu64 " m_plen %" PRIu64, + map.m_la, map.m_llen, map.m_plen); + ret = -EFSCORRUPTED; + goto out; + } + + /* the last lcluster can be divided into 3 parts */ + if (map.m_la + map.m_llen > inode->i_size) + map.m_llen = inode->i_size - map.m_la; + + pchunk_len += map.m_plen; + pos += map.m_llen; + + /* should skip decomp? */ + if (!(map.m_flags & EROFS_MAP_MAPPED) || !fsckcfg.check_decomp) + continue; + + if (map.m_plen > raw_size) { + raw_size = map.m_plen; + raw = realloc(raw, raw_size); + BUG_ON(!raw); + } + + mdev = (struct erofs_map_dev) { + .m_deviceid = map.m_deviceid, + .m_pa = map.m_pa, + }; + ret = erofs_map_dev(&sbi, &mdev); + if (ret) { + erofs_err("failed to map device of m_pa %" PRIu64 ", m_deviceid %u @ nid %llu: %d", + map.m_pa, map.m_deviceid, inode->nid | 0ULL, + ret); + goto out; + } + + if (compressed && map.m_llen > buffer_size) { + buffer_size = map.m_llen; + buffer = realloc(buffer, buffer_size); + BUG_ON(!buffer); + } + + ret = dev_read(mdev.m_deviceid, raw, mdev.m_pa, map.m_plen); + if (ret < 0) { + erofs_err("failed to read data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %d", + mdev.m_pa, map.m_plen, inode->nid | 0ULL, + ret); + goto out; + } + + if (compressed) { + struct z_erofs_decompress_req rq = { + .in = raw, + .out = buffer, + .decodedskip = 0, + .inputsize = map.m_plen, + .decodedlength = map.m_llen, + .alg = map.m_algorithmformat, + .partial_decoding = 0 + }; + + ret = z_erofs_decompress(&rq); + if (ret < 0) { + erofs_err("failed to decompress data of m_pa %" PRIu64 ", m_plen %" PRIu64 " @ nid %llu: %s", + mdev.m_pa, map.m_plen, + inode->nid | 0ULL, strerror(-ret)); + goto out; + } + } + + if (outfd >= 0 && write(outfd, compressed ? buffer : raw, + map.m_llen) < 0) { + erofs_err("I/O error occurred when verifying data chunk @ nid %llu", + inode->nid | 0ULL); + ret = -EIO; + goto out; + } + } + + if (fsckcfg.print_comp_ratio) { + fsckcfg.logical_blocks += + DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); + fsckcfg.physical_blocks += + DIV_ROUND_UP(pchunk_len, EROFS_BLKSIZ); + } +out: + if (raw) + free(raw); + if (buffer) + free(buffer); + return ret < 0 ? ret : 0; +} + +static inline int erofs_extract_dir(struct erofs_inode *inode) +{ + int ret; + erofs_dbg("create directory %s", fsckcfg.extract_path); + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, -1); + if (ret) + return ret; + + /* + * Make directory with default user rwx permissions rather than + * the permissions from the filesystem, as these may not have + * write/execute permission. These are fixed up later in + * erofsfsck_set_attributes(). + */ + if (mkdir(fsckcfg.extract_path, 0700) < 0) { + struct stat st; + + if (errno != EEXIST) { + erofs_err("failed to create directory %s: %s", + fsckcfg.extract_path, strerror(errno)); + return -errno; + } + + if (lstat(fsckcfg.extract_path, &st) || + !S_ISDIR(st.st_mode)) { + erofs_err("path is not a directory: %s", + fsckcfg.extract_path); + return -ENOTDIR; + } + + /* + * Try to change permissions of existing directory so + * that we can write to it + */ + if (chmod(fsckcfg.extract_path, 0700) < 0) + return -errno; + } + return 0; +} + +static inline int erofs_extract_file(struct erofs_inode *inode) +{ + bool tryagain = true; + int ret, fd; + + erofs_dbg("extract file to path: %s", fsckcfg.extract_path); + +again: + fd = open(fsckcfg.extract_path, + O_WRONLY | O_CREAT | (fsckcfg.overwrite ? O_TRUNC : O_EXCL), + 0700); + if (fd < 0) { + if (fsckcfg.overwrite && tryagain) { + if (errno == EISDIR) { + erofs_warn("try to forcely remove directory %s", + fsckcfg.extract_path); + if (rmdir(fsckcfg.extract_path) < 0) { + erofs_err("failed to remove: %s", + fsckcfg.extract_path); + return -EISDIR; + } + } else if (errno == EACCES && + chmod(fsckcfg.extract_path, 0700) < 0) { + return -errno; + } + tryagain = false; + goto again; + } + erofs_err("failed to open %s: %s", fsckcfg.extract_path, + strerror(errno)); + return -errno; + } + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, fd); + if (ret) + return ret; + + if (close(fd)) + return -errno; + return ret; +} + +static inline int erofs_extract_symlink(struct erofs_inode *inode) +{ + bool tryagain = true; + int ret; + char *buf = NULL; + + erofs_dbg("extract symlink to path: %s", fsckcfg.extract_path); + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, -1); + if (ret) + return ret; + + buf = malloc(inode->i_size + 1); + if (!buf) { + ret = -ENOMEM; + goto out; + } + + ret = erofs_pread(inode, buf, inode->i_size, 0); + if (ret) { + erofs_err("I/O error occurred when reading symlink @ nid %llu: %d", + inode->nid | 0ULL, ret); + goto out; + } + + buf[inode->i_size] = '\0'; +again: + if (symlink(buf, fsckcfg.extract_path) < 0) { + if (errno == EEXIST && fsckcfg.overwrite && tryagain) { + erofs_warn("try to forcely remove file %s", + fsckcfg.extract_path); + if (unlink(fsckcfg.extract_path) < 0) { + erofs_err("failed to remove: %s", + fsckcfg.extract_path); + ret = -errno; + goto out; + } + tryagain = false; + goto again; + } + erofs_err("failed to create symlink: %s", + fsckcfg.extract_path); + ret = -errno; + } +out: + if (buf) + free(buf); return ret; } static int erofsfsck_dirent_iter(struct erofs_dir_context *ctx) { + int ret; + size_t prev_pos = fsckcfg.extract_pos; + if (ctx->dot_dotdot) return 0; - return erofsfsck_check_inode(ctx->dir->nid, ctx->de_nid); + if (fsckcfg.extract_path) { + size_t curr_pos = prev_pos; + + fsckcfg.extract_path[curr_pos++] = '/'; + strncpy(fsckcfg.extract_path + curr_pos, ctx->dname, + ctx->de_namelen); + curr_pos += ctx->de_namelen; + fsckcfg.extract_path[curr_pos] = '\0'; + fsckcfg.extract_pos = curr_pos; + } + + ret = erofsfsck_check_inode(ctx->dir->nid, ctx->de_nid); + + if (fsckcfg.extract_path) { + fsckcfg.extract_path[prev_pos] = '\0'; + fsckcfg.extract_pos = prev_pos; + } + return ret; } static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) @@ -394,8 +630,26 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) if (ret) goto out; - /* verify data chunk layout */ - ret = erofs_verify_inode_data(&inode); + if (fsckcfg.extract_path) { + switch (inode.i_mode & S_IFMT) { + case S_IFDIR: + ret = erofs_extract_dir(&inode); + break; + case S_IFREG: + ret = erofs_extract_file(&inode); + break; + case S_IFLNK: + ret = erofs_extract_symlink(&inode); + break; + default: + /* TODO */ + goto verify; + } + } else { +verify: + /* verify data chunk layout */ + ret = erofs_verify_inode_data(&inode, -1); + } if (ret) goto out; @@ -410,6 +664,10 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) ret = erofs_iterate_dir(&ctx, true); } + + if (!ret) + erofsfsck_set_attributes(&inode, fsckcfg.extract_path); + out: if (ret && ret != -EIO) fsckcfg.corrupted = true; @@ -425,6 +683,8 @@ int main(int argc, char **argv) fsckcfg.corrupted = false; fsckcfg.print_comp_ratio = false; fsckcfg.check_decomp = false; + fsckcfg.extract_path = NULL; + fsckcfg.extract_pos = 0; fsckcfg.logical_blocks = 0; fsckcfg.physical_blocks = 0; @@ -435,6 +695,8 @@ int main(int argc, char **argv) goto exit; } + fsckcfg.umask = umask(0); + err = dev_open_ro(cfg.c_img_path); if (err) { erofs_err("failed to open image file"); @@ -454,10 +716,17 @@ int main(int argc, char **argv) err = erofsfsck_check_inode(sbi.root_nid, sbi.root_nid); if (fsckcfg.corrupted) { - erofs_err("Found some filesystem corruption"); + if (!fsckcfg.extract_path) + erofs_err("Found some filesystem corruption"); + else + erofs_err("Failed to extract filesystem"); err = -EFSCORRUPTED; } else if (!err) { - erofs_info("No error found"); + if (!fsckcfg.extract_path) + erofs_info("No error found"); + else + erofs_info("Extract data successfully"); + if (fsckcfg.print_comp_ratio) { double comp_ratio = (double)fsckcfg.physical_blocks * 100 / diff --git a/mkfs/main.c b/mkfs/main.c index 9eb696a..c755da1 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -595,7 +595,7 @@ int main(int argc, char **argv) err = lstat64(cfg.c_src_path, &st); if (err) return 1; - if ((st.st_mode & S_IFMT) != S_IFDIR) { + if (!S_ISDIR(st.st_mode)) { erofs_err("root of the filesystem is not a directory - %s", cfg.c_src_path); usage(); -- cgit v1.2.3 From 096417ed4cae6b0895ab2eb38664b29969347741 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Wed, 26 Jan 2022 11:37:25 +0800 Subject: erofs-utils: fsck: never follow exist symlinks when overwriting Otherwise, we could face potential security issues. Link: https://lore.kernel.org/r/20220126033726.15865-1-hsiangkao@linux.alibaba.com Fixes: 412c8f908132 ("erofs-utils: fsck: add --extract=X support to extract to path X") Signed-off-by: Gao Xiang --- fsck/main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fsck/main.c b/fsck/main.c index 14534b9..f2af609 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -496,8 +496,8 @@ static inline int erofs_extract_file(struct erofs_inode *inode) again: fd = open(fsckcfg.extract_path, - O_WRONLY | O_CREAT | (fsckcfg.overwrite ? O_TRUNC : O_EXCL), - 0700); + O_WRONLY | O_CREAT | O_NOFOLLOW | + (fsckcfg.overwrite ? O_TRUNC : O_EXCL), 0700); if (fd < 0) { if (fsckcfg.overwrite && tryagain) { if (errno == EISDIR) { -- cgit v1.2.3 From 14f10c2681dd9b2d6080ce9a088aed8725953984 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Wed, 26 Jan 2022 11:37:26 +0800 Subject: erofs-utils: fsck: don't set attributes when fscking only Otherwise, when --extract is specified, it will report: erofs: erofsfsck_set_attributes() Line[189] failed to set times: (null) erofs: erofsfsck_set_attributes() Line[197] failed to set permissions: (null) ... Link: https://lore.kernel.org/r/20220126033726.15865-2-hsiangkao@linux.alibaba.com Fixes: 412c8f908132 ("erofs-utils: fsck: add --extract=X support to extract to path X") Signed-off-by: Gao Xiang --- fsck/main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsck/main.c b/fsck/main.c index f2af609..92e0c76 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -665,7 +665,7 @@ verify: ret = erofs_iterate_dir(&ctx, true); } - if (!ret) + if (!ret && fsckcfg.extract_path) erofsfsck_set_attributes(&inode, fsckcfg.extract_path); out: -- cgit v1.2.3 From 69476e3ffa4985e767d33cb24d020f3669bc2c77 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 29 Jan 2022 15:45:40 +0800 Subject: erofs-utils: fsck: check extract_path in erofsfsck_set_attributes() Just cleanup. Link: https://lore.kernel.org/r/20220129074540.5501-1-xiang@kernel.org Signed-off-by: Gao Xiang --- fsck/main.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/fsck/main.c b/fsck/main.c index 92e0c76..ff7d9fe 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -173,15 +173,17 @@ static void erofsfsck_set_attributes(struct erofs_inode *inode, char *path) { int ret; -#ifdef HAVE_UTIMENSAT - const struct timespec times[2] = { - [0] = { .tv_sec = inode->i_ctime, - .tv_nsec = inode->i_ctime_nsec }, - [1] = { .tv_sec = inode->i_ctime, - .tv_nsec = inode->i_ctime_nsec }, - }; + /* don't apply attributes when fsck is used without extraction */ + if (!fsckcfg.extract_path) + return; - if (utimensat(AT_FDCWD, path, times, AT_SYMLINK_NOFOLLOW) < 0) +#ifdef HAVE_UTIMENSAT + if (utimensat(AT_FDCWD, path, (struct timespec []) { + [0] = { .tv_sec = inode->i_ctime, + .tv_nsec = inode->i_ctime_nsec }, + [1] = { .tv_sec = inode->i_ctime, + .tv_nsec = inode->i_ctime_nsec }, + }, AT_SYMLINK_NOFOLLOW) < 0) #else if (utime(path, &((struct utimbuf){.actime = inode->i_ctime, .modtime = inode->i_ctime})) < 0) @@ -665,7 +667,7 @@ verify: ret = erofs_iterate_dir(&ctx, true); } - if (!ret && fsckcfg.extract_path) + if (!ret) erofsfsck_set_attributes(&inode, fsckcfg.extract_path); out: -- cgit v1.2.3 From d47ec1f8c101570f8eba7d6109903daf1acbc9ae Mon Sep 17 00:00:00 2001 From: Igor Ostapenko Date: Fri, 28 Jan 2022 06:05:11 +0200 Subject: erofs-utils: fsck: fix issues related to --extract=X * Added tar-like default behaviors for --[no-]preserve options: normal user - uses user's owner ID + umask on perms by default; root user - preserve original owner IDs + perms by default; and add appropriate error message when used without --extract=X. * "--[no-]same-owner" and "--[no-]same-permissions" were renamed to "--[no-]preserve-owner" and "--[no-]preserve-perms" to better represent what these options do, the word "same" is ambiguous and tells nothing to the user ("same" to what?). * Added "--[no-]preserve" as shortcuts for both options in one. * Fixed option descriptions as they had typos and were too ambiguous ("extract information" to where? separate file?). * Added --force option to allow extracting directly to root path. Fixes: 412c8f908132 ("erofs-utils: fsck: add --extract=X support to extract to path X") Link: https://lore.kernel.org/r/20220128040511.27-1-igoreisberg@gmail.com Link: https://lore.kernel.org/r/20220128044613.26-1-igoreisberg@gmail.com Link: https://lore.kernel.org/r/20220128045018.26-1-igoreisberg@gmail.com Link: https://lore.kernel.org/r/20220128160036.101-1-igoreisberg@gmail.com Signed-off-by: Igor Ostapenko Signed-off-by: Gao Xiang --- fsck/main.c | 110 +++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 80 insertions(+), 30 deletions(-) diff --git a/fsck/main.c b/fsck/main.c index ff7d9fe..5b75ee3 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -18,15 +18,19 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid); struct erofsfsck_cfg { - bool corrupted; - bool print_comp_ratio; - bool check_decomp; + u64 physical_blocks; + u64 logical_blocks; char *extract_path; size_t extract_pos; - bool overwrite, preserve_owner, preserve_perms; mode_t umask; - u64 physical_blocks; - u64 logical_blocks; + bool superuser; + bool corrupted; + bool print_comp_ratio; + bool check_decomp; + bool force; + bool overwrite; + bool preserve_owner; + bool preserve_perms; }; static struct erofsfsck_cfg fsckcfg; @@ -34,11 +38,14 @@ static struct option long_options[] = { {"help", no_argument, 0, 1}, {"extract", optional_argument, 0, 2}, {"device", required_argument, 0, 3}, - {"no-same-owner", no_argument, 0, 4}, - {"no-same-permissions", no_argument, 0, 5}, - {"same-owner", no_argument, 0, 6}, - {"same-permissions", no_argument, 0, 7}, - {"overwrite", no_argument, 0, 8}, + {"force", no_argument, 0, 4}, + {"overwrite", no_argument, 0, 5}, + {"preserve", no_argument, 0, 6}, + {"preserve-owner", no_argument, 0, 7}, + {"preserve-perms", no_argument, 0, 8}, + {"no-preserve", no_argument, 0, 9}, + {"no-preserve-owner", no_argument, 0, 10}, + {"no-preserve-perms", no_argument, 0, 11}, {0, 0, 0, 0}, }; @@ -66,11 +73,16 @@ static void usage(void) " --extract[=X] check if all files are well encoded, optionally extract to X\n" " --help display this help and exit\n" "\nExtraction options (--extract=X is required):\n" - " --no-same-owner extract files as yourself\n" - " --no-same-permissions apply the user's umask when extracting permission\n" - " --same-permissions extract information about file permissions\n" - " --same-owner extract files about the file ownerships\n" - " --overwrite if file already exists then overwrite\n" + " --force allow extracting to root\n" + " --overwrite overwrite files that already exist\n" + " --preserve extract with the same ownership and permissions as on the filesystem\n" + " (default for superuser)\n" + " --preserve-owner extract with the same ownership as on the filesystem\n" + " --preserve-perms extract with the same permissions as on the filesystem\n" + " --no-preserve extract as yourself and apply user's umask on permissions\n" + " (default for ordinary users)\n" + " --no-preserve-owner extract as yourself\n" + " --no-preserve-perms apply user's umask when extracting permissions\n" "\nSupported algorithms are: ", stderr); print_available_decompressors(stderr, ", "); } @@ -83,6 +95,7 @@ static void erofsfsck_print_version(void) static int erofsfsck_parse_options_cfg(int argc, char **argv) { int opt, ret; + bool has_opt_preserve = false; while ((opt = getopt_long(argc, argv, "Vd:p", long_options, NULL)) != -1) { @@ -121,6 +134,9 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) return -ENOMEM; strncpy(fsckcfg.extract_path, optarg, len); fsckcfg.extract_path[len] = '\0'; + /* if path is root, start writing from position 0 */ + if (len == 1 && fsckcfg.extract_path[0] == '/') + len = 0; fsckcfg.extract_pos = len; } break; @@ -131,33 +147,63 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) ++sbi.extra_devices; break; case 4: - fsckcfg.preserve_owner = false; + fsckcfg.force = true; break; case 5: - fsckcfg.preserve_perms = false; + fsckcfg.overwrite = true; break; case 6: - fsckcfg.preserve_owner = true; + fsckcfg.preserve_owner = fsckcfg.preserve_perms = true; + has_opt_preserve = true; break; case 7: - fsckcfg.preserve_perms = true; + fsckcfg.preserve_owner = true; + has_opt_preserve = true; break; case 8: - fsckcfg.overwrite = true; + fsckcfg.preserve_perms = true; + has_opt_preserve = true; + break; + case 9: + fsckcfg.preserve_owner = fsckcfg.preserve_perms = false; + has_opt_preserve = true; + break; + case 10: + fsckcfg.preserve_owner = false; + has_opt_preserve = true; + break; + case 11: + fsckcfg.preserve_perms = false; + has_opt_preserve = true; break; default: return -EINVAL; } } + if (fsckcfg.extract_path) { + if (!fsckcfg.extract_pos && !fsckcfg.force) { + erofs_err("--extract=/ must be used together with --force"); + return -EINVAL; + } + } else { + if (fsckcfg.force) { + erofs_err("--force must be used together with --extract=X"); + return -EINVAL; + } + if (fsckcfg.overwrite) { + erofs_err("--overwrite must be used together with --extract=X"); + return -EINVAL; + } + if (has_opt_preserve) { + erofs_err("--[no-]preserve[-owner/-perms] must be used together with --extract=X"); + return -EINVAL; + } + } + if (optind >= argc) return -EINVAL; - if (!fsckcfg.extract_path) - if (fsckcfg.overwrite || fsckcfg.preserve_owner || - fsckcfg.preserve_perms) - return -EINVAL; - cfg.c_img_path = strdup(argv[optind++]); if (!cfg.c_img_path) return -ENOMEM; @@ -682,13 +728,19 @@ int main(int argc, char **argv) erofs_init_configure(); + fsckcfg.superuser = geteuid() == 0; + fsckcfg.umask = umask(0); fsckcfg.corrupted = false; + fsckcfg.logical_blocks = 0; + fsckcfg.physical_blocks = 0; fsckcfg.print_comp_ratio = false; fsckcfg.check_decomp = false; fsckcfg.extract_path = NULL; fsckcfg.extract_pos = 0; - fsckcfg.logical_blocks = 0; - fsckcfg.physical_blocks = 0; + fsckcfg.force = false; + fsckcfg.overwrite = false; + fsckcfg.preserve_owner = fsckcfg.superuser; + fsckcfg.preserve_perms = fsckcfg.superuser; err = erofsfsck_parse_options_cfg(argc, argv); if (err) { @@ -697,8 +749,6 @@ int main(int argc, char **argv) goto exit; } - fsckcfg.umask = umask(0); - err = dev_open_ro(cfg.c_img_path); if (err) { erofs_err("failed to open image file"); -- cgit v1.2.3 From ac30e2940fad88eff20d8a3df614968c98222709 Mon Sep 17 00:00:00 2001 From: Igor Ostapenko Date: Sat, 29 Jan 2022 20:22:04 +0200 Subject: erofs-utils: add missing errors and normalize errors to lower-case * Added useful error messages. * Most errors start with lower-case, let's make all non-summarizing error messages lower-case for better consistency. * Sorted default values in fsck's main function to match the struct. Link: https://lore.kernel.org/r/20220129182204.26-1-igoreisberg@gmail.com Link: https://lore.kernel.org/r/20220129194532.26-1-igoreisberg@gmail.com Signed-off-by: Igor Ostapenko Signed-off-by: Gao Xiang --- dump/main.c | 4 +++- fsck/main.c | 37 +++++++++++++++++++++++-------------- mkfs/main.c | 30 ++++++++++++++++-------------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/dump/main.c b/dump/main.c index 0616113..664780b 100644 --- a/dump/main.c +++ b/dump/main.c @@ -162,8 +162,10 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) } } - if (optind >= argc) + if (optind >= argc) { + erofs_err("missing argument: IMAGE"); return -EINVAL; + } cfg.c_img_path = strdup(argv[optind++]); if (!cfg.c_img_path) diff --git a/fsck/main.c b/fsck/main.c index 5b75ee3..e669b44 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -122,8 +122,10 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) if (optarg) { size_t len = strlen(optarg); - if (len == 0) + if (len == 0) { + erofs_err("empty value given for --extract=X"); return -EINVAL; + } /* remove trailing slashes except root */ while (len > 1 && optarg[len - 1] == '/') @@ -201,8 +203,10 @@ static int erofsfsck_parse_options_cfg(int argc, char **argv) } } - if (optind >= argc) + if (optind >= argc) { + erofs_err("missing argument: IMAGE"); return -EINVAL; + } cfg.c_img_path = strdup(argv[optind++]); if (!cfg.c_img_path) @@ -513,7 +517,7 @@ static inline int erofs_extract_dir(struct erofs_inode *inode) struct stat st; if (errno != EEXIST) { - erofs_err("failed to create directory %s: %s", + erofs_err("failed to create directory: %s (%s)", fsckcfg.extract_path, strerror(errno)); return -errno; } @@ -529,8 +533,11 @@ static inline int erofs_extract_dir(struct erofs_inode *inode) * Try to change permissions of existing directory so * that we can write to it */ - if (chmod(fsckcfg.extract_path, 0700) < 0) + if (chmod(fsckcfg.extract_path, 0700) < 0) { + erofs_err("failed to set permissions: %s (%s)", + fsckcfg.extract_path, strerror(errno)); return -errno; + } } return 0; } @@ -552,18 +559,20 @@ again: erofs_warn("try to forcely remove directory %s", fsckcfg.extract_path); if (rmdir(fsckcfg.extract_path) < 0) { - erofs_err("failed to remove: %s", - fsckcfg.extract_path); + erofs_err("failed to remove: %s (%s)", + fsckcfg.extract_path, strerror(errno)); return -EISDIR; } } else if (errno == EACCES && chmod(fsckcfg.extract_path, 0700) < 0) { + erofs_err("failed to set permissions: %s (%s)", + fsckcfg.extract_path, strerror(errno)); return -errno; } tryagain = false; goto again; } - erofs_err("failed to open %s: %s", fsckcfg.extract_path, + erofs_err("failed to open: %s (%s)", fsckcfg.extract_path, strerror(errno)); return -errno; } @@ -728,15 +737,15 @@ int main(int argc, char **argv) erofs_init_configure(); - fsckcfg.superuser = geteuid() == 0; + fsckcfg.physical_blocks = 0; + fsckcfg.logical_blocks = 0; + fsckcfg.extract_path = NULL; + fsckcfg.extract_pos = 0; fsckcfg.umask = umask(0); + fsckcfg.superuser = geteuid() == 0; fsckcfg.corrupted = false; - fsckcfg.logical_blocks = 0; - fsckcfg.physical_blocks = 0; fsckcfg.print_comp_ratio = false; fsckcfg.check_decomp = false; - fsckcfg.extract_path = NULL; - fsckcfg.extract_pos = 0; fsckcfg.force = false; fsckcfg.overwrite = false; fsckcfg.preserve_owner = fsckcfg.superuser; @@ -775,9 +784,9 @@ int main(int argc, char **argv) err = -EFSCORRUPTED; } else if (!err) { if (!fsckcfg.extract_path) - erofs_info("No error found"); + erofs_info("No errors found"); else - erofs_info("Extract data successfully"); + erofs_info("Extracted filesystem successfully"); if (fsckcfg.print_comp_ratio) { double comp_ratio = diff --git a/mkfs/main.c b/mkfs/main.c index c755da1..5f241a1 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -381,9 +381,6 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) } } - if (optind >= argc) - return -EINVAL; - if (cfg.c_blobdev_path && cfg.c_chunkbits < LOG_BLOCK_SIZE) { erofs_err("--blobdev must be used together with --chunksize"); return -EINVAL; @@ -396,24 +393,29 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) return -EINVAL; } + if (optind >= argc) { + erofs_err("missing argument: FILE"); + return -EINVAL; + } + cfg.c_img_path = strdup(argv[optind++]); if (!cfg.c_img_path) return -ENOMEM; if (optind >= argc) { - erofs_err("Source directory is missing"); + erofs_err("missing argument: DIRECTORY"); return -EINVAL; } cfg.c_src_path = realpath(argv[optind++], NULL); if (!cfg.c_src_path) { - erofs_err("Failed to parse source directory: %s", + erofs_err("failed to parse source directory: %s", erofs_strerror(-errno)); return -ENOENT; } if (optind < argc) { - erofs_err("Unexpected argument: %s\n", argv[optind]); + erofs_err("unexpected argument: %s\n", argv[optind]); return -EINVAL; } if (quiet) @@ -456,7 +458,7 @@ int erofs_mkfs_update_super_block(struct erofs_buffer_head *bh, buf = calloc(sb_blksize, 1); if (!buf) { - erofs_err("Failed to allocate memory for sb: %s", + erofs_err("failed to allocate memory for sb: %s", erofs_strerror(-errno)); return -ENOMEM; } @@ -538,7 +540,7 @@ int parse_source_date_epoch(void) epoch = strtoull(source_date_epoch, &endptr, 10); if (epoch == -1ULL || *endptr != '\0') { - erofs_err("Environment variable $SOURCE_DATE_EPOCH %s is invalid", + erofs_err("environment variable $SOURCE_DATE_EPOCH %s is invalid", source_date_epoch); return -EINVAL; } @@ -641,34 +643,34 @@ int main(int argc, char **argv) sb_bh = erofs_buffer_init(); if (IS_ERR(sb_bh)) { err = PTR_ERR(sb_bh); - erofs_err("Failed to initialize buffers: %s", + erofs_err("failed to initialize buffers: %s", erofs_strerror(err)); goto exit; } err = erofs_bh_balloon(sb_bh, EROFS_SUPER_END); if (err < 0) { - erofs_err("Failed to balloon erofs_super_block: %s", + erofs_err("failed to balloon erofs_super_block: %s", erofs_strerror(err)); goto exit; } err = erofs_load_compress_hints(); if (err) { - erofs_err("Failed to load compress hints %s", + erofs_err("failed to load compress hints %s", cfg.c_compress_hints_file); goto exit; } err = z_erofs_compress_init(sb_bh); if (err) { - erofs_err("Failed to initialize compressor: %s", + erofs_err("failed to initialize compressor: %s", erofs_strerror(err)); goto exit; } err = erofs_generate_devtable(); if (err) { - erofs_err("Failed to generate device table: %s", + erofs_err("failed to generate device table: %s", erofs_strerror(err)); goto exit; } @@ -681,7 +683,7 @@ int main(int argc, char **argv) err = erofs_build_shared_xattrs_from_path(cfg.c_src_path); if (err) { - erofs_err("Failed to build shared xattrs: %s", + erofs_err("failed to build shared xattrs: %s", erofs_strerror(err)); goto exit; } -- cgit v1.2.3 From fb769f30bc4655944b085af364902100cc9b7878 Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Mon, 31 Jan 2022 10:43:27 -0800 Subject: erofs-utils: lib: don't hard code constants Use sizeof(z_erofs_vle_decompressed_index) to compute legacy index count Link: https://lore.kernel.org/r/20220131184327.30176-1-zhangkelvin@google.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- lib/compress.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/compress.c b/lib/compress.c index e81c070..ee09950 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -418,7 +418,8 @@ int z_erofs_convert_to_compacted_format(struct erofs_inode *inode, inode->xattr_isize) + sizeof(struct z_erofs_map_header); const unsigned int totalidx = (legacymetasize - - Z_EROFS_LEGACY_MAP_HEADER_SIZE) / 8; + Z_EROFS_LEGACY_MAP_HEADER_SIZE) / + sizeof(struct z_erofs_vle_decompressed_index); const unsigned int logical_clusterbits = inode->z_logical_clusterbits; u8 *out, *in; struct z_erofs_compressindex_vec cv[16]; -- cgit v1.2.3 From 789ac9b03c2c0d27c5be81cb8d026e2300ae822e Mon Sep 17 00:00:00 2001 From: Kelvin Zhang Date: Tue, 8 Feb 2022 10:43:17 -0800 Subject: erofs-utils: lib: Fix 8MB bug on uncompressed extent size Previously, uncompressed extent can be at most 8MB before mkfs.erofs crashes on some error condition. This is due to a minor bug in how compressed indices are encoded. This patch fixes the issue. Link: https://lore.kernel.org/r/20220208184317.3639405-1-zhangkelvin@google.com Signed-off-by: Kelvin Zhang Signed-off-by: Gao Xiang --- lib/compress.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/compress.c b/lib/compress.c index ee09950..0297c9c 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -100,7 +100,22 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, } else if (d0) { type = Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD; - di.di_u.delta[0] = cpu_to_le16(d0); + /* + * If the |Z_EROFS_VLE_DI_D0_CBLKCNT| bit is set, parser + * will interpret |delta[0]| as size of pcluster, rather + * than distance to last head cluster. Normally this + * isn't a problem, because uncompressed extent size are + * below Z_EROFS_VLE_DI_D0_CBLKCNT * BLOCK_SIZE = 8MB. + * But with large pcluster it's possible to go over this + * number, resulting in corrupted compressed indices. + * To solve this, we replace d0 with + * Z_EROFS_VLE_DI_D0_CBLKCNT-1. + */ + if (d0 >= Z_EROFS_VLE_DI_D0_CBLKCNT) + di.di_u.delta[0] = cpu_to_le16( + Z_EROFS_VLE_DI_D0_CBLKCNT - 1); + else + di.di_u.delta[0] = cpu_to_le16(d0); di.di_u.delta[1] = cpu_to_le16(d1); } else { type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN : -- cgit v1.2.3 From 36a4f39374c692d099d70c2b5ca9d1abe8643246 Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Tue, 8 Feb 2022 16:53:07 -0800 Subject: erofs-utils: Print program name and version to stdout The program name and version is not an error message, so it should go to stdout, not stderr. Link: https://lore.kernel.org/r/20220209005307.1288161-2-pcc@google.com Signed-off-by: Peter Collingbourne Signed-off-by: Gao Xiang --- dump/main.c | 2 +- fsck/main.c | 2 +- fuse/main.c | 2 +- mkfs/main.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dump/main.c b/dump/main.c index 664780b..e6198a0 100644 --- a/dump/main.c +++ b/dump/main.c @@ -113,7 +113,7 @@ static void usage(void) static void erofsdump_print_version(void) { - fprintf(stderr, "dump.erofs %s\n", cfg.c_version); + printf("dump.erofs %s\n", cfg.c_version); } static int erofsdump_parse_options_cfg(int argc, char **argv) diff --git a/fsck/main.c b/fsck/main.c index e669b44..56595e3 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -89,7 +89,7 @@ static void usage(void) static void erofsfsck_print_version(void) { - fprintf(stderr, "fsck.erofs %s\n", cfg.c_version); + printf("fsck.erofs %s\n", cfg.c_version); } static int erofsfsck_parse_options_cfg(int argc, char **argv) diff --git a/fuse/main.c b/fuse/main.c index 2549d8a..b869a00 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -258,7 +258,7 @@ int main(int argc, char *argv[]) struct fuse_args args = FUSE_ARGS_INIT(argc, argv); erofs_init_configure(); - fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version); + printf("%s %s\n", basename(argv[0]), cfg.c_version); #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) if (signal(SIGSEGV, signal_handle_sigsegv) == SIG_ERR) { diff --git a/mkfs/main.c b/mkfs/main.c index 5f241a1..282126a 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -557,7 +557,7 @@ int parse_source_date_epoch(void) void erofs_show_progs(int argc, char *argv[]) { if (cfg.c_dbg_lvl >= EROFS_WARN) - fprintf(stderr, "%s %s\n", basename(argv[0]), cfg.c_version); + printf("%s %s\n", basename(argv[0]), cfg.c_version); } int main(int argc, char **argv) -- cgit v1.2.3 From e8dcfcb97c54ffa6e784990ff5b018fa58211f5f Mon Sep 17 00:00:00 2001 From: Peter Collingbourne Date: Tue, 8 Feb 2022 16:53:06 -0800 Subject: erofs-utils: Print configuration only at INFO debug level The information printed by erofs_show_config is not useful for ordinary users of the program, and it certainly doesn't count as a warning, so let's only print it at the INFO debug level or greater. Link: https://lore.kernel.org/r/20220209005307.1288161-1-pcc@google.com Signed-off-by: Peter Collingbourne Signed-off-by: Gao Xiang --- lib/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/config.c b/lib/config.c index d3298da..46d41e9 100644 --- a/lib/config.c +++ b/lib/config.c @@ -35,7 +35,7 @@ void erofs_show_config(void) { const struct erofs_configure *c = &cfg; - if (c->c_dbg_lvl < EROFS_WARN) + if (c->c_dbg_lvl < EROFS_INFO) return; erofs_dump("\tc_version: [%8s]\n", c->c_version); erofs_dump("\tc_dbg_lvl: [%8d]\n", c->c_dbg_lvl); -- cgit v1.2.3 From 63a8cd6ef0f51525b4a930fe9ccfda7acc6fcac9 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Wed, 16 Feb 2022 20:28:44 +0800 Subject: erofs-utils: lib: get rid of a redundent compress round No need another round if no remaining data. It can improve compression ratios a bit for specific data. Link: https://lore.kernel.org/r/20220216122845.47819-2-hsiangkao@linux.alibaba.com Tested-by: Yue Hu Signed-off-by: Gao Xiang --- lib/compress.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/lib/compress.c b/lib/compress.c index 0297c9c..95442c9 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -599,16 +599,11 @@ int erofs_write_compressed_file(struct erofs_inode *inode) remaining -= readcount; ctx.tail += readcount; - /* do one compress round */ - ret = vle_compress_one(inode, &ctx, false); + ret = vle_compress_one(inode, &ctx, !remaining); if (ret) - goto err_bdrop; + goto err_free_idata; } - - /* do the final round */ - ret = vle_compress_one(inode, &ctx, true); - if (ret) - goto err_free_idata; + DBG_BUGON(ctx.head != ctx.tail); /* fall back to no compression mode */ compressed_blocks = ctx.blkaddr - blkaddr; -- cgit v1.2.3 From a7c1f0575ef881f15bfd54f2116d471d0ad30cef Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Wed, 16 Feb 2022 20:28:45 +0800 Subject: erofs-utils: lib: refine tailpcluster compression approach As my previous comment [1] mentioned, currently, in order to inline a tail pcluster right after its inode metadata, tail data is split, compressed with many 4KiB pclusters and then the last pcluster is chosen. However, it can have some impacts to overall compression ratios if big pclusters are enabled. So instead, let's implement another approach: compressing with two pclusters by trying recompressing. It also enables EOF lcluster inlining for small compressed data, so please make sure the kernel is already fixed up with commit 24331050a3e6 ("erofs: fix small compressed files inlining") [1] https://lore.kernel.org/r/YXkBIpcCG12Y8qMN@B-P7TQMD6M-0146.local Link: https://lore.kernel.org/r/20220216122845.47819-3-hsiangkao@linux.alibaba.com Tested-by: Yue Hu Signed-off-by: Gao Xiang --- include/erofs/compress.h | 1 + include/erofs/internal.h | 4 ++ lib/compress.c | 146 ++++++++++++++++++++++++++++++++++------------- lib/inode.c | 9 +-- 4 files changed, 117 insertions(+), 43 deletions(-) diff --git a/include/erofs/compress.h b/include/erofs/compress.h index 21eac57..40df2bc 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -19,6 +19,7 @@ extern "C" #define EROFS_CONFIG_COMPR_MAX_SZ (900 * 1024) #define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024) +void z_erofs_drop_inline_pcluster(struct erofs_inode *inode); int erofs_write_compressed_file(struct erofs_inode *inode); int z_erofs_compress_init(struct erofs_buffer_head *bh); diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 3615297..d4efc18 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -186,6 +186,10 @@ struct erofs_inode { void *idata; + /* (ztailpacking) in order to recover uncompressed EOF data */ + void *eof_tailraw; + unsigned int eof_tailrawsize; + union { void *compressmeta; void *chunkindexes; diff --git a/lib/compress.c b/lib/compress.c index 95442c9..e050df0 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -70,14 +70,15 @@ static void vle_write_indexes(struct z_erofs_vle_compress_ctx *ctx, di.di_clusterofs = cpu_to_le16(ctx->clusterofs); - /* whether the tail-end uncompressed block or not */ + /* whether the tail-end (un)compressed block or not */ if (!d1) { /* - * A lcluster cannot have there parts with the middle one which - * is well-compressed. Such tail pcluster cannot be inlined. + * A lcluster cannot have three parts with the middle one which + * is well-compressed for !ztailpacking cases. */ - DBG_BUGON(!raw); - type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN; + DBG_BUGON(!raw && !cfg.c_ztailpacking); + type = raw ? Z_EROFS_VLE_CLUSTER_TYPE_PLAIN : + Z_EROFS_VLE_CLUSTER_TYPE_HEAD; advise = cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT); di.di_advise = advise; @@ -183,6 +184,7 @@ static unsigned int z_erofs_get_max_pclusterblks(struct erofs_inode *inode) static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data, unsigned int len, bool raw) { + inode->z_advise |= Z_EROFS_ADVISE_INLINE_PCLUSTER; inode->idata_size = len; inode->compressed_idata = !raw; @@ -195,6 +197,31 @@ static int z_erofs_fill_inline_data(struct erofs_inode *inode, void *data, return len; } +static void tryrecompress_trailing(void *in, unsigned int *insize, + void *out, int *compressedsize) +{ + static char tmp[Z_EROFS_PCLUSTER_MAX_SIZE]; + unsigned int count; + int ret = *compressedsize; + + /* no need to recompress */ + if (!(ret & (EROFS_BLKSIZ - 1))) + return; + + count = *insize; + ret = erofs_compress_destsize(&compresshandle, + in, &count, (void *)tmp, + rounddown(ret, EROFS_BLKSIZ), false); + if (ret <= 0 || ret + (*insize - count) >= + roundup(*compressedsize, EROFS_BLKSIZ)) + return; + + /* replace the original compressed data if any gain */ + memcpy(out, tmp, ret); + *insize = count; + *compressedsize = ret; +} + static int vle_compress_one(struct erofs_inode *inode, struct z_erofs_vle_compress_ctx *ctx, bool final) @@ -205,54 +232,32 @@ static int vle_compress_one(struct erofs_inode *inode, int ret; static char dstbuf[EROFS_CONFIG_COMPR_MAX_SZ + EROFS_BLKSIZ]; char *const dst = dstbuf + EROFS_BLKSIZ; - bool trailing = false, tailpcluster = false; while (len) { unsigned int pclustersize = z_erofs_get_max_pclusterblks(inode) * EROFS_BLKSIZ; + bool may_inline = (cfg.c_ztailpacking && final); bool raw; - DBG_BUGON(tailpcluster); if (len <= pclustersize) { - if (final) { - /* TODO: compress with 2 pclusters instead */ - if (cfg.c_ztailpacking) { - trailing = true; - pclustersize = EROFS_BLKSIZ; - } else if (len <= EROFS_BLKSIZ) { - goto nocompression; - } - } else { + if (!final) break; - } + if (!may_inline && len <= EROFS_BLKSIZ) + goto nocompression; } count = min(len, cfg.c_max_decompressed_extent_bytes); ret = erofs_compress_destsize(h, ctx->queue + ctx->head, &count, dst, pclustersize, !(final && len == count)); - - /* XXX: need to be polished, yet do it correctly first. */ - if (cfg.c_ztailpacking && final) { - if (ret <= 0 && len < EROFS_BLKSIZ) { - DBG_BUGON(!trailing); - tailpcluster = true; - } else if (ret > 0 && len == count && - /* less than 1 compressed block */ - ret < EROFS_BLKSIZ) { - tailpcluster = true; - } - } - - if (ret <= 0 || (tailpcluster && - ctx->clusterofs + len < EROFS_BLKSIZ)) { - if (ret <= 0 && ret != -EAGAIN) { + if (ret <= 0) { + if (ret != -EAGAIN) { erofs_err("failed to compress %s: %s", inode->i_srcpath, erofs_strerror(ret)); } - if (tailpcluster && len < EROFS_BLKSIZ) + if (may_inline && len < EROFS_BLKSIZ) ret = z_erofs_fill_inline_data(inode, ctx->queue + ctx->head, len, true); @@ -271,18 +276,33 @@ nocompression: */ ctx->compressedblks = 1; raw = true; - } else if (tailpcluster && ret < EROFS_BLKSIZ) { + /* tailpcluster should be less than 1 block */ + } else if (may_inline && len == count && + ret < EROFS_BLKSIZ) { + if (ctx->clusterofs + len <= EROFS_BLKSIZ) { + inode->eof_tailraw = malloc(len); + if (!inode->eof_tailraw) + return -ENOMEM; + + memcpy(inode->eof_tailraw, + ctx->queue + ctx->head, len); + inode->eof_tailrawsize = len; + } + ret = z_erofs_fill_inline_data(inode, dst, ret, false); if (ret < 0) return ret; ctx->compressedblks = 1; raw = false; } else { - const unsigned int tailused = ret & (EROFS_BLKSIZ - 1); - const unsigned int padding = - erofs_sb_has_lz4_0padding() && tailused ? - EROFS_BLKSIZ - tailused : 0; + unsigned int tailused, padding; + if (may_inline && len == count) + tryrecompress_trailing(ctx->queue + ctx->head, + &count, dst, &ret); + + tailused = ret & (EROFS_BLKSIZ - 1); + padding = 0; ctx->compressedblks = DIV_ROUND_UP(ret, EROFS_BLKSIZ); DBG_BUGON(ctx->compressedblks * EROFS_BLKSIZ >= count); @@ -290,6 +310,8 @@ nocompression: if (!erofs_sb_has_lz4_0padding()) memset(dst + ret, 0, roundup(ret, EROFS_BLKSIZ) - ret); + else if (tailused) + padding = EROFS_BLKSIZ - tailused; /* write compressed data */ erofs_dbg("Writing %u compressed data to %u of %u blocks", @@ -536,6 +558,52 @@ static void z_erofs_write_mapheader(struct erofs_inode *inode, memcpy(compressmeta, &h, sizeof(struct z_erofs_map_header)); } +void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) +{ + const unsigned int type = Z_EROFS_VLE_CLUSTER_TYPE_PLAIN; + struct z_erofs_map_header *h = inode->compressmeta; + + h->h_advise = cpu_to_le16(le16_to_cpu(h->h_advise) & + ~Z_EROFS_ADVISE_INLINE_PCLUSTER); + if (!inode->eof_tailraw) + return; + DBG_BUGON(inode->compressed_idata != true); + + /* patch the EOF lcluster to uncompressed type first */ + if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) { + struct z_erofs_vle_decompressed_index *di = + (inode->compressmeta + inode->extent_isize) - + sizeof(struct z_erofs_vle_decompressed_index); + __le16 advise = + cpu_to_le16(type << Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT); + + di->di_advise = advise; + } else if (inode->datalayout == EROFS_INODE_FLAT_COMPRESSION) { + /* handle the last compacted 4B pack */ + unsigned int eofs, base, pos, v, lo; + u8 *out; + + eofs = inode->extent_isize - + (4 << (DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ) & 1)); + base = round_down(eofs, 8); + pos = 16 /* encodebits */ * ((eofs - base) / 4); + out = inode->compressmeta + base; + lo = get_unaligned_le32(out + pos / 8) & (EROFS_BLKSIZ - 1); + v = (type << LOG_BLOCK_SIZE) | lo; + out[pos / 8] = v & 0xff; + out[pos / 8 + 1] = v >> 8; + } else { + DBG_BUGON(1); + return; + } + free(inode->idata); + /* replace idata with prepared uncompressed data */ + inode->idata = inode->eof_tailraw; + inode->idata_size = inode->eof_tailrawsize; + inode->compressed_idata = false; + inode->eof_tailraw = NULL; +} + int erofs_write_compressed_file(struct erofs_inode *inode) { struct erofs_buffer_head *bh; diff --git a/lib/inode.c b/lib/inode.c index 3fd4b69..e680b23 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -96,6 +96,8 @@ unsigned int erofs_iput(struct erofs_inode *inode) list_for_each_entry_safe(d, t, &inode->i_subdirs, d_child) free(d); + if (inode->eof_tailraw) + free(inode->eof_tailraw); list_del(&inode->i_hash); free(inode); return 0; @@ -614,7 +616,9 @@ static int erofs_prepare_inode_buffer(struct erofs_inode *inode) if (bh == ERR_PTR(-ENOSPC)) { int ret; - if (!is_inode_layout_compression(inode)) + if (is_inode_layout_compression(inode)) + z_erofs_drop_inline_pcluster(inode); + else inode->datalayout = EROFS_INODE_FLAT_PLAIN; noinline: /* expend an extra block for tail-end data */ @@ -629,10 +633,7 @@ noinline: return PTR_ERR(bh); } else if (inode->idata_size) { if (is_inode_layout_compression(inode)) { - struct z_erofs_map_header *h = inode->compressmeta; - DBG_BUGON(!cfg.c_ztailpacking); - h->h_advise |= Z_EROFS_ADVISE_INLINE_PCLUSTER; erofs_dbg("Inline %scompressed data (%u bytes) to %s", inode->compressed_idata ? "" : "un", inode->idata_size, inode->i_srcpath); -- cgit v1.2.3 From 4bfb656697cb19c02c6d089973b81b14014120c4 Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Fri, 18 Feb 2022 11:11:35 +0800 Subject: erofs-utils: check the return value of ftell Need to check if we got a normal length. Link: https://lore.kernel.org/r/20220218031137.18716-1-huangjianan@oppo.com Signed-off-by: Huang Jianan Reviewed-by: Gao Xiang Signed-off-by: Gao Xiang --- lib/blobchunk.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/blobchunk.c b/lib/blobchunk.c index a145be9..77b0c17 100644 --- a/lib/blobchunk.c +++ b/lib/blobchunk.c @@ -221,6 +221,8 @@ int erofs_blob_remap(void) fflush(blobfile); length = ftell(blobfile); + if (length < 0) + return -errno; if (multidev) { struct erofs_deviceslot dis = { .blocks = erofs_blknr(length), -- cgit v1.2.3 From b254ce8b6d7427dc2ab4540bfb22e68b8e0143b1 Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Wed, 23 Feb 2022 15:24:18 +0800 Subject: erofs-utils: fix fd leak when load compress hints file Execute fclose before return error. Link: https://lore.kernel.org/r/20220223072418.19180-1-huangjianan@oppo.com Signed-off-by: Huang Jianan Signed-off-by: Gao Xiang --- lib/compress_hints.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/compress_hints.c b/lib/compress_hints.c index 25adf35..92964eb 100644 --- a/lib/compress_hints.c +++ b/lib/compress_hints.c @@ -88,6 +88,7 @@ int erofs_load_compress_hints(void) char buf[PATH_MAX + 100]; FILE *f; unsigned int line, max_pclustersize = 0; + int ret = 0; if (!cfg.c_compress_hints_file) return 0; @@ -105,7 +106,8 @@ int erofs_load_compress_hints(void) if (!pattern || *pattern == '\0') { erofs_err("cannot find a match pattern at line %u", line); - return -EINVAL; + ret = -EINVAL; + goto out; } if (pclustersize % EROFS_BLKSIZ) { erofs_warn("invalid physical clustersize %u, " @@ -119,10 +121,12 @@ int erofs_load_compress_hints(void) if (pclustersize > max_pclustersize) max_pclustersize = pclustersize; } - fclose(f); + if (cfg.c_pclusterblks_max * EROFS_BLKSIZ < max_pclustersize) { cfg.c_pclusterblks_max = max_pclustersize / EROFS_BLKSIZ; erofs_warn("update max pclusterblks to %u", cfg.c_pclusterblks_max); } - return 0; +out: + fclose(f); + return ret; } -- cgit v1.2.3 From 1ab11aae4b78a4b0f75fa65047d2c419d4307dbb Mon Sep 17 00:00:00 2001 From: Huang Jianan Date: Thu, 3 Mar 2022 11:10:55 +0800 Subject: erofs-utils: fix some style problems Fix some minor issues, including: - Spelling mistakes; - Remove redundant spaces and parenthesis; - clean up file headers; - Match parameters with format parameters. Link: https://lore.kernel.org/r/20220303031055.2433-1-huangjianan@oppo.com Signed-off-by: Huang Jianan Signed-off-by: Gao Xiang --- fuse/main.c | 1 - include/erofs/defs.h | 1 - include/erofs/dir.h | 2 +- include/erofs/list.h | 1 - lib/cache.c | 1 - lib/compressor_liblzma.c | 2 -- lib/inode.c | 2 +- lib/io.c | 2 +- lib/namei.c | 1 - lib/super.c | 4 ++-- mkfs/main.c | 2 +- 11 files changed, 6 insertions(+), 13 deletions(-) diff --git a/fuse/main.c b/fuse/main.c index b869a00..90ed611 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -61,7 +61,6 @@ int erofsfuse_readdir(const char *path, void *buf, fuse_fill_dir_t filler, #else return erofs_iterate_dir(&ctx.ctx, true); #endif - } static void *erofsfuse_init(struct fuse_conn_info *info) diff --git a/include/erofs/defs.h b/include/erofs/defs.h index e745bc0..e5aa23c 100644 --- a/include/erofs/defs.h +++ b/include/erofs/defs.h @@ -61,7 +61,6 @@ typedef int16_t s16; typedef int32_t s32; typedef int64_t s64; - #if __BYTE_ORDER == __LITTLE_ENDIAN /* * The host byte order is the same as network byte order, diff --git a/include/erofs/dir.h b/include/erofs/dir.h index 1627d3d..74bffb5 100644 --- a/include/erofs/dir.h +++ b/include/erofs/dir.h @@ -42,7 +42,7 @@ struct erofs_dir_context { /* * During execution of |erofs_iterate_dir|, the function needs to * read the values inside |erofs_inode* dir|. So it is important - * that the callback function does not modify stuct pointed by + * that the callback function does not modify struct pointed by * |dir|. It is OK to repoint |dir| to other objects. * Unfortunately, it's not possible to enforce this restriction * with const keyword, as |erofs_iterate_dir| needs to modify diff --git a/include/erofs/list.h b/include/erofs/list.h index 2a0e961..3f5da1a 100644 --- a/include/erofs/list.h +++ b/include/erofs/list.h @@ -110,7 +110,6 @@ static inline int list_empty(struct list_head *head) &pos->member != (head); \ pos = n, n = list_next_entry(n, member)) - #ifdef __cplusplus } #endif diff --git a/lib/cache.c b/lib/cache.c index f820c0b..c735363 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -331,7 +331,6 @@ struct erofs_buffer_head *erofs_battach(struct erofs_buffer_head *bh, return ERR_PTR(ret); } return nbh; - } static erofs_blk_t __erofs_mapbh(struct erofs_buffer_block *bb) diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c index acf442f..4886d6a 100644 --- a/lib/compressor_liblzma.c +++ b/lib/compressor_liblzma.c @@ -1,7 +1,5 @@ // SPDX-License-Identifier: GPL-2.0+ OR Apache-2.0 /* - * erofs-utils/lib/compressor_liblzma.c - * * Copyright (C) 2021 Gao Xiang */ #include diff --git a/lib/inode.c b/lib/inode.c index e680b23..5db4691 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -1108,7 +1108,7 @@ fail: d->type = ftype; erofs_d_invalidate(d); - erofs_info("add file %s/%s (nid %llu, type %d)", + erofs_info("add file %s/%s (nid %llu, type %u)", dir->i_srcpath, d->name, (unsigned long long)d->nid, d->type); } diff --git a/lib/io.c b/lib/io.c index 5bc3432..9c663c5 100644 --- a/lib/io.c +++ b/lib/io.c @@ -370,7 +370,7 @@ ssize_t erofs_copy_file_range(int fd_in, erofs_off_t *off_in, ssize_t ret; ret = copy_file_range(fd_in, &off64_in, fd_out, &off64_out, - length, 0); + length, 0); if (ret >= 0) goto out; if (errno != ENOSYS) { diff --git a/lib/namei.c b/lib/namei.c index 97f0f80..6163238 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -144,7 +144,6 @@ bogusimode: return -EFSCORRUPTED; } - struct erofs_dirent *find_target_dirent(erofs_nid_t pnid, void *dentry_blk, const char *name, unsigned int len, diff --git a/lib/super.c b/lib/super.c index 69522bd..f486eb7 100644 --- a/lib/super.c +++ b/lib/super.c @@ -15,7 +15,7 @@ static bool check_layout_compatibility(struct erofs_sb_info *sbi, sbi->feature_incompat = feature; /* check if current kernel meets all mandatory requirements */ - if (feature & (~EROFS_ALL_FEATURE_INCOMPAT)) { + if (feature & ~EROFS_ALL_FEATURE_INCOMPAT) { erofs_err("unidentified incompatible feature %x, please upgrade kernel version", feature & ~EROFS_ALL_FEATURE_INCOMPAT); return false; @@ -87,7 +87,7 @@ int erofs_read_superblock(void) blkszbits = dsb->blkszbits; /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ if (blkszbits != LOG_BLOCK_SIZE) { - erofs_err("blksize %u isn't supported on this platform", + erofs_err("blksize %d isn't supported on this platform", 1 << blkszbits); return ret; } diff --git a/mkfs/main.c b/mkfs/main.c index 282126a..0724212 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -204,7 +204,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) bool quiet = false; while ((opt = getopt_long(argc, argv, "C:E:T:U:d:x:z:", - long_options, NULL)) != -1) { + long_options, NULL)) != -1) { switch (opt) { case 'z': if (!optarg) { -- cgit v1.2.3 From 34029db7f99a940712f47f72d44f0af7958ac15e Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 11 Mar 2022 04:17:23 +0000 Subject: erofs-utils: mkfs: rename ctime to mtime Currently mkfs.erofs picks up whatever the system time happened to be when the input file structure was created. Since there's no (easy) way for userspace to control ctime, there's no way to control the per-file ctime that mkfs.erofs uses. In preparation for switching to mtime, rename the "ctime" members of the inode structure. Link: https://lore.kernel.org/r/20220311041724.3107622-1-dvander@google.com Signed-off-by: David Anderson Signed-off-by: Gao Xiang --- dump/main.c | 4 ++-- fsck/main.c | 12 ++++++------ fuse/main.c | 2 +- include/erofs/internal.h | 4 ++-- include/erofs_fs.h | 4 ++-- lib/inode.c | 14 +++++++------- lib/namei.c | 8 ++++---- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/dump/main.c b/dump/main.c index e6198a0..84ef4fa 100644 --- a/dump/main.c +++ b/dump/main.c @@ -354,7 +354,7 @@ static void erofsdump_show_fileinfo(bool show_extent) } strftime(timebuf, sizeof(timebuf), - "%Y-%m-%d %H:%M:%S", localtime((time_t *)&inode.i_ctime)); + "%Y-%m-%d %H:%M:%S", localtime((time_t *)&inode.i_mtime)); access_mode = inode.i_mode & 0777; for (i = 8; i >= 0; i--) if (((access_mode >> i) & 1) == 0) @@ -373,7 +373,7 @@ static void erofsdump_show_fileinfo(bool show_extent) fprintf(stdout, "Xattr size: %u\n", inode.xattr_isize); fprintf(stdout, "Uid: %u Gid: %u ", inode.i_uid, inode.i_gid); fprintf(stdout, "Access: %04o/%s\n", access_mode, access_mode_str); - fprintf(stdout, "Timestamp: %s.%09d\n", timebuf, inode.i_ctime_nsec); + fprintf(stdout, "Timestamp: %s.%09d\n", timebuf, inode.i_mtime_nsec); if (!dumpcfg.show_extent) return; diff --git a/fsck/main.c b/fsck/main.c index 56595e3..7fc7b6f 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -229,14 +229,14 @@ static void erofsfsck_set_attributes(struct erofs_inode *inode, char *path) #ifdef HAVE_UTIMENSAT if (utimensat(AT_FDCWD, path, (struct timespec []) { - [0] = { .tv_sec = inode->i_ctime, - .tv_nsec = inode->i_ctime_nsec }, - [1] = { .tv_sec = inode->i_ctime, - .tv_nsec = inode->i_ctime_nsec }, + [0] = { .tv_sec = inode->i_mtime, + .tv_nsec = inode->i_mtime_nsec }, + [1] = { .tv_sec = inode->i_mtime, + .tv_nsec = inode->i_mtime_nsec }, }, AT_SYMLINK_NOFOLLOW) < 0) #else - if (utime(path, &((struct utimbuf){.actime = inode->i_ctime, - .modtime = inode->i_ctime})) < 0) + if (utime(path, &((struct utimbuf){.actime = inode->i_mtime, + .modtime = inode->i_mtime})) < 0) #endif erofs_warn("failed to set times: %s", path); diff --git a/fuse/main.c b/fuse/main.c index 90ed611..f4c2476 100644 --- a/fuse/main.c +++ b/fuse/main.c @@ -97,7 +97,7 @@ static int erofsfuse_getattr(const char *path, struct stat *stbuf) stbuf->st_gid = vi.i_gid; if (S_ISBLK(vi.i_mode) || S_ISCHR(vi.i_mode)) stbuf->st_rdev = vi.u.i_rdev; - stbuf->st_ctime = vi.i_ctime; + stbuf->st_ctime = vi.i_mtime; stbuf->st_mtime = stbuf->st_ctime; stbuf->st_atime = stbuf->st_ctime; return 0; diff --git a/include/erofs/internal.h b/include/erofs/internal.h index d4efc18..2686570 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -155,8 +155,8 @@ struct erofs_inode { u64 i_ino[2]; u32 i_uid; u32 i_gid; - u64 i_ctime; - u32 i_ctime_nsec; + u64 i_mtime; + u32 i_mtime_nsec; u32 i_nlink; union { diff --git a/include/erofs_fs.h b/include/erofs_fs.h index 59d9bbb..d70c891 100644 --- a/include/erofs_fs.h +++ b/include/erofs_fs.h @@ -185,8 +185,8 @@ struct erofs_inode_extended { __le32 i_uid; __le32 i_gid; - __le64 i_ctime; - __le32 i_ctime_nsec; + __le64 i_mtime; + __le32 i_mtime_nsec; __le32 i_nlink; __u8 i_reserved2[16]; }; diff --git a/lib/inode.c b/lib/inode.c index 5db4691..ffc750f 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -479,8 +479,8 @@ static bool erofs_bh_flush_write_inode(struct erofs_buffer_head *bh) u.die.i_uid = cpu_to_le32(inode->i_uid); u.die.i_gid = cpu_to_le32(inode->i_gid); - u.die.i_ctime = cpu_to_le64(inode->i_ctime); - u.die.i_ctime_nsec = cpu_to_le32(inode->i_ctime_nsec); + u.die.i_mtime = cpu_to_le64(inode->i_mtime); + u.die.i_mtime_nsec = cpu_to_le32(inode->i_mtime_nsec); switch (inode->i_mode & S_IFMT) { case S_IFCHR: @@ -832,16 +832,16 @@ static int erofs_fill_inode(struct erofs_inode *inode, inode->i_mode = st->st_mode; inode->i_uid = cfg.c_uid == -1 ? st->st_uid : cfg.c_uid; inode->i_gid = cfg.c_gid == -1 ? st->st_gid : cfg.c_gid; - inode->i_ctime = st->st_ctime; - inode->i_ctime_nsec = ST_CTIM_NSEC(st); + inode->i_mtime = st->st_ctime; + inode->i_mtime_nsec = ST_CTIM_NSEC(st); switch (cfg.c_timeinherit) { case TIMESTAMP_CLAMPING: - if (st->st_ctime < sbi.build_time) + if (inode->i_mtime < sbi.build_time) break; case TIMESTAMP_FIXED: - inode->i_ctime = sbi.build_time; - inode->i_ctime_nsec = sbi.build_time_nsec; + inode->i_mtime = sbi.build_time; + inode->i_mtime_nsec = sbi.build_time_nsec; default: break; } diff --git a/lib/namei.c b/lib/namei.c index 6163238..8e9867d 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -79,8 +79,8 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->i_gid = le32_to_cpu(die->i_gid); vi->i_nlink = le32_to_cpu(die->i_nlink); - vi->i_ctime = le64_to_cpu(die->i_ctime); - vi->i_ctime_nsec = le64_to_cpu(die->i_ctime_nsec); + vi->i_mtime = le64_to_cpu(die->i_mtime); + vi->i_mtime_nsec = le64_to_cpu(die->i_mtime_nsec); vi->i_size = le64_to_cpu(die->i_size); if (vi->datalayout == EROFS_INODE_CHUNK_BASED) /* fill chunked inode summary info */ @@ -114,8 +114,8 @@ int erofs_read_inode_from_disk(struct erofs_inode *vi) vi->i_gid = le16_to_cpu(dic->i_gid); vi->i_nlink = le16_to_cpu(dic->i_nlink); - vi->i_ctime = sbi.build_time; - vi->i_ctime_nsec = sbi.build_time_nsec; + vi->i_mtime = sbi.build_time; + vi->i_mtime_nsec = sbi.build_time_nsec; vi->i_size = le32_to_cpu(dic->i_size); if (vi->datalayout == EROFS_INODE_CHUNK_BASED) -- cgit v1.2.3 From 2aa9fba38aee29cfe41eeefc5590a2dce6e798ed Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 11 Mar 2022 04:17:24 +0000 Subject: erofs-utils: mkfs: use mtime instead of ctime Change the default timestamp behavior to use modification time rather than change time. This will allow more control over the output timestamps when not using TIMESTAMP_FIXED. EROFS_FEATURE_COMPAT_MTIME has been added so tooling can detect the change in timestamp behavior. Link: https://lore.kernel.org/r/20220311041724.3107622-2-dvander@google.com Signed-off-by: David Anderson Signed-off-by: Gao Xiang --- dump/main.c | 1 + include/erofs_fs.h | 1 + lib/inode.c | 4 ++-- mkfs/main.c | 3 ++- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dump/main.c b/dump/main.c index 84ef4fa..b431e18 100644 --- a/dump/main.c +++ b/dump/main.c @@ -88,6 +88,7 @@ struct erofsdump_feature { static struct erofsdump_feature feature_lists[] = { { true, EROFS_FEATURE_COMPAT_SB_CHKSUM, "sb_csum" }, + { true, EROFS_FEATURE_COMPAT_MTIME, "mtime" }, { false, EROFS_FEATURE_INCOMPAT_LZ4_0PADDING, "0padding" }, { false, EROFS_FEATURE_INCOMPAT_BIG_PCLUSTER, "big_pcluster" }, { false, EROFS_FEATURE_INCOMPAT_CHUNKED_FILE, "chunked_file" }, diff --git a/include/erofs_fs.h b/include/erofs_fs.h index d70c891..08f9761 100644 --- a/include/erofs_fs.h +++ b/include/erofs_fs.h @@ -13,6 +13,7 @@ #define EROFS_SUPER_OFFSET 1024 #define EROFS_FEATURE_COMPAT_SB_CHKSUM 0x00000001 +#define EROFS_FEATURE_COMPAT_MTIME 0x00000002 /* * Any bits that aren't in EROFS_ALL_FEATURE_INCOMPAT should diff --git a/lib/inode.c b/lib/inode.c index ffc750f..1097116 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -832,8 +832,8 @@ static int erofs_fill_inode(struct erofs_inode *inode, inode->i_mode = st->st_mode; inode->i_uid = cfg.c_uid == -1 ? st->st_uid : cfg.c_uid; inode->i_gid = cfg.c_gid == -1 ? st->st_gid : cfg.c_gid; - inode->i_mtime = st->st_ctime; - inode->i_mtime_nsec = ST_CTIM_NSEC(st); + inode->i_mtime = st->st_mtime; + inode->i_mtime_nsec = ST_MTIM_NSEC(st); switch (cfg.c_timeinherit) { case TIMESTAMP_CLAMPING: diff --git a/mkfs/main.c b/mkfs/main.c index 0724212..c5fd1e5 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -517,7 +517,8 @@ static void erofs_mkfs_default_options(void) { cfg.c_legacy_compress = false; sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING; - sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM; + sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM | + EROFS_FEATURE_COMPAT_MTIME; /* generate a default uuid first */ #ifdef HAVE_LIBUUID -- cgit v1.2.3 From 59f826ca013fd1ac03040fea83477f4dcf837466 Mon Sep 17 00:00:00 2001 From: David Anderson Date: Fri, 18 Mar 2022 05:25:36 +0000 Subject: erofs-utils: mkfs: use extended inodes when ctime is set Currently mtime is effectively ignored because most inodes are compact. If mtime was set, and it's different from the build time, then extended inodes should be used instead. To guarantee that timestamps do not cause extended inodes, a fixed timestamp should be used (-T). Additionally, a new --ignore-mtime option has been added to preserve the old behavior. Link: https://lore.kernel.org/r/20220318052536.1358747-1-dvander@google.com Reviewed-by: Gao Xiang Signed-off-by: David Anderson Signed-off-by: Gao Xiang --- include/erofs/config.h | 1 + lib/config.c | 1 + lib/inode.c | 4 ++++ man/mkfs.erofs.1 | 5 +++++ mkfs/main.c | 5 +++++ 5 files changed, 16 insertions(+) diff --git a/include/erofs/config.h b/include/erofs/config.h index a3f9bac..1e985b0 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -44,6 +44,7 @@ struct erofs_configure { char c_chunkbits; bool c_noinline_data; bool c_ztailpacking; + bool c_ignore_mtime; #ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd; diff --git a/lib/config.c b/lib/config.c index 46d41e9..24db751 100644 --- a/lib/config.c +++ b/lib/config.c @@ -20,6 +20,7 @@ void erofs_init_configure(void) cfg.c_dbg_lvl = EROFS_WARN; cfg.c_version = PACKAGE_VERSION; cfg.c_dry_run = false; + cfg.c_ignore_mtime = false; cfg.c_compr_level_master = -1; cfg.c_force_inodeversion = 0; cfg.c_inline_xattr_tolerance = 2; diff --git a/lib/inode.c b/lib/inode.c index 1097116..6c6e42e 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -756,6 +756,10 @@ static bool erofs_should_use_inode_extended(struct erofs_inode *inode) return true; if (inode->i_nlink > USHRT_MAX) return true; + if ((inode->i_mtime != sbi.build_time || + inode->i_mtime_nsec != sbi.build_time_nsec) && + !cfg.c_ignore_mtime) + return true; return false; } diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1 index 9c7788e..d61e33e 100644 --- a/man/mkfs.erofs.1 +++ b/man/mkfs.erofs.1 @@ -112,6 +112,11 @@ Set all file gids to \fIGID\fR. .B \-\-help Display this help and exit. .TP +.B "\-\-ignore-mtime" +File modification time is ignored whenever it would cause \fBmkfs.erofs\fR to +use extended inodes over compact inodes. When not using a fixed timestamp, this +can reduce total metadata size. +.TP .BI "\-\-max-extent-bytes " # Specify maximum decompressed extent size # in bytes. .SH AUTHOR diff --git a/mkfs/main.c b/mkfs/main.c index c5fd1e5..25b72ad 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -49,6 +49,7 @@ static struct option long_options[] = { {"chunksize", required_argument, NULL, 11}, {"quiet", no_argument, 0, 12}, {"blobdev", required_argument, NULL, 13}, + {"ignore-mtime", no_argument, NULL, 14}, #ifdef WITH_ANDROID {"mount-point", required_argument, NULL, 512}, {"product-out", required_argument, NULL, 513}, @@ -96,6 +97,7 @@ static void usage(void) " --force-uid=# set all file uids to # (# = UID)\n" " --force-gid=# set all file gids to # (# = GID)\n" " --help display this help and exit\n" + " --ignore-mtime use build time instead of strict per-file modification time\n" " --max-extent-bytes=# set maximum decompressed extent size # in bytes\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG @@ -372,6 +374,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) case 13: cfg.c_blobdev_path = optarg; break; + case 14: + cfg.c_ignore_mtime = true; + break; case 1: usage(); exit(0); -- cgit v1.2.3 From eb79816f85db164af732a5bcbb42d09214845874 Mon Sep 17 00:00:00 2001 From: Fabrice Fontaine Date: Thu, 21 Apr 2022 00:10:18 +0200 Subject: erofs-utils: add --disable-werror Add an option to disable -Werror to fix the following build failure [1] with gcc 4.8 raised since version 1.4 and [2] In file included from /home/buildroot/autobuild/instance-0/output-1/host/arm-buildroot-linux-gnueabi/sysroot/usr/include/string.h:636:0, from ../include/erofs/internal.h:242, from ../include/erofs/inode.h:11, from main.c:12: In function 'memset', inlined from 'erofsdump_filetype_distribution.constprop.2' at main.c:583:9: /home/buildroot/autobuild/instance-0/output-1/host/arm-buildroot-linux-gnueabi/sysroot/usr/include/bits/string3.h:81:30: error: call to '__warn_memset_zero_len' declared with attribute warning: memset used with constant zero length parameter; this could be due to transposed parameters [-Werror] __warn_memset_zero_len (); [1] http://autobuild.buildroot.org/results/4c776ec935bbb016231b6701471887a7c9ea79e9 [2] https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git/commit/?id=cf8be8a4352a Signed-off-by: Fabrice Fontaine Link: https://lore.kernel.org/r/20220420221018.1396105-1-fontaine.fabrice@gmail.com Signed-off-by: Gao Xiang --- configure.ac | 13 ++++++++++++- dump/Makefile.am | 2 +- fsck/Makefile.am | 2 +- fuse/Makefile.am | 2 +- lib/Makefile.am | 2 +- mkfs/Makefile.am | 2 +- 6 files changed, 17 insertions(+), 6 deletions(-) diff --git a/configure.ac b/configure.ac index fa917e6..53bf882 100644 --- a/configure.ac +++ b/configure.ac @@ -11,7 +11,7 @@ AC_CONFIG_SRCDIR([config.h.in]) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR(config) -AM_INIT_AUTOMAKE([foreign -Wall -Werror]) +AM_INIT_AUTOMAKE([foreign -Wall]) # Checks for programs. AM_PROG_AR @@ -65,6 +65,12 @@ AC_ARG_ENABLE([debug], [enable_debug="$enableval"], [enable_debug="no"]) +AC_ARG_ENABLE([werror], + [AS_HELP_STRING([--enable-werror], + [enable -Werror @<:@default=no@:>@])], + [enable_werror="$enableval"], + [enable_werror="no"]) + AC_ARG_ENABLE(lz4, [AS_HELP_STRING([--disable-lz4], [disable LZ4 compression support @<:@default=enabled@:>@])], [enable_lz4="$enableval"], [enable_lz4="yes"]) @@ -197,6 +203,11 @@ AS_IF([test "x$enable_debug" != "xno"], [], [ CPPFLAGS="$CPPFLAGS -DNDEBUG" ]) +# Configure -Werror +AS_IF([test "x$enable_werror" != "xyes"], [], [ + CPPFLAGS="$CPPFLAGS -Werror" +]) + # Configure libuuid AS_IF([test "x$with_uuid" != "xno"], [ PKG_CHECK_MODULES([libuuid], [uuid]) diff --git a/dump/Makefile.am b/dump/Makefile.am index 9f0cd3f..c2bef6d 100644 --- a/dump/Makefile.am +++ b/dump/Makefile.am @@ -5,6 +5,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = dump.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} dump_erofs_SOURCES = main.c -dump_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +dump_erofs_CFLAGS = -Wall -I$(top_srcdir)/include dump_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} diff --git a/fsck/Makefile.am b/fsck/Makefile.am index 55b31ea..e6a1fb6 100644 --- a/fsck/Makefile.am +++ b/fsck/Makefile.am @@ -5,6 +5,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = fsck.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} fsck_erofs_SOURCES = main.c -fsck_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +fsck_erofs_CFLAGS = -Wall -I$(top_srcdir)/include fsck_erofs_LDADD = $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${libuuid_LIBS} ${liblz4_LIBS} ${liblzma_LIBS} diff --git a/fuse/Makefile.am b/fuse/Makefile.am index 5aa5ac0..3179a2b 100644 --- a/fuse/Makefile.am +++ b/fuse/Makefile.am @@ -4,7 +4,7 @@ AUTOMAKE_OPTIONS = foreign noinst_HEADERS = $(top_srcdir)/fuse/macosx.h bin_PROGRAMS = erofsfuse erofsfuse_SOURCES = main.c -erofsfuse_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +erofsfuse_CFLAGS = -Wall -I$(top_srcdir)/include erofsfuse_CFLAGS += -DFUSE_USE_VERSION=26 ${libfuse_CFLAGS} ${libselinux_CFLAGS} erofsfuse_LDADD = $(top_builddir)/lib/liberofs.la ${libfuse_LIBS} ${liblz4_LIBS} \ ${libselinux_LIBS} ${liblzma_LIBS} diff --git a/lib/Makefile.am b/lib/Makefile.am index 4a25013..3fad357 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -28,7 +28,7 @@ noinst_HEADERS += compressor.h liberofs_la_SOURCES = config.c io.c cache.c super.c inode.c xattr.c exclude.c \ namei.c data.c compress.c compressor.c zmap.c decompress.c \ compress_hints.c hashmap.c sha256.c blobchunk.c dir.c -liberofs_la_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +liberofs_la_CFLAGS = -Wall -I$(top_srcdir)/include if ENABLE_LZ4 liberofs_la_CFLAGS += ${LZ4_CFLAGS} liberofs_la_SOURCES += compressor_lz4.c diff --git a/mkfs/Makefile.am b/mkfs/Makefile.am index 2a4bc1d..709d9bf 100644 --- a/mkfs/Makefile.am +++ b/mkfs/Makefile.am @@ -4,6 +4,6 @@ AUTOMAKE_OPTIONS = foreign bin_PROGRAMS = mkfs.erofs AM_CPPFLAGS = ${libuuid_CFLAGS} ${libselinux_CFLAGS} mkfs_erofs_SOURCES = main.c -mkfs_erofs_CFLAGS = -Wall -Werror -I$(top_srcdir)/include +mkfs_erofs_CFLAGS = -Wall -I$(top_srcdir)/include mkfs_erofs_LDADD = ${libuuid_LIBS} $(top_builddir)/lib/liberofs.la ${libselinux_LIBS} \ ${liblz4_LIBS} ${liblzma_LIBS} -- cgit v1.2.3 From a450f38fbbc71747046b76a73dd819f507eda23d Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 18 Apr 2022 22:21:17 +0800 Subject: erofs-utils: dump: support listing sub-directories Add a new option helper to list sub-directories. Link: https://lore.kernel.org/r/20220418142117.96109-1-hsiangkao@linux.alibaba.com Signed-off-by: Gao Xiang --- dump/main.c | 40 ++++++++++++++++++++++++++++++++++++++-- man/dump.erofs.1 | 5 ++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/dump/main.c b/dump/main.c index b431e18..49ff2b7 100644 --- a/dump/main.c +++ b/dump/main.c @@ -26,6 +26,7 @@ struct erofsdump_cfg { bool show_extent; bool show_superblock; bool show_statistics; + bool show_subdirectories; erofs_nid_t nid; const char *inode_path; }; @@ -77,6 +78,7 @@ static struct option long_options[] = { {"nid", required_argument, NULL, 2}, {"device", required_argument, NULL, 3}, {"path", required_argument, NULL, 4}, + {"ls", no_argument, NULL, 5}, {0, 0, 0, 0}, }; @@ -103,9 +105,10 @@ static void usage(void) "Dump erofs layout from IMAGE, and [options] are:\n" " -S show statistic information of the image\n" " -V print the version number of dump.erofs and exit.\n" - " -e show extent info (--nid is required)\n" + " -e show extent info (INODE required)\n" " -s show information about superblock\n" " --device=X specify an extra device to be used together\n" + " --ls show directory contents (INODE required)\n" " --nid=# show the target inode info of nid #\n" " --path=X show the target inode info of path X\n" " --help display this help and exit.\n", @@ -158,6 +161,9 @@ static int erofsdump_parse_options_cfg(int argc, char **argv) dumpcfg.show_inode = true; ++dumpcfg.totalshow; break; + case 5: + dumpcfg.show_subdirectories = true; + break; default: return -EINVAL; } @@ -250,6 +256,17 @@ static void update_file_size_statatics(erofs_off_t occupied_size, stats.file_comp_size[occupied_size_mark]++; } +static int erofsdump_ls_dirent_iter(struct erofs_dir_context *ctx) +{ + char fname[EROFS_NAME_LEN + 1]; + + strncpy(fname, ctx->dname, ctx->de_namelen); + fname[ctx->de_namelen] = '\0'; + fprintf(stdout, "%10llu %u %s\n", ctx->de_nid | 0ULL, + ctx->de_ftype, fname); + return 0; +} + static int erofsdump_dirent_iter(struct erofs_dir_context *ctx) { /* skip "." and ".." dentry */ @@ -376,10 +393,29 @@ static void erofsdump_show_fileinfo(bool show_extent) fprintf(stdout, "Access: %04o/%s\n", access_mode, access_mode_str); fprintf(stdout, "Timestamp: %s.%09d\n", timebuf, inode.i_mtime_nsec); + if (dumpcfg.show_subdirectories) { + struct erofs_dir_context ctx = { + .flags = EROFS_READDIR_VALID_PNID, + .pnid = inode.nid, + .dir = &inode, + .cb = erofsdump_ls_dirent_iter, + .de_nid = 0, + .dname = "", + .de_namelen = 0, + }; + + fprintf(stdout, "\n NID TYPE FILENAME\n"); + err = erofs_iterate_dir(&ctx, false); + if (err) { + erofs_err("failed to list directory contents"); + return; + } + } + if (!dumpcfg.show_extent) return; - fprintf(stdout, "\n Ext: logical offset | length : physical offset | length \n"); + fprintf(stdout, "\n Ext: logical offset | length : physical offset | length\n"); while (map.m_la < inode.i_size) { struct erofs_map_dev mdev; diff --git a/man/dump.erofs.1 b/man/dump.erofs.1 index fd437cf..209e5f9 100644 --- a/man/dump.erofs.1 +++ b/man/dump.erofs.1 @@ -19,6 +19,9 @@ is used to retrieve erofs metadata from \fIIMAGE\fP and demonstrate Specify an extra device to be used together. You may give multiple `--device' options in the correct order. .TP +.BI "\-\-ls" +List directory contents. An inode should be specified together. +.TP .BI "\-\-nid=" NID Specify an inode NID in order to print its file information. .TP @@ -26,7 +29,7 @@ Specify an inode NID in order to print its file information. Specify an inode path in order to print its file information. .TP .BI \-e -Show the file extent information. The option depends on option --nid to specify NID. +Show the file extent information. An inode should be specified together. .TP .BI \-V Print the version number and exit. -- cgit v1.2.3 From a134ce7c39a5427343029e97a62665157bef9bc3 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 17 May 2022 01:24:35 +0800 Subject: erofs-utils: mkfs: show per-file progress Generally, users want to know the latest progress since it may take long time to build a image. Let's add a per-file progress as a start. Link: https://lore.kernel.org/r/20220516172435.167023-1-xiang@kernel.org Signed-off-by: Gao Xiang --- configure.ac | 2 ++ include/erofs/config.h | 4 +++ include/erofs/print.h | 26 +++++++++++--------- lib/config.c | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ lib/inode.c | 6 ++++- mkfs/main.c | 7 +++++- 6 files changed, 98 insertions(+), 14 deletions(-) diff --git a/configure.ac b/configure.ac index 53bf882..a736ff0 100644 --- a/configure.ac +++ b/configure.ac @@ -140,6 +140,8 @@ AC_CHECK_HEADERS(m4_flatten([ unistd.h ])) +AC_HEADER_TIOCGWINSZ + # Checks for typedefs, structures, and compiler characteristics. AC_C_INLINE AC_TYPE_INT64_T diff --git a/include/erofs/config.h b/include/erofs/config.h index 1e985b0..0d0916c 100644 --- a/include/erofs/config.h +++ b/include/erofs/config.h @@ -45,6 +45,7 @@ struct erofs_configure { bool c_noinline_data; bool c_ztailpacking; bool c_ignore_mtime; + bool c_showprogress; #ifdef HAVE_LIBSELINUX struct selabel_handle *sehnd; @@ -92,6 +93,9 @@ static inline int erofs_selabel_open(const char *file_contexts) } #endif +void erofs_update_progressinfo(const char *fmt, ...); +char *erofs_trim_for_progressinfo(const char *str, int placeholder); + #ifdef __cplusplus } #endif diff --git a/include/erofs/print.h b/include/erofs/print.h index f188a6b..a896d75 100644 --- a/include/erofs/print.h +++ b/include/erofs/print.h @@ -41,37 +41,39 @@ enum { #define PR_FMT_FUNC_LINE(fmt) pr_fmt(fmt), __func__, __LINE__ #endif +void erofs_msg(int dbglv, const char *fmt, ...); + #define erofs_dbg(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_DBG) { \ - fprintf(stdout, \ - " " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_DBG, \ + " " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ } \ } while (0) #define erofs_info(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_INFO) { \ - fprintf(stdout, \ - " " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_INFO, \ + " " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ fflush(stdout); \ } \ } while (0) #define erofs_warn(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_WARN) { \ - fprintf(stdout, \ - " " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_WARN, \ + " " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ fflush(stdout); \ } \ } while (0) #define erofs_err(fmt, ...) do { \ if (cfg.c_dbg_lvl >= EROFS_ERR) { \ - fprintf(stderr, \ - " " PR_FMT_FUNC_LINE(fmt), \ - ##__VA_ARGS__); \ + erofs_msg(EROFS_ERR, \ + " " PR_FMT_FUNC_LINE(fmt), \ + ##__VA_ARGS__); \ } \ } while (0) diff --git a/lib/config.c b/lib/config.c index 24db751..3963df2 100644 --- a/lib/config.c +++ b/lib/config.c @@ -6,9 +6,13 @@ */ #include #include +#include #include "erofs/print.h" #include "erofs/internal.h" #include "liberofs_private.h" +#ifdef HAVE_SYS_IOCTL_H +#include +#endif struct erofs_configure cfg; struct erofs_sb_info sbi; @@ -91,3 +95,66 @@ int erofs_selabel_open(const char *file_contexts) return 0; } #endif + +static bool __erofs_is_progressmsg; + +char *erofs_trim_for_progressinfo(const char *str, int placeholder) +{ + struct winsize winsize; + int col, len; + +#ifdef GWINSZ_IN_SYS_IOCTL + if(ioctl(1, TIOCGWINSZ, &winsize) >= 0 && + winsize.ws_col > 0) + col = winsize.ws_col; + else +#endif + col = 80; + + if (col <= placeholder) + return strdup(""); + + len = strlen(str); + /* omit over long prefixes */ + if (len > col - placeholder) { + char *s = strdup(str + len - (col - placeholder)); + + if (col > placeholder + 2) { + s[0] = '['; + s[1] = ']'; + } + return s; + } + return strdup(str); +} + +void erofs_msg(int dbglv, const char *fmt, ...) +{ + va_list ap; + FILE *f = dbglv >= EROFS_ERR ? stderr : stdout; + + if (__erofs_is_progressmsg) { + fputc('\n', f); + __erofs_is_progressmsg = false; + } + va_start(ap, fmt); + vfprintf(f, fmt, ap); + va_end(ap); +} + +void erofs_update_progressinfo(const char *fmt, ...) +{ + char msg[8192]; + va_list ap; + + if (cfg.c_dbg_lvl >= EROFS_INFO || !cfg.c_showprogress) + return; + + va_start(ap, fmt); + vsprintf(msg, fmt, ap); + va_end(ap); + + printf("\r\033[K%s", msg); + __erofs_is_progressmsg = true; + fflush(stdout); +} diff --git a/lib/inode.c b/lib/inode.c index 6c6e42e..f192510 100644 --- a/lib/inode.c +++ b/lib/inode.c @@ -1083,7 +1083,7 @@ static struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir) erofs_fixup_meta_blkaddr(dir); list_for_each_entry(d, &dir->i_subdirs, d_child) { - char buf[PATH_MAX]; + char buf[PATH_MAX], *trimmed; unsigned char ftype; if (is_dot_dotdot(d->name)) { @@ -1098,6 +1098,10 @@ static struct erofs_inode *erofs_mkfs_build_tree(struct erofs_inode *dir) goto fail; } + trimmed = erofs_trim_for_progressinfo(erofs_fspath(buf), + sizeof("Processing ...") - 1); + erofs_update_progressinfo("Processing %s ...", trimmed); + free(trimmed); d->inode = erofs_mkfs_build_tree_from_path(dir, buf); if (IS_ERR(d->inode)) { ret = PTR_ERR(d->inode); diff --git a/mkfs/main.c b/mkfs/main.c index 25b72ad..9d43cd4 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -423,8 +423,10 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) erofs_err("unexpected argument: %s\n", argv[optind]); return -EINVAL; } - if (quiet) + if (quiet) { cfg.c_dbg_lvl = EROFS_ERR; + cfg.c_showprogress = false; + } return 0; } @@ -520,6 +522,7 @@ static int erofs_mkfs_superblock_csum_set(void) static void erofs_mkfs_default_options(void) { + cfg.c_showprogress = true; cfg.c_legacy_compress = false; sbi.feature_incompat = EROFS_FEATURE_INCOMPAT_LZ4_0PADDING; sbi.feature_compat = EROFS_FEATURE_COMPAT_SB_CHKSUM | @@ -738,6 +741,8 @@ exit: erofs_err("\tCould not format the device : %s\n", erofs_strerror(err)); return 1; + } else { + erofs_update_progressinfo("Build completed.\n"); } return 0; } -- cgit v1.2.3 From 3393ed613a2c90360e657363d69eae15d07444c5 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Tue, 31 May 2022 11:23:47 +0800 Subject: erofs-utils: don't warn `big pcluster' feature anymore It's already used by some vendors. Therefore, let's follow commit 1e59af07c7f3 ("erofs: do not prompt for risk any more when using big pcluster") here as well. Link: https://lore.kernel.org/r/20220531032347.62211-1-hsiangkao@linux.alibaba.com Signed-off-by: Gao Xiang --- lib/compress.c | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/compress.c b/lib/compress.c index e050df0..7ebc534 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -840,7 +840,6 @@ int z_erofs_compress_init(struct erofs_buffer_head *sb_bh) return -EINVAL; } erofs_sb_set_big_pcluster(); - erofs_warn("EXPERIMENTAL big pcluster feature in use. Use at your own risk!"); } if (ret != Z_EROFS_COMPRESSION_LZ4) -- cgit v1.2.3 From 48226b3f472d9dfc29f9c79d8342fc0947467727 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 11 Jun 2022 16:22:48 +0800 Subject: erofs-utils: mkfs: introduce `--preserve-mtime' In the past versions, `-Eforce-inode-compact' worked since timestamps were ignored. Currently, since we don't ignore mtime by default any more, `-Eforce-inode-compact' fails and that breaks compatibility. Let's fix `-Eforce-inode-compact' to ignore mtime as the past versions did, also add another option `--preserve-mtime' for this. Link: https://lore.kernel.org/r/20220611082248.338369-1-xiang@kernel.org Signed-off-by: Gao Xiang --- man/mkfs.erofs.1 | 4 ++++ mkfs/main.c | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1 index d61e33e..6017760 100644 --- a/man/mkfs.erofs.1 +++ b/man/mkfs.erofs.1 @@ -119,6 +119,10 @@ can reduce total metadata size. .TP .BI "\-\-max-extent-bytes " # Specify maximum decompressed extent size # in bytes. +.TP +.B "\-\-preserve-mtime" +File modification time is preserved whenever \fBmkfs.erofs\fR decides to use +extended inodes over compact inodes. .SH AUTHOR This version of \fBmkfs.erofs\fR is written by Li Guifu , Miao Xie and Gao Xiang with diff --git a/mkfs/main.c b/mkfs/main.c index 9d43cd4..d2c9830 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -50,6 +50,7 @@ static struct option long_options[] = { {"quiet", no_argument, 0, 12}, {"blobdev", required_argument, NULL, 13}, {"ignore-mtime", no_argument, NULL, 14}, + {"preserve-mtime", no_argument, NULL, 15}, #ifdef WITH_ANDROID {"mount-point", required_argument, NULL, 512}, {"product-out", required_argument, NULL, 513}, @@ -99,6 +100,7 @@ static void usage(void) " --help display this help and exit\n" " --ignore-mtime use build time instead of strict per-file modification time\n" " --max-extent-bytes=# set maximum decompressed extent size # in bytes\n" + " --preserve-mtime keep per-file modification time strictly\n" " --quiet quiet execution (do not write anything to standard output.)\n" #ifndef NDEBUG " --random-pclusterblks randomize pclusterblks for big pcluster (debugging only)\n" @@ -158,6 +160,7 @@ static int parse_extended_opts(const char *opts) if (vallen) return -EINVAL; cfg.c_force_inodeversion = FORCE_INODE_COMPACT; + cfg.c_ignore_mtime = true; } if (MATCH_EXTENTED_OPT("force-inode-extended", token, keylen)) { @@ -377,6 +380,9 @@ static int mkfs_parse_options_cfg(int argc, char *argv[]) case 14: cfg.c_ignore_mtime = true; break; + case 15: + cfg.c_ignore_mtime = false; + break; case 1: usage(); exit(0); -- cgit v1.2.3 From 9551ebad6efae339112e6fed85b091730edcbf1a Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 11 Jun 2022 17:28:55 +0800 Subject: erofs-utils: fsck: support extracting special files Now device special files, named pipes and UNIX domain sockets can be extracted too. Link: https://lore.kernel.org/r/20220611092855.347106-1-xiang@kernel.org Signed-off-by: Gao Xiang --- fsck/main.c | 48 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/fsck/main.c b/fsck/main.c index 7fc7b6f..5a2f659 100644 --- a/fsck/main.c +++ b/fsck/main.c @@ -638,6 +638,44 @@ out: return ret; } +static int erofs_extract_special(struct erofs_inode *inode) +{ + bool tryagain = true; + int ret; + + erofs_dbg("extract special to path: %s", fsckcfg.extract_path); + + /* verify data chunk layout */ + ret = erofs_verify_inode_data(inode, -1); + if (ret) + return ret; + +again: + if (mknod(fsckcfg.extract_path, inode->i_mode, inode->u.i_rdev) < 0) { + if (errno == EEXIST && fsckcfg.overwrite && tryagain) { + erofs_warn("try to forcely remove file %s", + fsckcfg.extract_path); + if (unlink(fsckcfg.extract_path) < 0) { + erofs_err("failed to remove: %s", + fsckcfg.extract_path); + return -errno; + } + tryagain = false; + goto again; + } + if (errno == EEXIST || fsckcfg.superuser) { + erofs_err("failed to create special file: %s", + fsckcfg.extract_path); + ret = -errno; + } else { + erofs_warn("failed to create special file: %s, skipped", + fsckcfg.extract_path); + ret = -ECANCELED; + } + } + return ret; +} + static int erofsfsck_dirent_iter(struct erofs_dir_context *ctx) { int ret; @@ -698,6 +736,12 @@ static int erofsfsck_check_inode(erofs_nid_t pnid, erofs_nid_t nid) case S_IFLNK: ret = erofs_extract_symlink(&inode); break; + case S_IFCHR: + case S_IFBLK: + case S_IFIFO: + case S_IFSOCK: + ret = erofs_extract_special(&inode); + break; default: /* TODO */ goto verify; @@ -707,7 +751,7 @@ verify: /* verify data chunk layout */ ret = erofs_verify_inode_data(&inode, -1); } - if (ret) + if (ret && ret != -ECANCELED) goto out; /* XXXX: the dir depth should be restricted in order to avoid loops */ @@ -725,6 +769,8 @@ verify: if (!ret) erofsfsck_set_attributes(&inode, fsckcfg.extract_path); + if (ret == -ECANCELED) + ret = 0; out: if (ret && ret != -EIO) fsckcfg.corrupted = true; -- cgit v1.2.3 From b2cedc4fd653992864bc532a19c7289f11706366 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 11 Jun 2022 19:36:21 +0800 Subject: erofs-utils: update README update README for the upcoming erofs-utils v1.5, such as: - Image extraction with fsck.erofs; - Container image use cases. Link: https://lore.kernel.org/r/20220611113621.359723-1-xiang@kernel.org Signed-off-by: Gao Xiang --- README | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/README b/README index aadd880..92b3128 100644 --- a/README +++ b/README @@ -1,9 +1,13 @@ erofs-utils =========== -erofs-utils includes user-space tools for EROFS filesystem. -Currently mkfs.erofs, (experimental) erofsfuse, dump.erofs, fsck.erofs -are available. +userspace tools for EROFS filesystem, currently including: + + mkfs.erofs filesystem formatter + erofsfuse FUSE daemon alternative + dump.erofs filesystem analyzer + fsck.erofs filesystem compatibility & consistency checker as well + as extractor Dependencies & build -------------------- @@ -59,6 +63,7 @@ In order to enable LZMA support, build with the following commands: Additionally, you could specify liblzma build paths with: --with-liblzma-incdir and --with-liblzma-libdir + mkfs.erofs ---------- @@ -133,8 +138,9 @@ git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs-utils.git -b obsoleted PLEASE NOTE: This version is highly _NOT recommended_ now. -erofsfuse (experimental) ------------------------- + +erofsfuse +--------- erofsfuse is introduced to support EROFS format for various platforms (including older linux kernels) and new on-disk features iteration. @@ -147,9 +153,9 @@ significant I/O overhead, double caching, etc.) Therefore, NEVER use it if performance is the top concern. -Note that xattr & ACL aren't implemented yet due to the current Android -use-case vs limited time. If you have some interest, contribution is, -as always, welcome. +Note that extended attributes and ACLs aren't implemented yet due to +the current Android use case vs limited time. If you are interested, +contribution is, as always, welcome. How to build erofsfuse ~~~~~~~~~~~~~~~~~~~~~~ @@ -178,24 +184,52 @@ To debug erofsfuse (also automatically run in foreground): To unmount an erofsfuse mountpoint as a non-root user: $ fusermount -u foo/ -dump.erofs and fsck.erofs (experimental) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -dump.erofs and fsck.erofs are two new experimental tools to analyse -and check EROFS file systems. +dump.erofs and fsck.erofs +------------------------- + +dump.erofs and fsck.erofs are used to analyze, check, and extract +EROFS filesystems. Note that extended attributes and ACLs are still +unsupported when extracting images with fsck.erofs. + +Container images +---------------- + +EROFS filesystem is well-suitably used for container images with +advanced features like chunk-based files, multi-devices (blobs) +and new fscache backend for lazy pulling and cache management, etc. + +For example, CNCF Dragonfly Nydus image service [7] introduces an +(EROFS-compatible) RAFS v6 image format to overcome flaws of the +current OCIv1 tgz images so that: -They are still incomplete and actively under development by the -community. But you could check them out if needed in advance. + - Images can be downloaded on demand in chunks aka lazy pulling with + new fscache backend (5.19+) or userspace block devices (5.16+); + + - Finer chunk-based content-addressable data deduplication to minimize + storage, transmission and memory footprints; + + - Merged filesystem tree to remove all metadata of intermediate layers + as an option; + + - (e)stargz, zstd::chunked and other formats can be converted and run + on the fly; + + - and more. + +Apart from Dragonfly Nydus, a native user daemon is planned to be added +to erofs-utils to parse EROFS, (e)stargz and zstd::chunked images from +network too as a real part of EROFS filesystem project. -Report, feedback and/or contribution are welcomed. Contribution ------------ -erofs-utils is under GPLv2+ as a part of EROFS filesystem project, -feel free to send patches or feedback to: +erofs-utils is a part of EROFS filesystem project, feel free to send +patches or feedback to: linux-erofs mailing list + Comments -------- @@ -251,3 +285,6 @@ Comments which is also resolved in lz4-1.9.3. [6] https://tukaani.org/xz/xz-5.3.2alpha.tar.xz + +[7] https://nydus.dev + https://github.com/dragonflyoss/image-service -- cgit v1.2.3 From 100cae2b98a70ac69ae7a09ed94f86ddf8c89fe2 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sat, 11 Jun 2022 20:01:35 +0800 Subject: erofs-utils: use EROFS_BLKSIZ unconditionally Use EROFS_BLKSIZ only in the codebase and just warn potential incompatible PAGE_SIZE. It fixes, http://autobuild.buildroot.net/results/2daa0f7a7418f7491b8b4c5495904abb86efa809/build-end.log Link: https://lore.kernel.org/r/20220611120135.363323-1-xiang@kernel.org Signed-off-by: Gao Xiang --- include/erofs/internal.h | 2 +- lib/compressor.c | 4 ++-- lib/data.c | 6 +++--- lib/dir.c | 2 +- lib/namei.c | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/erofs/internal.h b/include/erofs/internal.h index 2686570..6a70f11 100644 --- a/include/erofs/internal.h +++ b/include/erofs/internal.h @@ -36,7 +36,7 @@ typedef unsigned short umode_t; /* no obvious reason to support explicit PAGE_SIZE != 4096 for now */ #if PAGE_SIZE != 4096 -#error incompatible PAGE_SIZE is already defined +#warning EROFS may be incompatible on your platform #endif #ifndef PAGE_MASK diff --git a/lib/compressor.c b/lib/compressor.c index 3666496..a46bc39 100644 --- a/lib/compressor.c +++ b/lib/compressor.c @@ -76,8 +76,8 @@ int erofs_compressor_init(struct erofs_compress *c, char *alg_name) c->compress_threshold = 100; /* optimize for 4k size page */ - c->destsize_alignsize = PAGE_SIZE; - c->destsize_redzone_begin = PAGE_SIZE - 16; + c->destsize_alignsize = EROFS_BLKSIZ; + c->destsize_redzone_begin = EROFS_BLKSIZ - 16; c->destsize_redzone_end = EROFS_CONFIG_COMPR_DEF_BOUNDARY; if (!alg_name) { diff --git a/lib/data.c b/lib/data.c index e57707e..6bc554d 100644 --- a/lib/data.c +++ b/lib/data.c @@ -22,7 +22,7 @@ static int erofs_map_blocks_flatmode(struct erofs_inode *inode, trace_erofs_map_blocks_flatmode_enter(inode, map, flags); - nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); + nblocks = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ); lastblk = nblocks - tailendpacking; /* there is no hole in flatmode */ @@ -37,8 +37,8 @@ static int erofs_map_blocks_flatmode(struct erofs_inode *inode, vi->xattr_isize + erofs_blkoff(map->m_la); map->m_plen = inode->i_size - offset; - /* inline data should be located in one meta block */ - if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) { + /* inline data should be located in the same meta block */ + if (erofs_blkoff(map->m_pa) + map->m_plen > EROFS_BLKSIZ) { erofs_err("inline data cross block boundary @ nid %" PRIu64, vi->nid); DBG_BUGON(1); diff --git a/lib/dir.c b/lib/dir.c index 8955931..e6b9283 100644 --- a/lib/dir.c +++ b/lib/dir.c @@ -148,7 +148,7 @@ int erofs_iterate_dir(struct erofs_dir_context *ctx, bool fsck) nameoff = le16_to_cpu(de->nameoff); if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { + nameoff >= EROFS_BLKSIZ) { erofs_err("invalid de[0].nameoff %u @ nid %llu, lblk %u", nameoff, dir->nid | 0ULL, lblk); return -EFSCORRUPTED; diff --git a/lib/namei.c b/lib/namei.c index 8e9867d..7b69a59 100644 --- a/lib/namei.c +++ b/lib/namei.c @@ -212,7 +212,7 @@ int erofs_namei(struct nameidata *nd, nameoff = le16_to_cpu(de->nameoff); if (nameoff < sizeof(struct erofs_dirent) || - nameoff >= PAGE_SIZE) { + nameoff >= EROFS_BLKSIZ) { erofs_err("invalid de[0].nameoff %u @ nid %llu", nameoff, nid | 0ULL); return -EFSCORRUPTED; -- cgit v1.2.3 From a0536697ab5309bbf07c1d90f3b645eb89731560 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Sun, 12 Jun 2022 13:50:10 +0800 Subject: erofs-utils: manpage: add missing -Eztailpacking option Complete the manpage for ztailpacking feature and clean up the whole subsection. Link: https://lore.kernel.org/r/20220612055010.485042-1-xiang@kernel.org Signed-off-by: Gao Xiang --- man/mkfs.erofs.1 | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/man/mkfs.erofs.1 b/man/mkfs.erofs.1 index 6017760..d811f20 100644 --- a/man/mkfs.erofs.1 +++ b/man/mkfs.erofs.1 @@ -42,24 +42,28 @@ and may take an argument using the equals ('=') sign. The following extended options are supported: .RS 1.2i .TP -.BI legacy-compress -Disable "decompression in-place" and "compacted indexes" support, which is used -when generating EROFS images for kernel version < 5.3. -.TP .BI force-inode-compact Forcely generate compact inodes (32-byte inodes) to output. .TP .BI force-inode-extended Forcely generate extended inodes (64-byte inodes) to output. .TP -.BI noinline_data -Don't inline regular files for FSDAX support (Linux v5.15+). -.TP .BI force-inode-blockmap Forcely generate inode chunk format in 4-byte block address array. .TP .BI force-chunk-indexes Forcely generate inode chunk format in 8-byte chunk indexes (with device id). +.TP +.BI legacy-compress +Drop "inplace decompression" and "compacted indexes" support, which is used +to generate compatible EROFS images for Linux v4.19 - 5.3. +.TP +.BI noinline_data +Don't inline regular files to enable FSDAX for these files (Linux v5.15+). +.TP +.BI ztailpacking +Pack the tail part (pcluster) of compressed files into its metadata to save +more space and the tail part I/O. (Linux v5.17+) .RE .TP .BI "\-T " # -- cgit v1.2.3 From 22a0d49117ec7b7a7972dd1bd4f8115f237dc05f Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 13 Jun 2022 00:45:18 +0800 Subject: erofs-utils: bump up EROFS_CONFIG_COMPR_MAX_SZ Otherwise, compression ratios could be limited when pcluster size is large. Use a static variable for now. Link: https://lore.kernel.org/r/20220612164518.503836-1-xiang@kernel.org Signed-off-by: Gao Xiang --- include/erofs/compress.h | 4 +--- lib/compress.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/erofs/compress.h b/include/erofs/compress.h index 40df2bc..24f6204 100644 --- a/include/erofs/compress.h +++ b/include/erofs/compress.h @@ -14,9 +14,7 @@ extern "C" #include "internal.h" -/* workaround for an upstream lz4 compression issue, which can crash us */ -/* #define EROFS_CONFIG_COMPR_MAX_SZ (1024 * 1024) */ -#define EROFS_CONFIG_COMPR_MAX_SZ (900 * 1024) +#define EROFS_CONFIG_COMPR_MAX_SZ (3000 * 1024) #define EROFS_CONFIG_COMPR_MIN_SZ (32 * 1024) void z_erofs_drop_inline_pcluster(struct erofs_inode *inode); diff --git a/lib/compress.c b/lib/compress.c index 7ebc534..ee3b856 100644 --- a/lib/compress.c +++ b/lib/compress.c @@ -607,7 +607,7 @@ void z_erofs_drop_inline_pcluster(struct erofs_inode *inode) int erofs_write_compressed_file(struct erofs_inode *inode) { struct erofs_buffer_head *bh; - struct z_erofs_vle_compress_ctx ctx; + static struct z_erofs_vle_compress_ctx ctx; erofs_off_t remaining; erofs_blk_t blkaddr, compressed_blocks; unsigned int legacymetasize; -- cgit v1.2.3 From a2821a66b42aee5430bccee82c280e38d1e9ab29 Mon Sep 17 00:00:00 2001 From: Gao Xiang Date: Mon, 13 Jun 2022 00:00:00 +0800 Subject: erofs-utils: release 1.5 Signed-off-by: Gao Xiang --- ChangeLog | 13 +++++++++++++ VERSION | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ChangeLog b/ChangeLog index 6b53d24..97d7336 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,16 @@ +erofs-utils 1.5 + + * This release includes the following updates: + - (fsck.erofs) support filesystem extraction (Igor Ostapenko); + - support ztailpacking inline feature for compressed files (Yue Hu); + - (dump.erofs) support listing directories; + - more liberofs APIs (including iterate APIs) (me, Kelvin Zhang); + - use mtime to allow more control over the timestamps (David Anderson); + - switch to GPL-2.0+ OR Apache-2.0 dual license for liberofs; + - various bugfixes and cleanups; + + -- Gao Xiang Mon, 13 Jun 2022 00:00:00 +0800 + erofs-utils 1.4 * This release includes the following updates: diff --git a/VERSION b/VERSION index babe602..ef7a460 100644 --- a/VERSION +++ b/VERSION @@ -1,2 +1,2 @@ -1.4 -2021-11-22 +1.5 +2022-06-13 -- cgit v1.2.3