diff options
40 files changed, 376 insertions, 243 deletions
@@ -1,5 +1,16 @@ // Copyright 2017 The Android Open Source Project +cc_defaults { + name: "e2fsprogs-defaults", + cflags: ["-Wall", "-Werror"], + target: { + darwin: { + // Still has unfixed/unsuppressed warnings. + cflags: ["-Wno-error"], + }, + }, +} + subdirs = [ "contrib", "debugfs", @@ -1,3 +1 @@ -adriens@google.com -jinqian@google.com tytso@google.com diff --git a/contrib/Android.bp b/contrib/Android.bp index d4d7874b..8bde657f 100644 --- a/contrib/Android.bp +++ b/contrib/Android.bp @@ -2,33 +2,15 @@ subdirs = ["android"] -//########################################################################## -// Build fsstress - -cc_binary { - name: "fsstress", - host_supported: true, - - srcs: ["fsstress.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], - system_shared_libs: ["libc", "libdl"], - - target: { - darwin: { - enabled: false, - }, - }, -} - //######################################################################## // Build add_ext4_encrypt cc_binary { name: "add_ext4_encrypt", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["add_ext4_encrypt.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_com_err", diff --git a/contrib/android/Android.bp b/contrib/android/Android.bp index 18d38501..67844f98 100644 --- a/contrib/android/Android.bp +++ b/contrib/android/Android.bp @@ -6,6 +6,7 @@ cc_binary { name: "e2fsdroid", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "e2fsdroid.c", @@ -15,9 +16,7 @@ cc_binary { "base_fs.c", "perms.c", "basefs_allocator.c", - "hashmap.c", ], - cflags: ["-W", "-Wall"], target: { host: { static_libs: [ @@ -32,6 +31,9 @@ cc_binary { "libcrypto", "liblog", ], + sanitize: { + address: false, // http://b/68387795 - heap overflow in e2fsdroid + }, }, android: { shared_libs: [ @@ -54,21 +56,13 @@ cc_binary { cc_binary { name: "ext2simg", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["ext2simg.c"], - cflags: ["-W", "-Wall"], shared_libs: [ "libext2fs", "libext2_com_err", "libsparse", + "libz", ], - - target: { - host: { - shared_libs: ["libz-host"], - }, - android: { - shared_libs: ["libz"], - }, - }, } diff --git a/contrib/android/Android.mk b/contrib/android/Android.mk index 98ce01d8..68d925de 100644 --- a/contrib/android/Android.mk +++ b/contrib/android/Android.mk @@ -10,9 +10,8 @@ e2fsdroid_src_files := \ base_fs.c \ perms.c \ basefs_allocator.c \ - hashmap.c \ -e2fsdroid_cflags := -W -Wall +e2fsdroid_cflags := -W -Wall -Werror -Wno-error=macro-redefined e2fsdroid_static_libraries := \ libext2_com_err \ diff --git a/contrib/android/base_fs.c b/contrib/android/base_fs.c index 2dcb5fe3..14203050 100644 --- a/contrib/android/base_fs.c +++ b/contrib/android/base_fs.c @@ -99,24 +99,25 @@ static void free_base_fs_entry(void *e) } } -struct hashmap *basefs_parse(const char *file, const char *mountpoint) +struct ext2fs_hashmap *basefs_parse(const char *file, const char *mountpoint) { int err; - struct hashmap *entries = NULL; + struct ext2fs_hashmap *entries = NULL; struct basefs_entry *entry; FILE *f = basefs_open(file); if (!f) return NULL; - entries = hashmap_create(djb2_hash, free_base_fs_entry, 1024); + entries = ext2fs_hashmap_create(ext2fs_djb2_hash, free_base_fs_entry, 1024); if (!entries) goto end; while ((entry = basefs_readline(f, mountpoint, &err))) - hashmap_add(entries, entry, entry->path); + ext2fs_hashmap_add(entries, entry, entry->path, + strlen(entry->path)); if (err) { fclose(f); - hashmap_free(entries); + ext2fs_hashmap_free(entries); return NULL; } end: diff --git a/contrib/android/base_fs.h b/contrib/android/base_fs.h index 94bae293..e9f46b4a 100644 --- a/contrib/android/base_fs.h +++ b/contrib/android/base_fs.h @@ -13,6 +13,6 @@ struct basefs_entry { extern struct fsmap_format base_fs_format; -struct hashmap *basefs_parse(const char *file, const char *mountpoint); +struct ext2fs_hashmap *basefs_parse(const char *file, const char *mountpoint); #endif /* !BASE_FS_H */ diff --git a/contrib/android/basefs_allocator.c b/contrib/android/basefs_allocator.c index 3d014a22..c44532f6 100644 --- a/contrib/android/basefs_allocator.c +++ b/contrib/android/basefs_allocator.c @@ -6,7 +6,7 @@ #include "base_fs.h" struct base_fs_allocator { - struct hashmap *entries; + struct ext2fs_hashmap *entries; struct basefs_entry *cur_entry; }; @@ -36,9 +36,9 @@ errcode_t base_fs_alloc_load(ext2_filsys fs, const char *file, { errcode_t retval; struct basefs_entry *e; - struct hashmap_entry *it = NULL; + struct ext2fs_hashmap_entry *it = NULL; struct base_fs_allocator *allocator; - struct hashmap *entries = basefs_parse(file, mountpoint); + struct ext2fs_hashmap *entries = basefs_parse(file, mountpoint); if (!entries) return -1; @@ -49,7 +49,7 @@ errcode_t base_fs_alloc_load(ext2_filsys fs, const char *file, retval = ext2fs_read_bitmaps(fs); if (retval) goto err_bitmap; - while ((e = hashmap_iter_in_order(entries, &it))) + while ((e = ext2fs_hashmap_iter_in_order(entries, &it))) fs_reserve_blocks_range(fs, e->head); allocator->cur_entry = NULL; @@ -64,7 +64,7 @@ errcode_t base_fs_alloc_load(ext2_filsys fs, const char *file, err_bitmap: free(allocator); err_alloc: - hashmap_free(entries); + ext2fs_hashmap_free(entries); return EXIT_FAILURE; } @@ -97,10 +97,10 @@ static errcode_t basefs_block_allocator(ext2_filsys fs, blk64_t goal, void base_fs_alloc_cleanup(ext2_filsys fs) { struct basefs_entry *e; - struct hashmap_entry *it = NULL; + struct ext2fs_hashmap_entry *it = NULL; struct base_fs_allocator *allocator = fs->priv_data; - while ((e = hashmap_iter_in_order(allocator->entries, &it))) { + while ((e = ext2fs_hashmap_iter_in_order(allocator->entries, &it))) { fs_free_blocks_range(fs, e->head); delete_block_ranges(e->head); e->head = e->tail = NULL; @@ -108,7 +108,7 @@ void base_fs_alloc_cleanup(ext2_filsys fs) fs->priv_data = NULL; fs->get_alloc_block2 = NULL; - hashmap_free(allocator->entries); + ext2fs_hashmap_free(allocator->entries); free(allocator); } @@ -123,8 +123,9 @@ errcode_t base_fs_alloc_set_target(ext2_filsys fs, const char *target_path, return 0; if (allocator) - allocator->cur_entry = hashmap_lookup(allocator->entries, - target_path); + allocator->cur_entry = ext2fs_hashmap_lookup(allocator->entries, + target_path, + strlen(target_path)); return 0; } diff --git a/contrib/android/e2fsdroid.c b/contrib/android/e2fsdroid.c index f043822b..2fe922d8 100644 --- a/contrib/android/e2fsdroid.c +++ b/contrib/android/e2fsdroid.c @@ -1,3 +1,5 @@ +#define _GNU_SOURCE + #include <stdio.h> #include <getopt.h> #include <string.h> @@ -30,7 +32,7 @@ static void usage(int ret) { fprintf(stderr, "%s [-B block_list] [-D basefs_out] [-T timestamp]\n" "\t[-C fs_config] [-S file_contexts] [-p product_out]\n" - "\t[-a mountpoint] [-d basefs_in] [-f src_dir] [-e] image\n", + "\t[-a mountpoint] [-d basefs_in] [-f src_dir] [-e] [-s] image\n", prog_name); exit(ret); } @@ -41,7 +43,10 @@ static char *absolute_path(const char *file) char cwd[PATH_MAX]; if (file[0] != '/') { - getcwd(cwd, PATH_MAX); + if (getcwd(cwd, PATH_MAX) == NULL) { + fprintf(stderr, "Failed to getcwd\n"); + exit(EXIT_FAILURE); + } ret = malloc(strlen(cwd) + 1 + strlen(file) + 1); if (ret) sprintf(ret, "%s/%s", cwd, file); @@ -68,7 +73,7 @@ int main(int argc, char *argv[]) add_error_table(&et_ext2_error_table); - while ((c = getopt (argc, argv, "T:C:S:p:a:D:d:B:f:e")) != EOF) { + while ((c = getopt (argc, argv, "T:C:S:p:a:D:d:B:f:es")) != EOF) { switch (c) { case 'T': fixed_time = strtoul(optarg, &p, 0); @@ -114,6 +119,9 @@ int main(int argc, char *argv[]) case 'e': android_sparse_file = 0; break; + case 's': + flags |= EXT2_FLAG_SHARE_DUP; + break; default: usage(EXIT_FAILURE); } diff --git a/contrib/android/ext2simg.c b/contrib/android/ext2simg.c index fcefe1d7..017e16ff 100644 --- a/contrib/android/ext2simg.c +++ b/contrib/android/ext2simg.c @@ -179,7 +179,6 @@ int main(int argc, char *argv[]) { int opt; int out_fd; - errcode_t retval; struct sparse_file *s; while ((opt = getopt(argc, argv, "czS")) != -1) { diff --git a/contrib/android/hashmap.c b/contrib/android/hashmap.c deleted file mode 100644 index eee00714..00000000 --- a/contrib/android/hashmap.c +++ /dev/null @@ -1,76 +0,0 @@ -#include "hashmap.h" -#include <string.h> - -uint32_t djb2_hash(const void *str) -{ - int c; - const char *s = str; - uint32_t hash = 5381; - - while ((c = *s++)) - hash = ((hash << 5) + hash) + c; - return hash; -} - -struct hashmap *hashmap_create(uint32_t(*hash_fct)(const void*), - void(*free_fct)(void*), size_t size) -{ - struct hashmap *h = calloc(sizeof(struct hashmap) + - sizeof(struct hashmap_entry) * size, 1); - h->size = size; - h->free = free_fct; - h->hash = hash_fct; - h->first = h->last = NULL; - return h; -} - -void hashmap_add(struct hashmap *h, void *data, const void *key) -{ - uint32_t hash = h->hash(key) % h->size; - struct hashmap_entry *e = malloc(sizeof(*e)); - - e->data = data; - e->key = key; - e->next = h->entries[hash]; - h->entries[hash] = e; - - e->list_prev = NULL; - e->list_next = h->first; - if (h->first) - h->first->list_prev = e; - h->first = e; - if (!h->last) - h->last = e; -} - -void *hashmap_lookup(struct hashmap *h, const void *key) -{ - struct hashmap_entry *iter; - uint32_t hash = h->hash(key) % h->size; - - for (iter = h->entries[hash]; iter; iter = iter->next) - if (!strcmp(iter->key, key)) - return iter->data; - return NULL; -} - -void *hashmap_iter_in_order(struct hashmap *h, struct hashmap_entry **it) -{ - *it = *it ? (*it)->list_next : h->first; - return *it ? (*it)->data : NULL; -} - -void hashmap_free(struct hashmap *h) -{ - for (size_t i = 0; i < h->size; ++i) { - struct hashmap_entry *it = h->entries[i]; - while (it) { - struct hashmap_entry *tmp = it->next; - if (h->free) - h->free(it->data); - free(it); - it = tmp; - } - } - free(h); -} diff --git a/contrib/android/hashmap.h b/contrib/android/hashmap.h deleted file mode 100644 index 70d0ed19..00000000 --- a/contrib/android/hashmap.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef HASHMAP_H -# define HASHMAP_H - -# include <stdlib.h> -# include <stdint.h> - -struct hashmap { - uint32_t size; - uint32_t(*hash)(const void *key); - void(*free)(void*); - struct hashmap_entry *first; - struct hashmap_entry *last; - struct hashmap_entry { - void *data; - const void *key; - struct hashmap_entry *next; - struct hashmap_entry *list_next; - struct hashmap_entry *list_prev; - } *entries[0]; -}; - -struct hashmap *hashmap_create(uint32_t(*hash_fct)(const void*), - void(*free_fct)(void*), size_t size); -void hashmap_add(struct hashmap *h, void *data, const void *key); -void *hashmap_lookup(struct hashmap *h, const void *key); -void *hashmap_iter_in_order(struct hashmap *h, struct hashmap_entry **it); -void hashmap_del(struct hashmap *h, struct hashmap_entry *e); -void hashmap_free(struct hashmap *h); - -uint32_t djb2_hash(const void *str); - -#endif /* !HASHMAP_H */ diff --git a/contrib/android/perms.c b/contrib/android/perms.c index 9ae8e586..d83ad35a 100644 --- a/contrib/android/perms.c +++ b/contrib/android/perms.c @@ -131,7 +131,7 @@ static errcode_t set_perms_and_caps(ext2_filsys fs, ext2_ino_t ino, if (!capabilities) return 0; memset(&cap_data, 0, sizeof(cap_data)); - cap_data.magic_etc = VFS_CAP_REVISION | VFS_CAP_FLAGS_EFFECTIVE; + cap_data.magic_etc = VFS_CAP_REVISION_2 | VFS_CAP_FLAGS_EFFECTIVE; cap_data.data[0].permitted = (uint32_t) (capabilities & 0xffffffff); cap_data.data[1].permitted = (uint32_t) (capabilities >> 32); return ino_add_xattr(fs, ino, "security." XATTR_CAPS_SUFFIX, @@ -288,8 +288,8 @@ errcode_t __android_configure_fs(ext2_filsys fs, char *src_dir, errcode_t android_configure_fs(ext2_filsys fs, char *src_dir, char *target_out, char *mountpoint, - struct selinux_opt *seopts, - unsigned int nopt, + struct selinux_opt *seopts EXT2FS_ATTR((unused)), + unsigned int nopt EXT2FS_ATTR((unused)), char *fs_config_file, time_t fixed_time) { errcode_t retval; diff --git a/debugfs/Android.bp b/debugfs/Android.bp index adaa6354..2a7d6222 100644 --- a/debugfs/Android.bp +++ b/debugfs/Android.bp @@ -5,6 +5,7 @@ cc_defaults { name: "debugfs-defaults", + defaults: ["e2fsprogs-defaults"], srcs: [ "debug_cmds.c", "debugfs.c", @@ -31,9 +32,7 @@ cc_defaults { "do_journal.c", ], cflags: [ - "-W", - "-Wall", - "-Wno-macro-redefined", + "-Wno-unused-variable", "-fno-strict-aliasing", "-DDEBUGFS", ], diff --git a/e2fsck/Android.bp b/e2fsck/Android.bp index 3eb86595..475cea12 100644 --- a/e2fsck/Android.bp +++ b/e2fsck/Android.bp @@ -5,6 +5,7 @@ cc_defaults { name: "e2fsck-defaults", + defaults: ["e2fsprogs-defaults"], srcs: [ "e2fsck.c", "super.c", @@ -34,7 +35,10 @@ cc_defaults { "readahead.c", "extents.c", ], - cflags: ["-W", "-Wall", "-Wno-macro-redefined", "-fno-strict-aliasing"], + cflags: [ + "-Wno-sign-compare", + "-fno-strict-aliasing" + ], } e2fsck_libs = [ diff --git a/include/mingw/linux/types.h b/include/mingw/linux/types.h index eb870113..96314566 100644 --- a/include/mingw/linux/types.h +++ b/include/mingw/linux/types.h @@ -10,9 +10,6 @@ typedef unsigned __int8 __u8; typedef signed __int8 __s8; -typedef signed __int64 __s64; -typedef unsigned __int64 __u64; - typedef signed __int16 __s16; typedef unsigned __int16 __u16; diff --git a/lib/blkid/Android.bp b/lib/blkid/Android.bp index 53381df8..225716cd 100644 --- a/lib/blkid/Android.bp +++ b/lib/blkid/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_blkid", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "cache.c", "dev.c", @@ -25,14 +26,19 @@ cc_library { include_dirs: [ "external/e2fsprogs/include/mingw" ], cflags: [ "-Wno-pointer-to-int-cast", + "-Wno-unused-variable", + ], + clang_cflags: [ + "-Wno-error=typedef-redefinition", ], enabled: true }, }, cflags: [ - "-W", - "-Wall", + "-Wno-error=attributes", + "-Wno-error=pointer-sign", + "-Wno-unused-parameter", "-fno-strict-aliasing", ], diff --git a/lib/blkid/blkid_types.h b/lib/blkid/blkid_types.h index 9c3f7149..a43ed2fe 100644 --- a/lib/blkid/blkid_types.h +++ b/lib/blkid/blkid_types.h @@ -1,4 +1,4 @@ -/* +/* * If linux/types.h is already been included, assume it has defined * everything we need. (cross fingers) Other header files may have * also defined the types that we need. @@ -27,7 +27,9 @@ typedef __signed__ long long __s64; #define __bitwise __attribute__((bitwise)) #define __force __attribute__((force)) #else +#ifndef __bitwise #define __bitwise +#endif #define __force #endif diff --git a/lib/blkid/probe.c b/lib/blkid/probe.c index 3bcf4c3d..0293b903 100644 --- a/lib/blkid/probe.c +++ b/lib/blkid/probe.c @@ -41,7 +41,7 @@ static int figure_label_len(const unsigned char *label, int len) { const unsigned char *end = label + len - 1; - while ((*end == ' ' || *end == 0) && end >= label) + while (end >= label && (*end == ' ' || *end == 0)) --end; if (end >= label) return end - label + 1; @@ -1160,6 +1160,37 @@ static void unicode_16be_to_utf8(unsigned char *str, int out_len, str[j] = '\0'; } +static void unicode_16le_to_utf8(unsigned char *str, int out_len, + const unsigned char *buf, int in_len) +{ + int i, j; + unsigned int c; + + for (i = j = 0; i + 2 <= in_len; i += 2) { + c = (buf[i+1] << 8) | buf[i]; + if (c == 0) { + str[j] = '\0'; + break; + } else if (c < 0x80) { + if (j+1 >= out_len) + break; + str[j++] = (unsigned char) c; + } else if (c < 0x800) { + if (j+2 >= out_len) + break; + str[j++] = (unsigned char) (0xc0 | (c >> 6)); + str[j++] = (unsigned char) (0x80 | (c & 0x3f)); + } else { + if (j+3 >= out_len) + break; + str[j++] = (unsigned char) (0xe0 | (c >> 12)); + str[j++] = (unsigned char) (0x80 | ((c >> 6) & 0x3f)); + str[j++] = (unsigned char) (0x80 | (c & 0x3f)); + } + } + str[j] = '\0'; +} + static int probe_hfs(struct blkid_probe *probe __BLKID_ATTR((unused)), struct blkid_magic *id __BLKID_ATTR((unused)), unsigned char *buf) @@ -1482,7 +1513,9 @@ static int probe_exfat(struct blkid_probe *probe, struct blkid_magic *id, label = find_exfat_entry_label(probe, sb); if (label) { - blkid_set_tag(probe->dev, "LABEL", label->name, label->length); + char utf8_label[128]; + unicode_16le_to_utf8(utf8_label, sizeof(utf8_label), label->name, label->length * 2); + blkid_set_tag(probe->dev, "LABEL", utf8_label, 0); } else { blkid_set_tag(probe->dev, "LABEL", "disk", 4); } diff --git a/lib/e2p/Android.bp b/lib/e2p/Android.bp index 9d5b0b61..21089d7b 100644 --- a/lib/e2p/Android.bp +++ b/lib/e2p/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_e2p", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "feature.c", "fgetflags.c", @@ -29,16 +30,19 @@ cc_library { "percent.c", ], - cflags: [ - "-W", - "-Wall", - "-Wno-macro-redefined", - ], + cflags: ["-Wno-error=attributes"], target: { windows: { include_dirs: [ "external/e2fsprogs/include/mingw" ], - enabled: true + enabled: true, + cflags: [ + "-Wno-unused-variable", + "-Wno-unused-parameter", + ], + clang_cflags: [ + "-Wno-error=typedef-redefinition", + ], }, }, diff --git a/lib/e2p/feature.c b/lib/e2p/feature.c index b7f6c1d2..0fab9c74 100644 --- a/lib/e2p/feature.c +++ b/lib/e2p/feature.c @@ -72,6 +72,8 @@ static struct feature feature_list[] = { "read-only" }, { E2P_FEATURE_RO_INCOMPAT, EXT4_FEATURE_RO_COMPAT_PROJECT, "project"}, + { E2P_FEATURE_RO_INCOMPAT, EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS, + "shared_blocks"}, { E2P_FEATURE_INCOMPAT, EXT2_FEATURE_INCOMPAT_COMPRESSION, "compression" }, diff --git a/lib/et/Android.bp b/lib/et/Android.bp index f6ad416b..670d8ee4 100644 --- a/lib/et/Android.bp +++ b/lib/et/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_com_err", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "error_message.c", "et_name.c", @@ -12,13 +13,12 @@ cc_library { "com_right.c", ], - cflags: [ - "-W", - "-Wall", - ], target: { windows: { enabled: true, + cflags: [ + "-Wno-unused-variable", + ], }, }, diff --git a/lib/ext2fs/Android.bp b/lib/ext2fs/Android.bp index cc28de17..06a750eb 100644 --- a/lib/ext2fs/Android.bp +++ b/lib/ext2fs/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2fs", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "ext2_err.c", "alloc.c", @@ -46,6 +47,7 @@ cc_library { "get_pathname.c", "getsize.c", "getsectsize.c", + "hashmap.c", "i_block.c", "icount.c", "imager.c", @@ -89,19 +91,14 @@ cc_library { ], shared_libs: [ "libsparse", + "libz", ], whole_static_libs: [ "libext2_com_err" ], - cflags: [ - "-W", - "-Wall", - "-Wno-unused-parameter", - "-Wno-macro-redefined", - ], + cflags: ["-Wno-unused-parameter"], target: { host: { - shared_libs: ["libz-host"], // Consider removing this library as a whole for the host. It is not // in the android side. whole_static_libs: ["libext2_com_err"], @@ -110,14 +107,18 @@ cc_library { shared_libs: [ "libext2_com_err", "libext2_uuid", - "libz" ], }, windows: { enabled: true, include_dirs: [ "external/e2fsprogs/include/mingw" ], cflags: [ + "-Wno-error=cpp", "-Wno-format", + "-Wno-unused-variable", + ], + clang_cflags: [ + "-Wno-error=typedef-redefinition", ], host_ldlibs: ["-lws2_32"], }, diff --git a/lib/ext2fs/ext2_fs.h b/lib/ext2fs/ext2_fs.h index 27a7d3a7..ab1db6e2 100644 --- a/lib/ext2fs/ext2_fs.h +++ b/lib/ext2fs/ext2_fs.h @@ -812,6 +812,7 @@ struct ext2_super_block { #define EXT4_FEATURE_RO_COMPAT_REPLICA 0x0800 #define EXT4_FEATURE_RO_COMPAT_READONLY 0x1000 #define EXT4_FEATURE_RO_COMPAT_PROJECT 0x2000 /* Project quota */ +#define EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS 0x4000 #define EXT2_FEATURE_INCOMPAT_COMPRESSION 0x0001 @@ -904,6 +905,7 @@ EXT4_FEATURE_RO_COMPAT_FUNCS(metadata_csum, 4, METADATA_CSUM) EXT4_FEATURE_RO_COMPAT_FUNCS(replica, 4, REPLICA) EXT4_FEATURE_RO_COMPAT_FUNCS(readonly, 4, READONLY) EXT4_FEATURE_RO_COMPAT_FUNCS(project, 4, PROJECT) +EXT4_FEATURE_RO_COMPAT_FUNCS(shared_blocks, 4, SHARED_BLOCKS) EXT4_FEATURE_INCOMPAT_FUNCS(compression, 2, COMPRESSION) EXT4_FEATURE_INCOMPAT_FUNCS(filetype, 2, FILETYPE) diff --git a/lib/ext2fs/ext2_types.h b/lib/ext2fs/ext2_types.h index 9c3f7149..a43ed2fe 100644 --- a/lib/ext2fs/ext2_types.h +++ b/lib/ext2fs/ext2_types.h @@ -1,4 +1,4 @@ -/* +/* * If linux/types.h is already been included, assume it has defined * everything we need. (cross fingers) Other header files may have * also defined the types that we need. @@ -27,7 +27,9 @@ typedef __signed__ long long __s64; #define __bitwise __attribute__((bitwise)) #define __force __attribute__((force)) #else +#ifndef __bitwise #define __bitwise +#endif #define __force #endif diff --git a/lib/ext2fs/ext2_types.h.in b/lib/ext2fs/ext2_types.h.in index 07adc64a..e815ba65 100644 --- a/lib/ext2fs/ext2_types.h.in +++ b/lib/ext2fs/ext2_types.h.in @@ -1,6 +1,6 @@ -/* +/* * If linux/types.h is already been included, assume it has defined - * everything we need. (cross fingers) Other header files may have + * everything we need. (cross fingers) Other header files may have * also defined the types that we need. */ #if (!defined(_LINUX_TYPES_H) && !defined(_BLKID_TYPES_H) && \ @@ -176,7 +176,9 @@ typedef long __s64; #define __bitwise __attribute__((bitwise)) #define __force __attribute__((force)) #else +#ifndef __bitwise #define __bitwise +#endif #define __force #endif diff --git a/lib/ext2fs/ext2fs.h b/lib/ext2fs/ext2fs.h index e153c817..470e7d7a 100644 --- a/lib/ext2fs/ext2fs.h +++ b/lib/ext2fs/ext2fs.h @@ -94,6 +94,8 @@ typedef __u32 __bitwise ext2_dirhash_t; #include <ext2fs/ext2_ext_attr.h> #endif +#include "hashmap.h" + /* * Portability help for Microsoft Visual C++ */ @@ -195,6 +197,7 @@ typedef struct ext2_file *ext2_file_t; #define EXT2_FLAG_DIRECT_IO 0x80000 #define EXT2_FLAG_SKIP_MMP 0x100000 #define EXT2_FLAG_IGNORE_CSUM_ERRORS 0x200000 +#define EXT2_FLAG_SHARE_DUP 0x400000 /* * Special flag in the ext2 inode i_flag field that means that this is @@ -295,6 +298,9 @@ struct struct_ext2_filsys { blk64_t len, blk64_t *pblk, blk64_t *plen); void (*block_alloc_stats_range)(ext2_filsys fs, blk64_t blk, blk_t num, int inuse); + + /* hashmap for SHA of data blocks */ + struct ext2fs_hashmap* block_sha_map; }; #if EXT2_FLAT_INCLUDES @@ -614,7 +620,8 @@ typedef struct ext2_icount *ext2_icount_t; EXT4_FEATURE_RO_COMPAT_QUOTA|\ EXT4_FEATURE_RO_COMPAT_METADATA_CSUM|\ EXT4_FEATURE_RO_COMPAT_READONLY |\ - EXT4_FEATURE_RO_COMPAT_PROJECT) + EXT4_FEATURE_RO_COMPAT_PROJECT |\ + EXT4_FEATURE_RO_COMPAT_SHARED_BLOCKS) /* * These features are only allowed if EXT2_FLAG_SOFTSUPP_FEATURES is passed diff --git a/lib/ext2fs/fileio.c b/lib/ext2fs/fileio.c index 810a7fd8..5bc02d03 100644 --- a/lib/ext2fs/fileio.c +++ b/lib/ext2fs/fileio.c @@ -32,6 +32,12 @@ struct ext2_file { char *buf; }; +struct block_entry { + blk64_t physblock; + unsigned char sha[EXT2FS_SHA512_LENGTH]; +}; +typedef struct block_entry *block_entry_t; + #define BMAP_BUFFER (file->buf + fs->blocksize) errcode_t ext2fs_file_open2(ext2_filsys fs, ext2_ino_t ino, @@ -389,6 +395,8 @@ errcode_t ext2fs_file_write(ext2_file_t file, const void *buf, errcode_t retval = 0; unsigned int start, c, count = 0; const char *ptr = (const char *) buf; + block_entry_t new_block = NULL, old_block = NULL; + int bmap_flags = 0; EXT2_CHECK_MAGIC(file, EXT2_ET_MAGIC_EXT2_FILE); fs = file->fs; @@ -424,22 +432,51 @@ errcode_t ext2fs_file_write(ext2_file_t file, const void *buf, if (retval) goto fail; + file->flags |= EXT2_FILE_BUF_DIRTY; + memcpy(file->buf+start, ptr, c); + /* * OK, the physical block hasn't been allocated yet. * Allocate it. */ if (!file->physblock) { + bmap_flags = (file->ino ? BMAP_ALLOC : 0); + if (fs->flags & EXT2_FLAG_SHARE_DUP) { + new_block = calloc(1, sizeof(*new_block)); + if (!new_block) { + retval = EXT2_ET_NO_MEMORY; + goto fail; + } + ext2fs_sha512((const unsigned char*)file->buf, + fs->blocksize, new_block->sha); + old_block = ext2fs_hashmap_lookup( + fs->block_sha_map, + new_block->sha, + sizeof(new_block->sha)); + } + + if (old_block) { + file->physblock = old_block->physblock; + bmap_flags |= BMAP_SET; + free(new_block); + new_block = NULL; + } + retval = ext2fs_bmap2(fs, file->ino, &file->inode, BMAP_BUFFER, - file->ino ? BMAP_ALLOC : 0, + bmap_flags, file->blockno, 0, &file->physblock); if (retval) goto fail; + + if (new_block) { + new_block->physblock = file->physblock; + ext2fs_hashmap_add(fs->block_sha_map, new_block, + new_block->sha, sizeof(new_block->sha)); + } } - file->flags |= EXT2_FILE_BUF_DIRTY; - memcpy(file->buf+start, ptr, c); file->pos += c; ptr += c; count += c; diff --git a/lib/ext2fs/freefs.c b/lib/ext2fs/freefs.c index ea9742ef..68b8e9a4 100644 --- a/lib/ext2fs/freefs.c +++ b/lib/ext2fs/freefs.c @@ -17,6 +17,7 @@ #include "ext2_fs.h" #include "ext2fsP.h" +#include "hashmap.h" void ext2fs_free(ext2_filsys fs) { @@ -59,6 +60,9 @@ void ext2fs_free(ext2_filsys fs) if (fs->mmp_cmp) ext2fs_free_mem(&fs->mmp_cmp); + if (fs->block_sha_map) + ext2fs_hashmap_free(fs->block_sha_map); + fs->magic = 0; ext2fs_zero_blocks2(NULL, 0, 0, NULL, NULL); diff --git a/lib/ext2fs/hashmap.c b/lib/ext2fs/hashmap.c new file mode 100644 index 00000000..ade5d89e --- /dev/null +++ b/lib/ext2fs/hashmap.c @@ -0,0 +1,83 @@ +#include "hashmap.h" +#include <string.h> + +uint32_t ext2fs_djb2_hash(const void *str, size_t size) +{ + int c; + const char *s = str; + uint32_t hash = 5381; + + while (size-- > 0) { + c = *s++; + hash = ((hash << 5) + hash) + c; + } + return hash; +} + +struct ext2fs_hashmap *ext2fs_hashmap_create( + uint32_t(*hash_fct)(const void*, size_t), + void(*free_fct)(void*), size_t size) +{ + struct ext2fs_hashmap *h = calloc(sizeof(struct ext2fs_hashmap) + + sizeof(struct ext2fs_hashmap_entry) * size, 1); + h->size = size; + h->free = free_fct; + h->hash = hash_fct; + h->first = h->last = NULL; + return h; +} + +void ext2fs_hashmap_add(struct ext2fs_hashmap *h, void *data, const void *key, + size_t key_len) +{ + uint32_t hash = h->hash(key, key_len) % h->size; + struct ext2fs_hashmap_entry *e = malloc(sizeof(*e)); + + e->data = data; + e->key = key; + e->key_len = key_len; + e->next = h->entries[hash]; + h->entries[hash] = e; + + e->list_prev = NULL; + e->list_next = h->first; + if (h->first) + h->first->list_prev = e; + h->first = e; + if (!h->last) + h->last = e; +} + +void *ext2fs_hashmap_lookup(struct ext2fs_hashmap *h, const void *key, + size_t key_len) +{ + struct ext2fs_hashmap_entry *iter; + uint32_t hash = h->hash(key, key_len) % h->size; + + for (iter = h->entries[hash]; iter; iter = iter->next) + if (iter->key_len == key_len && !memcmp(iter->key, key, key_len)) + return iter->data; + return NULL; +} + +void *ext2fs_hashmap_iter_in_order(struct ext2fs_hashmap *h, + struct ext2fs_hashmap_entry **it) +{ + *it = *it ? (*it)->list_next : h->first; + return *it ? (*it)->data : NULL; +} + +void ext2fs_hashmap_free(struct ext2fs_hashmap *h) +{ + for (size_t i = 0; i < h->size; ++i) { + struct ext2fs_hashmap_entry *it = h->entries[i]; + while (it) { + struct ext2fs_hashmap_entry *tmp = it->next; + if (h->free) + h->free(it->data); + free(it); + it = tmp; + } + } + free(h); +} diff --git a/lib/ext2fs/hashmap.h b/lib/ext2fs/hashmap.h new file mode 100644 index 00000000..71271866 --- /dev/null +++ b/lib/ext2fs/hashmap.h @@ -0,0 +1,38 @@ +#ifndef HASHMAP_H +# define HASHMAP_H + +# include <stdlib.h> +# include <stdint.h> + +struct ext2fs_hashmap { + uint32_t size; + uint32_t(*hash)(const void *key, size_t len); + void(*free)(void*); + struct ext2fs_hashmap_entry *first; + struct ext2fs_hashmap_entry *last; + struct ext2fs_hashmap_entry { + void *data; + const void *key; + size_t key_len; + struct ext2fs_hashmap_entry *next; + struct ext2fs_hashmap_entry *list_next; + struct ext2fs_hashmap_entry *list_prev; + } *entries[0]; +}; + +struct ext2fs_hashmap *ext2fs_hashmap_create( + uint32_t(*hash_fct)(const void*, size_t), + void(*free_fct)(void*), size_t size); +void ext2fs_hashmap_add(struct ext2fs_hashmap *h, void *data, const void *key, + size_t key_len); +void *ext2fs_hashmap_lookup(struct ext2fs_hashmap *h, const void *key, + size_t key_len); +void *ext2fs_hashmap_iter_in_order(struct ext2fs_hashmap *h, + struct ext2fs_hashmap_entry **it); +void ext2fs_hashmap_del(struct ext2fs_hashmap *h, + struct ext2fs_hashmap_entry *e); +void ext2fs_hashmap_free(struct ext2fs_hashmap *h); + +uint32_t ext2fs_djb2_hash(const void *str, size_t size); + +#endif /* !HASHMAP_H */ diff --git a/lib/ext2fs/openfs.c b/lib/ext2fs/openfs.c index 94d0ef1c..43d68ad3 100644 --- a/lib/ext2fs/openfs.c +++ b/lib/ext2fs/openfs.c @@ -94,6 +94,12 @@ errcode_t ext2fs_open(const char *name, int flags, int superblock, manager, ret_fs); } +static void block_sha_map_free_entry(void *data) +{ + free(data); + return; +} + /* * Note: if superblock is non-zero, block-size must also be non-zero. * Superblock and block_size can be zero to use the default size. @@ -474,6 +480,16 @@ errcode_t ext2fs_open2(const char *name, const char *io_options, } } + if (fs->flags & EXT2_FLAG_SHARE_DUP) { + fs->block_sha_map = ext2fs_hashmap_create(ext2fs_djb2_hash, + block_sha_map_free_entry, 4096); + if (!fs->block_sha_map) { + retval = EXT2_ET_NO_MEMORY; + goto cleanup; + } + ext2fs_set_feature_shared_blocks(fs->super); + } + fs->flags &= ~EXT2_FLAG_NOFREE_ON_ERROR; *ret_fs = fs; diff --git a/lib/ext2fs/sparse_io.c b/lib/ext2fs/sparse_io.c index a83baf1f..1b39b2ec 100644 --- a/lib/ext2fs/sparse_io.c +++ b/lib/ext2fs/sparse_io.c @@ -270,7 +270,8 @@ static errcode_t sparse_merge_blocks(struct sparse_map *sm, uint64_t start, buf = calloc(num, block_size); if (!buf) { - fprintf(stderr, "failed to alloc %lu\n", num * block_size); + fprintf(stderr, "failed to alloc %llu\n", + (unsigned long long)num * block_size); return EXT2_ET_NO_MEMORY; } diff --git a/lib/ss/Android.bp b/lib/ss/Android.bp index 06925d73..09332512 100644 --- a/lib/ss/Android.bp +++ b/lib/ss/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_ss", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "ss_err.c", "std_rqs.c", @@ -22,11 +23,6 @@ cc_library { "get_readline.c", ], shared_libs: ["libext2_com_err"], - cflags: [ - "-W", - "-Wall", - ], - header_libs: ["libext2-headers"], export_include_dirs: ["."], export_header_lib_headers: ["libext2-headers"], diff --git a/lib/support/Android.bp b/lib/support/Android.bp index 4a89b8f3..0ddd0793 100644 --- a/lib/support/Android.bp +++ b/lib/support/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_quota", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "dict.c", "mkquota.c", @@ -27,12 +28,6 @@ cc_library { }, }, - cflags: [ - "-W", - "-Wall", - "-Wno-macro-redefined", - ], - header_libs: ["libext2-headers"], export_include_dirs: ["."], export_header_lib_headers: ["libext2-headers"], @@ -42,15 +37,12 @@ cc_library_shared { name: "libext2_profile", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "prof_err.c", "profile.c", ], - cflags = [ - "-W", - "-Wall", - ], shared_libs: ["libext2_com_err"], header_libs: ["libext2-headers"], diff --git a/lib/uuid/Android.bp b/lib/uuid/Android.bp index b6664f99..15c12921 100644 --- a/lib/uuid/Android.bp +++ b/lib/uuid/Android.bp @@ -4,6 +4,7 @@ cc_library { name: "libext2_uuid", host_supported: true, unique_host_soname: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "clear.c", "compare.c", @@ -17,13 +18,13 @@ cc_library { "uuid_time.c", ], cflags: [ - "-W", - "-Wall", "-Wno-unused-function", "-Wno-unused-parameter", ], target: { windows: { + // Cannot suppress the _WIN32_WINNT redefined warning. + cflags: ["-Wno-error"], include_dirs: [ "external/e2fsprogs/include/mingw" ], enabled: true }, diff --git a/misc/Android.bp b/misc/Android.bp index e2c3d11b..e73150f8 100644 --- a/misc/Android.bp +++ b/misc/Android.bp @@ -5,18 +5,20 @@ cc_library { name: "libext2_misc", host_supported: true, + defaults: ["e2fsprogs-defaults"], target: { windows: { include_dirs: [ "external/e2fsprogs/include/mingw" ], - enabled: true + enabled: true, + cflags: ["-Wno-unused-variable"], }, }, srcs: [ "create_inode.c", ], - cflags: ["-W", "-Wall"], + cflags: ["-Wno-error=format-extra-args"], shared_libs: [ "libext2_quota", "libext2fs", @@ -31,6 +33,7 @@ cc_library { cc_binary { name: "mke2fs", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "mke2fs.c", @@ -38,10 +41,11 @@ cc_binary { "mk_hugefiles.c", "default_profile.c", ], - required: [ - "mke2fs.conf", + cflags: [ + "-Wno-error=format", + "-Wno-error=type-limits", + "-Wno-format-extra-args", ], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], target: { host: { static_libs: [ @@ -57,14 +61,29 @@ cc_binary { "libz", ], }, + not_windows: { + required: [ + "mke2fs.conf", + ], + }, windows: { include_dirs: [ "external/e2fsprogs/include/mingw" ], - cflags: ["-D_POSIX", "-D__USE_MINGW_ALARM"], + cflags: [ + "-D_POSIX", + "-D__USE_MINGW_ALARM", + // mke2fs.c has a warning from gcc which cannot be suppressed: + // passing argument 3 of 'ext2fs_get_device_size' from + // incompatible pointer type + "-Wno-error" + ], ldflags: ["-static"], host_ldlibs: ["-lws2_32"], enabled: true }, android: { + required: [ + "mke2fs.conf", + ], shared_libs: [ "libext2fs", "libext2_blkid", @@ -74,6 +93,7 @@ cc_binary { "libext2_com_err", "libext2_e2p", ], + symlinks: ["mkfs.ext2", "mkfs.ext3", "mkfs.ext4"], }, }, stl: "libc++_static", @@ -85,16 +105,12 @@ cc_binary { cc_defaults { name: "tune2fs-defaults", + defaults: ["e2fsprogs-defaults"], srcs: [ "tune2fs.c", "util.c", ], - cflags: [ - "-W", - "-Wall", - "-DNO_RECOVERY", - "-Wno-macro-redefined", - ], + cflags: ["-DNO_RECOVERY"], include_dirs: ["external/e2fsprogs/e2fsck"], } @@ -138,9 +154,9 @@ cc_library_static { cc_binary { name: "badblocks", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["badblocks.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_com_err", @@ -157,9 +173,9 @@ cc_binary { cc_binary { name: "chattr", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["chattr.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2_com_err", "libext2_e2p", @@ -173,7 +189,7 @@ cc_binary { cc_defaults { name: "lsattr-defaults", srcs: ["lsattr.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], + defaults: ["e2fsprogs-defaults"], } lsattr_libs = [ @@ -203,9 +219,9 @@ cc_binary { cc_binary { name: "blkid", + defaults: ["e2fsprogs-defaults"], srcs: ["blkid.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_blkid", @@ -221,9 +237,9 @@ cc_binary { cc_binary { name: "e4crypt", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["e4crypt.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_uuid", @@ -243,9 +259,9 @@ cc_binary { cc_binary { name: "e2image", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: ["e2image.c"], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_blkid", diff --git a/misc/Android.mk b/misc/Android.mk index d3bb8207..c9d07353 100644 --- a/misc/Android.mk +++ b/misc/Android.mk @@ -12,7 +12,7 @@ mke2fs_src_files := \ mke2fs_c_includes := \ external/e2fsprogs/e2fsck -mke2fs_cflags := -W -Wall -Wno-macro-redefined +mke2fs_cflags := -W -Wall -Werror -Wno-macro-redefined mke2fs_static_libraries := \ libext2_blkid \ diff --git a/misc/create_inode.c b/misc/create_inode.c index 94c29b8d..1373b46b 100644 --- a/misc/create_inode.c +++ b/misc/create_inode.c @@ -775,8 +775,11 @@ static errcode_t __populate_fs(ext2_filsys fs, ext2_ino_t parent_ino, cur_dir_path_len = target->path_len; retval = path_append(target, name); - if (retval) - return retval; + if (retval) { + com_err(__func__, retval, + "while appending %s", name); + goto out; + } if (fs_callbacks && fs_callbacks->create_new_inode) { retval = fs_callbacks->create_new_inode(fs, diff --git a/resize/Android.bp b/resize/Android.bp index dcdf6591..d55e24c2 100644 --- a/resize/Android.bp +++ b/resize/Android.bp @@ -3,6 +3,7 @@ cc_binary { name: "resize2fs", host_supported: true, + defaults: ["e2fsprogs-defaults"], srcs: [ "extent.c", @@ -12,7 +13,6 @@ cc_binary { "sim_progress.c", "resource_track.c", ], - cflags: ["-W", "-Wall", "-Wno-macro-redefined"], shared_libs: [ "libext2fs", "libext2_com_err", |