5 files changed, 366 insertions, 1 deletions

diff --git a/meta-oe/classes/image_types_sparse.bbclass b/meta-oe/classes/image_types_sparse.bbclass
index 355a3dafc2..69e24cbb79 100644
--- a/meta-oe/classes/image_types_sparse.bbclass
+++ b/meta-oe/classes/image_types_sparse.bbclass
@@ -11,6 +11,6 @@ CONVERSIONTYPES += "sparse"
 CONVERSION_CMD:sparse() {
     INPUT="${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}"
     truncate --no-create --size=%${SPARSE_BLOCK_SIZE} "$INPUT"
-    img2simg "$INPUT" "$INPUT.sparse" ${SPARSE_BLOCK_SIZE}
+    img2simg -s "$INPUT" "$INPUT.sparse" ${SPARSE_BLOCK_SIZE}
 }
 CONVERSION_DEPENDS_sparse = "android-tools-native"
diff --git a/meta-oe/recipes-devtools/android-tools/android-tools/core/0015-libsparse-Split-off-most-of-sparse_file_read_normal-.patch b/meta-oe/recipes-devtools/android-tools/android-tools/core/0015-libsparse-Split-off-most-of-sparse_file_read_normal-.patch
new file mode 100644
index 0000000000..b3893f0c80
--- /dev/null
+++ b/meta-oe/recipes-devtools/android-tools/android-tools/core/0015-libsparse-Split-off-most-of-sparse_file_read_normal-.patch
@@ -0,0 +1,60 @@
+From 7b74d23ed955206a789a96bdc3288593e702afac Mon Sep 17 00:00:00 2001
+From: Sean Anderson <sean.anderson@seco.com>
+Date: Thu, 30 Dec 2021 15:16:08 -0500
+Subject: [PATCH] libsparse: Split off most of sparse_file_read_normal into a
+ helper function
+
+This carves out the core of sparse_file_read_normal and splits it off so
+it can be reused in the next patch. No functional change intended.
+
+Change-Id: Id00491fd7e5bb6fa28c517a0bb32b8b506539d4d
+Upstream-status: Backport [95657f3e5976d96073f7bbfe3a49192509999d1d]
+Signed-off-by: Sean Anderson <sean.anderson@seco.com>
+---
+ libsparse/sparse_read.c | 21 ++++++++++++++++-----
+ 1 file changed, 16 insertions(+), 5 deletions(-)
+
+diff --git a/libsparse/sparse_read.c b/libsparse/sparse_read.c
+index 8e188e9a4..ee4abd86a 100644
+--- a/libsparse/sparse_read.c
++++ b/libsparse/sparse_read.c
+@@ -353,13 +353,11 @@ static int sparse_file_read_sparse(struct sparse_file *s, int fd, bool crc)
+        return 0;
+ }
+ 
+-static int sparse_file_read_normal(struct sparse_file *s, int fd)
++static int do_sparse_file_read_normal(struct sparse_file *s, int fd, uint32_t* buf, int64_t offset,
++                                     int64_t remain)
+ {
+        int ret;
+-       uint32_t *buf = malloc(s->block_size);
+-       unsigned int block = 0;
+-       int64_t remain = s->len;
+-       int64_t offset = 0;
++       unsigned int block = offset / s->block_size;
+        unsigned int to_read;
+        unsigned int i;
+        bool sparse_block;
+@@ -403,6 +401,19 @@ static int sparse_file_read_normal(struct sparse_file *s, int fd)
+        return 0;
+ }
+ 
++static int sparse_file_read_normal(struct sparse_file* s, int fd)
++{
++       int ret;
++       uint32_t* buf = (uint32_t*)malloc(s->block_size);
++
++       if (!buf)
++               return -ENOMEM;
++
++       ret = do_sparse_file_read_normal(s, fd, buf, 0, s->len);
++       free(buf);
++       return ret;
++}
++
+ int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
+ {
+        if (crc && !sparse) {
+-- 
+2.35.1.1320.gc452695387.dirty
+
diff --git a/meta-oe/recipes-devtools/android-tools/android-tools/core/0016-libsparse-Add-hole-mode-to-sparse_file_read.patch b/meta-oe/recipes-devtools/android-tools/android-tools/core/0016-libsparse-Add-hole-mode-to-sparse_file_read.patch
new file mode 100644
index 0000000000..e5221d2b4c
--- /dev/null
+++ b/meta-oe/recipes-devtools/android-tools/android-tools/core/0016-libsparse-Add-hole-mode-to-sparse_file_read.patch
@@ -0,0 +1,188 @@
+From 41574b628ec4229c24dfe289af7b6978edcca4ed Mon Sep 17 00:00:00 2001
+From: Sean Anderson <sean.anderson@seco.com>
+Date: Thu, 30 Dec 2021 15:19:41 -0500
+Subject: [PATCH] libsparse: Add "hole" mode to sparse_file_read
+
+This adds support for filesystem-level sparse files. These files have
+holes which are not stored in the filesystem and when read are full of
+zeros. While these zeros may be significant in some types of files,
+other types of files may not care about the contents of holes. For
+example, most filesystem creation tools write to all the blocks they
+care about. Those blocks not written to will remain holes, and can be
+safely represented by "don't care" chunks. Using "don't care" chunks
+instead of fill chunks can result in a substantial reduction of the time
+it takes to program a sparse image.
+
+To accomplish this, we extend the existing "sparse" boolean parameter to
+be an enum of mode types. This enum represents the strategy we take when
+reading in a file. For the most part the implementation is
+straightforward. We use lseek to determine where the holes in the file
+are, and then use do_sparse_file_read_normal to create chunks for the
+data section. Note that every file has an implicit hole at its end.
+
+Change-Id: I0cfbf08886fca9a91cb753ec8734c84fcbe52c9f
+Upstream-Status: Backport [f96466b05543b984ef7315d830bab4a409228d35]
+Signed-off-by: Sean Anderson <sean.anderson@seco.com>
+---
+ libsparse/img2simg.c              |  2 +-
+ libsparse/include/sparse/sparse.h | 32 +++++++++++---
+ libsparse/sparse_read.c           | 71 +++++++++++++++++++++++++++++--
+ 3 files changed, 93 insertions(+), 12 deletions(-)
+
+diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c
+index a0db36f45..2e171b613 100644
+--- a/libsparse/img2simg.c
++++ b/libsparse/img2simg.c
+@@ -96,7 +96,7 @@ int main(int argc, char *argv[])
+        }
+ 
+        sparse_file_verbose(s);
+-       ret = sparse_file_read(s, in, false, false);
++       ret = sparse_file_read(s, in, SPARSE_READ_MODE_NORMAL, false);
+        if (ret) {
+                fprintf(stderr, "Failed to read file\n");
+                exit(-1);
+diff --git a/libsparse/include/sparse/sparse.h b/libsparse/include/sparse/sparse.h
+index 8b757d22a..b68aa21a8 100644
+--- a/libsparse/include/sparse/sparse.h
++++ b/libsparse/include/sparse/sparse.h
+@@ -196,23 +196,41 @@ int64_t sparse_file_len(struct sparse_file *s, bool sparse, bool crc);
+ int sparse_file_callback(struct sparse_file *s, bool sparse, bool crc,
+                int (*write)(void *priv, const void *data, int len), void *priv);
+ 
++/**
++ * enum sparse_read_mode - The method to use when reading in files
++ * @SPARSE_READ_MODE_NORMAL: The input is a regular file. Constant chunks of
++ *                           data (including holes) will be be converted to
++ *                           fill chunks.
++ * @SPARSE_READ_MODE_SPARSE: The input is an Android sparse file.
++ * @SPARSE_READ_MODE_HOLE: The input is a regular file. Holes will be converted
++ *                         to "don't care" chunks. Other constant chunks will
++ *                         be converted to fill chunks.
++ */
++enum sparse_read_mode {
++       SPARSE_READ_MODE_NORMAL = false,
++       SPARSE_READ_MODE_SPARSE = true,
++       SPARSE_READ_MODE_HOLE,
++};
++
+ /**
+  * sparse_file_read - read a file into a sparse file cookie
+  *
+  * @s - sparse file cookie
+  * @fd - file descriptor to read from
+- * @sparse - read a file in the Android sparse file format
++ * @mode - mode to use when reading the input file
+  * @crc - verify the crc of a file in the Android sparse file format
+  *
+- * Reads a file into a sparse file cookie.  If sparse is true, the file is
+- * assumed to be in the Android sparse file format.  If sparse is false, the
+- * file will be sparsed by looking for block aligned chunks of all zeros or
+- * another 32 bit value.  If crc is true, the crc of the sparse file will be
+- * verified.
++ * Reads a file into a sparse file cookie. If @mode is
++ * %SPARSE_READ_MODE_SPARSE, the file is assumed to be in the Android sparse
++ * file format. If @mode is %SPARSE_READ_MODE_NORMAL, the file will be sparsed
++ * by looking for block aligned chunks of all zeros or another 32 bit value. If
++ * @mode is %SPARSE_READ_MODE_HOLE, the file will be sparsed like
++ * %SPARSE_READ_MODE_NORMAL, but holes in the file will be converted to "don't
++ * care" chunks. If crc is true, the crc of the sparse file will be verified.
+  *
+  * Returns 0 on success, negative errno on error.
+  */
+-int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc);
++int sparse_file_read(struct sparse_file *s, int fd, enum sparse_read_mode mode, bool crc);
+ 
+ /**
+  * sparse_file_import - import an existing sparse file
+diff --git a/libsparse/sparse_read.c b/libsparse/sparse_read.c
+index ee4abd86a..81f48cc13 100644
+--- a/libsparse/sparse_read.c
++++ b/libsparse/sparse_read.c
+@@ -414,16 +414,79 @@ static int sparse_file_read_normal(struct sparse_file* s, int fd)
+        return ret;
+ }
+ 
+-int sparse_file_read(struct sparse_file *s, int fd, bool sparse, bool crc)
++#ifdef __linux__
++static int sparse_file_read_hole(struct sparse_file* s, int fd)
+ {
+-       if (crc && !sparse) {
++       int ret;
++       uint32_t* buf = (uint32_t*)malloc(s->block_size);
++       int64_t end = 0;
++       int64_t start = 0;
++
++       if (!buf) {
++               return -ENOMEM;
++       }
++
++       do {
++               start = lseek(fd, end, SEEK_DATA);
++               if (start < 0) {
++                       if (errno == ENXIO)
++                               /* The rest of the file is a hole */
++                               break;
++
++                       error("could not seek to data");
++                       free(buf);
++                       return -errno;
++               } else if (start > s->len) {
++                       break;
++               }
++
++               end = lseek(fd, start, SEEK_HOLE);
++               if (end < 0) {
++                       error("could not seek to end");
++                       free(buf);
++                       return -errno;
++               }
++               end = min(end, s->len);
++
++               start = ALIGN_DOWN(start, s->block_size);
++               end = ALIGN(end, s->block_size);
++               if (lseek(fd, start, SEEK_SET) < 0) {
++                       free(buf);
++                       return -errno;
++               }
++
++               ret = do_sparse_file_read_normal(s, fd, buf, start, end - start);
++               if (ret) {
++                       free(buf);
++                       return ret;
++               }
++       } while (end < s->len);
++
++       free(buf);
++       return 0;
++}
++#else
++static int sparse_file_read_hole(struct sparse_file* s __unused, int fd __unused)
++{
++       return -ENOTSUP;
++}
++#endif
++
++int sparse_file_read(struct sparse_file *s, int fd, enum sparse_read_mode mode, bool crc)
++{
++       if (crc && mode != SPARSE_READ_MODE_SPARSE) {
+                return -EINVAL;
+        }
+ 
+-       if (sparse) {
++       switch (mode) {
++       case SPARSE_READ_MODE_SPARSE:
+                return sparse_file_read_sparse(s, fd, crc);
+-       } else {
++       case SPARSE_READ_MODE_NORMAL:
+                return sparse_file_read_normal(s, fd);
++       case SPARSE_READ_MODE_HOLE:
++               return sparse_file_read_hole(s, fd);
++       default:
++               return -EINVAL;
+        }
+ }
+ 
+-- 
+2.35.1.1320.gc452695387.dirty
+
diff --git a/meta-oe/recipes-devtools/android-tools/android-tools/core/0017-img2simg-Add-support-for-converting-holes-to-don-t-c.patch b/meta-oe/recipes-devtools/android-tools/android-tools/core/0017-img2simg-Add-support-for-converting-holes-to-don-t-c.patch
new file mode 100644
index 0000000000..9d19f58095
--- /dev/null
+++ b/meta-oe/recipes-devtools/android-tools/android-tools/core/0017-img2simg-Add-support-for-converting-holes-to-don-t-c.patch
@@ -0,0 +1,114 @@
+From 00cce57eff1a0de3b93efa5da225e9eb33d19659 Mon Sep 17 00:00:00 2001
+From: Sean Anderson <sean.anderson@seco.com>
+Date: Thu, 30 Dec 2021 15:34:28 -0500
+Subject: [PATCH] img2simg: Add support for converting holes to "don't care"
+ chunks
+
+This adds support for converting files with holes to "don't care"
+chunks. This can result in a substantial reduction in the time it takes
+to program an image if it has many holes.
+
+Generally, constants compared to argc have been reduced by one, since we
+no longer have the program name as the first argument.
+
+Change-Id: I00750edc07d6415dcc07ae0351e9397b0222b7ba
+Upstream-Status: Backport [6150b00b6025918da8c28e5c2f929ecdf480a9d6]
+Signed-off-by: Sean Anderson <sean.anderson@seco.com>
+---
+ libsparse/img2simg.c | 41 ++++++++++++++++++++++++++++++-----------
+ 1 file changed, 30 insertions(+), 11 deletions(-)
+
+diff --git a/libsparse/img2simg.c b/libsparse/img2simg.c
+index 2e171b613..c985d5449 100644
+--- a/libsparse/img2simg.c
++++ b/libsparse/img2simg.c
+@@ -40,25 +40,42 @@
+ 
+ void usage()
+ {
+-    fprintf(stderr, "Usage: img2simg <raw_image_file> <sparse_image_file> [<block_size>]\n");
++    fprintf(stderr, "Usage: img2simg [-s] <raw_image_file> <sparse_image_file> [<block_size>]\n");
+ }
+ 
+ int main(int argc, char *argv[])
+ {
++       char *arg_in;
++       char *arg_out;
++       enum sparse_read_mode mode = SPARSE_READ_MODE_NORMAL;
++       int extra;
+        int in;
++       int opt;
+        int out;
+        int ret;
+        struct sparse_file *s;
+        unsigned int block_size = 4096;
+        off64_t len;
+ 
+-       if (argc < 3 || argc > 4) {
++       while ((opt = getopt(argc, argv, "s")) != -1) {
++               switch (opt) {
++               case 's':
++                       mode = SPARSE_READ_MODE_HOLE;
++                       break;
++               default:
++                       usage();
++                       exit(-1);
++               }
++       }
++
++       extra = argc - optind;
++       if (extra < 2 || extra > 3) {
+                usage();
+                exit(-1);
+        }
+ 
+-       if (argc == 4) {
+-               block_size = atoi(argv[3]);
++       if (extra == 3) {
++               block_size = atoi(argv[optind + 2]);
+        }
+ 
+        if (block_size < 1024 || block_size % 4 != 0) {
+@@ -66,22 +83,24 @@ int main(int argc, char *argv[])
+                exit(-1);
+        }
+ 
+-       if (strcmp(argv[1], "-") == 0) {
++       arg_in = argv[optind];
++       if (strcmp(arg_in, "-") == 0) {
+                in = STDIN_FILENO;
+        } else {
+-               in = open(argv[1], O_RDONLY | O_BINARY);
++               in = open(arg_in, O_RDONLY | O_BINARY);
+                if (in < 0) {
+-                       fprintf(stderr, "Cannot open input file %s\n", argv[1]);
++                       fprintf(stderr, "Cannot open input file %s\n", arg_in);
+                        exit(-1);
+                }
+        }
+ 
+-       if (strcmp(argv[2], "-") == 0) {
++       arg_out = argv[optind + 1];
++       if (strcmp(arg_out, "-") == 0) {
+                out = STDOUT_FILENO;
+        } else {
+-               out = open(argv[2], O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
++               out = open(arg_out, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0664);
+                if (out < 0) {
+-                       fprintf(stderr, "Cannot open output file %s\n", argv[2]);
++                       fprintf(stderr, "Cannot open output file %s\n", arg_out);
+                        exit(-1);
+                }
+        }
+@@ -96,7 +115,7 @@ int main(int argc, char *argv[])
+        }
+ 
+        sparse_file_verbose(s);
+-       ret = sparse_file_read(s, in, SPARSE_READ_MODE_NORMAL, false);
++       ret = sparse_file_read(s, in, mode, false);
+        if (ret) {
+                fprintf(stderr, "Failed to read file\n");
+                exit(-1);
+-- 
+2.35.1.1320.gc452695387.dirty
+
diff --git a/meta-oe/recipes-devtools/android-tools/android-tools_5.1.1.r37.bb b/meta-oe/recipes-devtools/android-tools/android-tools_5.1.1.r37.bb
index 8f28abbd19..26393605d4 100644
--- a/meta-oe/recipes-devtools/android-tools/android-tools_5.1.1.r37.bb
+++ b/meta-oe/recipes-devtools/android-tools/android-tools_5.1.1.r37.bb
@@ -41,6 +41,9 @@ SRC_URI = " \
     file://core/adb_libssl_11.diff;patchdir=system/core \
     file://core/0013-adb-Support-riscv64.patch;patchdir=system/core \
     file://core/0014-add-u3-ss-descriptor-support-for-adb.patch;patchdir=system/core \
+    file://core/0015-libsparse-Split-off-most-of-sparse_file_read_normal-.patch;patchdir=system/core \
+    file://core/0016-libsparse-Add-hole-mode-to-sparse_file_read.patch;patchdir=system/core \
+    file://core/0017-img2simg-Add-support-for-converting-holes-to-don-t-c.patch;patchdir=system/core \
     file://extras/0001-ext4_utils-remove-selinux-extensions.patch;patchdir=system/extras \
     file://extras/0002-ext4_utils-add-o-argument-to-preserve-ownership.patch;patchdir=system/extras \
     file://libselinux/0001-Remove-bionic-specific-calls.patch;patchdir=external/libselinux \