libjxl: fix CVE-2024-11403 & CVE-2024-11498

* CVE-2024-11403 - Upstream-Status: Backport from https://github.com/libjxl/libjxl/commit/9cc451b91b74ba470fd72bd48c121e9f33d24c99
* CVE-2024-11498 - Upstream-Status: Backport from https://github.com/libjxl/libjxl/commit/bf4781a2eed2eef664790170977d1d3d8347efb9

Signed-off-by: Hitendra Prajapati <hprajapati@mvista.com>
Signed-off-by: Anuj Mittal <anuj.mittal@intel.com>

3 files changed, 187 insertions, 2 deletions

diff --git a/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11403.patch b/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11403.patch
new file mode 100644
index 0000000000..625218c2d3
--- /dev/null
+++ b/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11403.patch
@@ -0,0 +1,70 @@
+From 9cc451b91b74ba470fd72bd48c121e9f33d24c99 Mon Sep 17 00:00:00 2001
+From: szabadka <9074039+szabadka@users.noreply.github.com>
+Date: Thu, 3 Oct 2024 18:07:38 +0200
+Subject: [PATCH] Port the Huffman lookup table size fix from brunsli. (#3871)
+
+CVE: CVE-2024-11403
+Upstream-Status: Backport [https://github.com/libjxl/libjxl/commit/9cc451b91b74ba470fd72bd48c121e9f33d24c99]
+Signed-off-by: Hitendra Prajapati <hprajapati@mvista.com>
+---
+ lib/jpegli/huffman.h                   | 16 ++++++++++++----
+ lib/jxl/jpeg/enc_jpeg_huffman_decode.h | 16 ++++++++++++----
+ 2 files changed, 24 insertions(+), 8 deletions(-)
+
+diff --git a/lib/jpegli/huffman.h b/lib/jpegli/huffman.h
+index f0e5e1de..99549668 100644
+--- a/lib/jpegli/huffman.h
++++ b/lib/jpegli/huffman.h
+@@ -15,10 +15,18 @@ namespace jpegli {
+ 
+ constexpr int kJpegHuffmanRootTableBits = 8;
+ // Maximum huffman lookup table size.
+-// According to zlib/examples/enough.c, 758 entries are always enough for
+-// an alphabet of 257 symbols (256 + 1 special symbol for the all 1s code) and
+-// max bit length 16 if the root table has 8 bits.
+-constexpr int kJpegHuffmanLutSize = 758;
++// Requirements: alphabet of 257 symbols (256 + 1 special symbol for the all 1s
++// code) and max bit length 16, the root table has 8 bits.
++// zlib/examples/enough.c works with an assumption that Huffman code is
++// "complete". Input JPEGs might have this assumption broken, hence the
++// following sum is used as estimate:
++//  + number of 1-st level cells
++//  + number of symbols
++//  + asymptotic amount of repeated 2nd level cells
++// The third number is 1 + 3 + ... + 255 i.e. it is assumed that sub-table of
++// each "size" might be almost completely be filled with repetitions.
++// Total sum is slightly less than 1024,...
++constexpr int kJpegHuffmanLutSize = 1024;
+ 
+ struct HuffmanTableEntry {
+   uint8_t bits;    // number of bits used for this symbol
+diff --git a/lib/jxl/jpeg/enc_jpeg_huffman_decode.h b/lib/jxl/jpeg/enc_jpeg_huffman_decode.h
+index b8a60e41..fc9bd17b 100644
+--- a/lib/jxl/jpeg/enc_jpeg_huffman_decode.h
++++ b/lib/jxl/jpeg/enc_jpeg_huffman_decode.h
+@@ -15,10 +15,18 @@ namespace jpeg {
+ 
+ constexpr int kJpegHuffmanRootTableBits = 8;
+ // Maximum huffman lookup table size.
+-// According to zlib/examples/enough.c, 758 entries are always enough for
+-// an alphabet of 257 symbols (256 + 1 special symbol for the all 1s code) and
+-// max bit length 16 if the root table has 8 bits.
+-constexpr int kJpegHuffmanLutSize = 758;
++// Requirements: alphabet of 257 symbols (256 + 1 special symbol for the all 1s
++// code) and max bit length 16, the root table has 8 bits.
++// zlib/examples/enough.c works with an assumption that Huffman code is
++// "complete". Input JPEGs might have this assumption broken, hence the
++// following sum is used as estimate:
++//  + number of 1-st level cells
++//  + number of symbols
++//  + asymptotic amount of repeated 2nd level cells
++// The third number is 1 + 3 + ... + 255 i.e. it is assumed that sub-table of
++// each "size" might be almost completely be filled with repetitions.
++// Total sum is slightly less than 1024,...
++constexpr int kJpegHuffmanLutSize = 1024;
+ 
+ struct HuffmanTableEntry {
+   // Initialize the value to an invalid symbol so that we can recognize it
+-- 
+2.50.1
+
diff --git a/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11498.patch b/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11498.patch
new file mode 100644
index 0000000000..25f85e1527
--- /dev/null
+++ b/meta-oe/recipes-multimedia/libjxl/libjxl/CVE-2024-11498.patch
@@ -0,0 +1,113 @@
+From bf4781a2eed2eef664790170977d1d3d8347efb9 Mon Sep 17 00:00:00 2001
+From: Luca Versari <veluca@google.com>
+Date: Thu, 21 Nov 2024 16:33:08 +0100
+Subject: [PATCH] Check height limit in modular trees. (#3943)
+
+Also rewrite the implementation to use iterative checking instead of
+recursive checking of tree property values, to ensure stack usage is
+low.
+
+Before, it was possible for appropriately-crafted files to use a
+significant amount of stack (in the order of hundreds of MB).
+
+CVE: CVE-2024-11498
+Upstream-Status: Backport [https://github.com/libjxl/libjxl/commit/bf4781a2eed2eef664790170977d1d3d8347efb9]
+Signed-off-by: Hitendra Prajapati <hprajapati@mvista.com>
+---
+ lib/jxl/modular/encoding/dec_ma.cc | 66 ++++++++++++++++++++----------
+ 1 file changed, 45 insertions(+), 21 deletions(-)
+
+diff --git a/lib/jxl/modular/encoding/dec_ma.cc b/lib/jxl/modular/encoding/dec_ma.cc
+index b53b9a91..df2948d8 100644
+--- a/lib/jxl/modular/encoding/dec_ma.cc
++++ b/lib/jxl/modular/encoding/dec_ma.cc
+@@ -6,6 +6,7 @@
+ #include "lib/jxl/modular/encoding/dec_ma.h"
+ 
+ #include <limits>
++#include <vector>
+ 
+ #include "lib/jxl/base/printf_macros.h"
+ #include "lib/jxl/dec_ans.h"
+@@ -17,23 +18,49 @@ namespace jxl {
+ 
+ namespace {
+ 
+-Status ValidateTree(
+-    const Tree &tree,
+-    const std::vector<std::pair<pixel_type, pixel_type>> &prop_bounds,
+-    size_t root) {
+-  if (tree[root].property == -1) return true;
+-  size_t p = tree[root].property;
+-  int val = tree[root].splitval;
+-  if (prop_bounds[p].first > val) return JXL_FAILURE("Invalid tree");
+-  // Splitting at max value makes no sense: left range will be exactly same
+-  // as parent, right range will be invalid (min > max).
+-  if (prop_bounds[p].second <= val) return JXL_FAILURE("Invalid tree");
+-  auto new_bounds = prop_bounds;
+-  new_bounds[p].first = val + 1;
+-  JXL_RETURN_IF_ERROR(ValidateTree(tree, new_bounds, tree[root].lchild));
+-  new_bounds[p] = prop_bounds[p];
+-  new_bounds[p].second = val;
+-  return ValidateTree(tree, new_bounds, tree[root].rchild);
++Status ValidateTree(const Tree &tree) {
++  int num_properties = 0;
++  for (auto node : tree) {
++    if (node.property >= num_properties) {
++      num_properties = node.property + 1;
++    }
++  }
++  std::vector<int> height(tree.size());
++  std::vector<std::pair<pixel_type, pixel_type>> property_ranges(
++      num_properties * tree.size());
++  for (int i = 0; i < num_properties; i++) {
++    property_ranges[i].first = std::numeric_limits<pixel_type>::min();
++    property_ranges[i].second = std::numeric_limits<pixel_type>::max();
++  }
++  const int kHeightLimit = 2048;
++  for (size_t i = 0; i < tree.size(); i++) {
++    if (height[i] > kHeightLimit) {
++      return JXL_FAILURE("Tree too tall: %d", height[i]);
++    }
++    if (tree[i].property == -1) continue;
++    height[tree[i].lchild] = height[i] + 1;
++    height[tree[i].rchild] = height[i] + 1;
++    for (size_t p = 0; p < static_cast<size_t>(num_properties); p++) {
++      if (p == static_cast<size_t>(tree[i].property)) {
++        pixel_type l = property_ranges[i * num_properties + p].first;
++        pixel_type u = property_ranges[i * num_properties + p].second;
++        pixel_type val = tree[i].splitval;
++        if (l > val || u <= val) {
++          return JXL_FAILURE("Invalid tree");
++        }
++        property_ranges[tree[i].lchild * num_properties + p] =
++            std::make_pair(val + 1, u);
++        property_ranges[tree[i].rchild * num_properties + p] =
++            std::make_pair(l, val);
++      } else {
++        property_ranges[tree[i].lchild * num_properties + p] =
++            property_ranges[i * num_properties + p];
++        property_ranges[tree[i].rchild * num_properties + p] =
++            property_ranges[i * num_properties + p];
++      }
++    }
++  }
++  return true;
+ }
+ 
+ Status DecodeTree(BitReader *br, ANSSymbolReader *reader,
+@@ -82,10 +109,7 @@ Status DecodeTree(BitReader *br, ANSSymbolReader *reader,
+                        tree->size() + to_decode + 2, Predictor::Zero, 0, 1);
+     to_decode += 2;
+   }
+-  std::vector<std::pair<pixel_type, pixel_type>> prop_bounds;
+-  prop_bounds.resize(256, {std::numeric_limits<pixel_type>::min(),
+-                           std::numeric_limits<pixel_type>::max()});
+-  return ValidateTree(*tree, prop_bounds, 0);
++  return ValidateTree(*tree);
+ }
+ }  // namespace
+ 
+-- 
+2.50.1
+
diff --git a/meta-oe/recipes-multimedia/libjxl/libjxl_0.10.2.bb b/meta-oe/recipes-multimedia/libjxl/libjxl_0.10.2.bb
index eced6c7726..2bf0f126b0 100644
--- a/meta-oe/recipes-multimedia/libjxl/libjxl_0.10.2.bb
+++ b/meta-oe/recipes-multimedia/libjxl/libjxl_0.10.2.bb
@@ -8,8 +8,10 @@ inherit cmake pkgconfig mime
 
 DEPENDS = "highway brotli"
 
-SRC_URI = "gitsm://github.com/libjxl/libjxl.git;protocol=https;nobranch=1"
-
+SRC_URI = "gitsm://github.com/libjxl/libjxl.git;protocol=https;nobranch=1 \
+           file://CVE-2024-11403.patch \
+           file://CVE-2024-11498.patch \
+          "
 SRCREV = "e1489592a770b989303b0edc5cc1dc447bbe0515"
 S = "${WORKDIR}/git"