ffmpeg: fix for CVE-2022-3965

A vulnerability classified as problematic was found in ffmpeg. This vulnerability affects the function
smc_encode_stream of the file libavcodec/smcenc.c of the component QuickTime Graphics Video Encoder. The
manipulation of the argument y_size leads to out-of-bounds read. The attack can be initiated remotely.
The name of the patch is 13c13109759090b7f7182480d075e13b36ed8edd. It is recommended to apply a patch to
fix this issue. The identifier of this vulnerability is VDB-213544.

Reference:
https://nvd.nist.gov/vuln/detail/CVE-2022-3965

Upstream Fix:
https://github.com/FFmpeg/FFmpeg/commit/13c13109759090b7f7182480d075e13b36ed8edd

(From OE-Core rev: 81b75c0331eadc920d27f7115fa9ec99e089bb5a)

Signed-off-by: Narpat Mali <narpat.mali@windriver.com>
Signed-off-by: Alexandre Belloni <alexandre.belloni@bootlin.com>
(cherry picked from commit b88c96fe8964614978aa25a65dd34fc3c05c664c)
Signed-off-by: Steve Sakoman <steve@sakoman.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

2 files changed, 111 insertions, 1 deletions

diff --git a/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-avcodec-smcenc-stop-accessing-out-of-bounds-frame.patch b/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-avcodec-smcenc-stop-accessing-out-of-bounds-frame.patch
new file mode 100644
index 0000000000..923fc6a9c1
--- /dev/null
+++ b/meta/recipes-multimedia/ffmpeg/ffmpeg/0001-avcodec-smcenc-stop-accessing-out-of-bounds-frame.patch
@@ -0,0 +1,108 @@
+From 13c13109759090b7f7182480d075e13b36ed8edd Mon Sep 17 00:00:00 2001
+From: Paul B Mahol <onemda@gmail.com>
+Date: Sat, 12 Nov 2022 15:19:21 +0100
+Subject: [PATCH] avcodec/smcenc: stop accessing out of bounds frame
+
+Upstream-Status: Backport [https://github.com/FFmpeg/FFmpeg/commit/13c13109759090b7f7182480d075e13b36ed8edd]
+
+Signed-off-by: <narpat.mali@windriver.com>
+
+---
+ libavcodec/smcenc.c | 18 ++++++++++++++----
+ 1 file changed, 14 insertions(+), 4 deletions(-)
+
+diff --git a/libavcodec/smcenc.c b/libavcodec/smcenc.c
+index f3d26a4e8d..33549b8ab4 100644
+--- a/libavcodec/smcenc.c
++++ b/libavcodec/smcenc.c
+@@ -61,6 +61,7 @@ typedef struct SMCContext {
+         { \
+             row_ptr += stride * 4; \
+             pixel_ptr = row_ptr; \
++            cur_y += 4; \
+         } \
+     } \
+ }
+@@ -117,6 +118,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+     const uint8_t *prev_pixels = (const uint8_t *)s->prev_frame->data[0];
+     uint8_t *distinct_values = s->distinct_values;
+     const uint8_t *pixel_ptr, *row_ptr;
++    const int height = frame->height;
+     const int width = frame->width;
+     uint8_t block_values[16];
+     int block_counter = 0;
+@@ -125,13 +127,14 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+     int color_octet_index = 0;
+     int color_table_index;  /* indexes to color pair, quad, or octet tables */
+     int total_blocks;
++    int cur_y = 0;
+ 
+     memset(s->color_pairs, 0, sizeof(s->color_pairs));
+     memset(s->color_quads, 0, sizeof(s->color_quads));
+     memset(s->color_octets, 0, sizeof(s->color_octets));
+ 
+     /* Number of 4x4 blocks in frame. */
+-    total_blocks = ((frame->width + 3) / 4) * ((frame->height + 3) / 4);
++    total_blocks = ((width + 3) / 4) * ((height + 3) / 4);
+ 
+     pixel_ptr = row_ptr = src_pixels;
+ 
+@@ -145,11 +148,13 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+         int cache_index;
+         int distinct = 0;
+         int blocks = 0;
++        int frame_y = cur_y;
+ 
+         while (prev_pixels && s->key_frame == 0 && block_counter + inter_skip_blocks < total_blocks) {
++            const int y_size = FFMIN(4, height - cur_y);
+             int compare = 0;
+ 
+-            for (int y = 0; y < 4; y++) {
++            for (int y = 0; y < y_size; y++) {
+                 const ptrdiff_t offset = pixel_ptr - src_pixels;
+                 const uint8_t *prev_pixel_ptr = prev_pixels + offset;
+ 
+@@ -170,8 +175,10 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+ 
+         pixel_ptr = xpixel_ptr;
+         row_ptr = xrow_ptr;
++        cur_y = frame_y;
+ 
+         while (block_counter > 0 && block_counter + intra_skip_blocks < total_blocks) {
++            const int y_size = FFMIN(4, height - cur_y);
+             const ptrdiff_t offset = pixel_ptr - src_pixels;
+             const int sy = offset / stride;
+             const int sx = offset % stride;
+@@ -180,7 +187,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+             const uint8_t *old_pixel_ptr = src_pixels + nx + ny * stride;
+             int compare = 0;
+ 
+-            for (int y = 0; y < 4; y++) {
++            for (int y = 0; y < y_size; y++) {
+                 compare |= memcmp(old_pixel_ptr + y * stride, pixel_ptr + y * stride, 4);
+                 if (compare)
+                     break;
+@@ -197,9 +204,11 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+ 
+         pixel_ptr = xpixel_ptr;
+         row_ptr = xrow_ptr;
++        cur_y = frame_y;
+ 
+         while (block_counter + coded_blocks < total_blocks && coded_blocks < 256) {
+-            for (int y = 0; y < 4; y++)
++            const int y_size = FFMIN(4, height - cur_y);
++            for (int y = 0; y < y_size; y++)
+                 memcpy(block_values + y * 4, pixel_ptr + y * stride, 4);
+ 
+             qsort(block_values, 16, sizeof(block_values[0]), smc_cmp_values);
+@@ -224,6 +233,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
+ 
+         pixel_ptr = xpixel_ptr;
+         row_ptr = xrow_ptr;
++        cur_y = frame_y;
+ 
+         blocks = coded_blocks;
+         distinct = coded_distinct;
+-- 
+2.34.1
+
diff --git a/meta/recipes-multimedia/ffmpeg/ffmpeg_5.1.2.bb b/meta/recipes-multimedia/ffmpeg/ffmpeg_5.1.2.bb
index 43b858984b..06eca4fefe 100644
--- a/meta/recipes-multimedia/ffmpeg/ffmpeg_5.1.2.bb
+++ b/meta/recipes-multimedia/ffmpeg/ffmpeg_5.1.2.bb
@@ -23,7 +23,9 @@ LIC_FILES_CHKSUM = "file://COPYING.GPLv2;md5=b234ee4d69f5fce4486a80fdaf4a4263 \
                     file://COPYING.LGPLv3;md5=e6a600fd5e1d9cbde2d983680233ad02"
 
 SRC_URI = "https://www.ffmpeg.org/releases/${BP}.tar.xz \
-           file://0001-avcodec-rpzaenc-stop-accessing-out-of-bounds-frame.patch"
+           file://0001-avcodec-rpzaenc-stop-accessing-out-of-bounds-frame.patch \
+           file://0001-avcodec-smcenc-stop-accessing-out-of-bounds-frame.patch \
+           "
 
 SRC_URI[sha256sum] = "619e706d662c8420859832ddc259cd4d4096a48a2ce1eefd052db9e440eef3dc"