rpm: Handle proper return value to avoid major issues

0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch changed
to avoid critical issues
Handled return values of getrlimit() and lzma_cputhreads() functions
to avoid unexpected behaviours like devide by zero and potential read
of uninitialized variable 'virtual_memory'
Upstream-Status: Pending [merge of multithreading patches to upstream]

(From OE-Core rev: 5aae9c2cb464350bc443a0f60fd6602942e61f46)

Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 14 insertions, 11 deletions

diff --git a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
index 6454785254..dc3f74fecd 100644
--- a/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
+++ b/meta/recipes-devtools/rpm/files/0001-rpm-rpmio.c-restrict-virtual-memory-usage-if-limit-s.patch
@@ -11,36 +11,39 @@ CPU thread.
 Upstream-Status: Pending [merge of multithreading patches to upstream]
 
 Signed-off-by: Peter Bergin <peter@berginkonsult.se>
+Signed-off-by: Ranjitsinh Rathod <ranjitsinh.rathod@kpit.com>
 ---
- rpmio/rpmio.c | 34 ++++++++++++++++++++++++++++++++++
- 1 file changed, 34 insertions(+)
+ rpmio/rpmio.c | 36 ++++++++++++++++++++++++++++++++++++
+ 1 file changed, 36 insertions(+)
 
 diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
 index e051c98..b3c56b6 100644
 --- a/rpmio/rpmio.c
 +++ b/rpmio/rpmio.c
-@@ -845,6 +845,40 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
+@@ -845,6 +845,42 @@ static LZFILE *lzopen_internal(const char *mode, int fd, int xz)
                 }
  #endif
  
-+               struct rlimit virtual_memory;
-+               getrlimit(RLIMIT_AS, &virtual_memory);
-+               if (virtual_memory.rlim_cur != RLIM_INFINITY) {
++               struct rlimit virtual_memory = {RLIM_INFINITY , RLIM_INFINITY};
++               int status = getrlimit(RLIMIT_AS, &virtual_memory);
++               if ((status != -1) && (virtual_memory.rlim_cur != RLIM_INFINITY)) {
 +                       const uint64_t virtual_memlimit = virtual_memory.rlim_cur;
++                       uint32_t threads_max = lzma_cputhreads();
 +                       const uint64_t virtual_memlimit_per_cpu_thread =
-+                               virtual_memlimit / lzma_cputhreads();
-+                       uint64_t memory_usage_virt;
++                               virtual_memlimit / ((threads_max == 0) ? 1 : threads_max);
 +                       rpmlog(RPMLOG_NOTICE, "XZ: virtual memory restricted to %lu and "
 +                              "per CPU thread %lu\n", virtual_memlimit, virtual_memlimit_per_cpu_thread);
++                       uint64_t memory_usage_virt;
 +                       /* keep reducing the number of compression threads until memory
 +                          usage falls below the limit per CPU thread*/
 +                       while ((memory_usage_virt = lzma_stream_encoder_mt_memusage(&mt_options)) >
 +                              virtual_memlimit_per_cpu_thread) {
-+                               /* If number of threads goes down to zero lzma_stream_encoder will
-+                                * will return UINT64_MAX. We must check here to avoid an infinite loop.
++                               /* If number of threads goes down to zero or in case of any other error
++                                * lzma_stream_encoder_mt_memusage will return UINT64_MAX. We must check
++                                * for both the cases here to avoid an infinite loop.
 +                                * If we get into situation that one thread requires more virtual memory
 +                                * than available we set one thread, print error message and try anyway. */
-+                               if (--mt_options.threads == 0) {
++                               if ((--mt_options.threads == 0) || (memory_usage_virt == UINT64_MAX)) {
 +                                       mt_options.threads = 1;
 +                                       rpmlog(RPMLOG_WARNING,
 +                                              "XZ: Could not adjust number of threads to get below "