16 files changed, 320 insertions, 294 deletions

diff --git a/scripts/gen-site-config b/scripts/gen-site-config
deleted file mode 100755
index 727b809c0f..0000000000
--- a/scripts/gen-site-config
+++ /dev/null
@@ -1,43 +0,0 @@
-#! /bin/sh
-#  Copyright (c) 2005-2008 Wind River Systems, Inc.
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-cat << EOF
-AC_PREREQ(2.57)
-AC_INIT([site_wide],[1.0.0])
-
-EOF
-
-# Disable as endian is set in the default config
-#echo AC_C_BIGENDIAN
-#echo
-
-if [ -e $1/types ] ; then
-  while read type ; do
-    echo "AC_CHECK_SIZEOF([$type])"
-  done < $1/types
-
-  echo
-fi
-
-if [ -e $1/funcs ]; then
-  while read func ; do
-    echo "AC_CHECK_FUNCS([$func])"
-  done < $1/funcs
-
-  echo
-fi
-
-if [ -e $1/headers ]; then
-  while read header ; do
-    echo "AC_CHECK_HEADERS([$header])"
-  done < $1/headers
-
-  echo
-fi
-
-cat << EOF
-AC_OUTPUT
-EOF
diff --git a/scripts/lib/build_perf/html/measurement_chart.html b/scripts/lib/build_perf/html/measurement_chart.html
index 65f1a227ad..05bd84e6ce 100644
--- a/scripts/lib/build_perf/html/measurement_chart.html
+++ b/scripts/lib/build_perf/html/measurement_chart.html
@@ -1,50 +1,100 @@
-<script type="text/javascript">
-  chartsDrawing += 1;
-  google.charts.setOnLoadCallback(drawChart_{{ chart_elem_id }});
-  function drawChart_{{ chart_elem_id }}() {
-    var data = new google.visualization.DataTable();
+<script type="module">
+  // Get raw data
+  const rawData = [
+    {% for sample in measurement.samples %}
+      [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}, {{ sample.start_time }}, '{{sample.commit}}'],
+    {% endfor %}
+  ];
 
-    // Chart options
-    var options = {
-      theme : 'material',
-      legend: 'none',
-      hAxis: { format: '', title: 'Commit number',
-               minValue: {{ chart_opts.haxis.min }},
-               maxValue: {{ chart_opts.haxis.max }} },
-      {% if measurement.type == 'time' %}
-      vAxis: { format: 'h:mm:ss' },
-      {% else %}
-      vAxis: { format: '' },
-      {% endif %}
-      pointSize: 5,
-      chartArea: { left: 80, right: 15 },
-    };
+  const convertToMinute = (time) => {
+    return time[0]*60 + time[1] + time[2]/60 + time[3]/3600;
+  }
 
-    // Define data columns
-    data.addColumn('number', 'Commit');
-    data.addColumn('{{ measurement.value_type.gv_data_type }}',
-                   '{{ measurement.value_type.quantity }}');
-    // Add data rows
-    data.addRows([
-      {% for sample in measurement.samples %}
-        [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}],
-      {% endfor %}
-    ]);
+  // Update value format to either minutes or leave as size value
+  const updateValue = (value) => {
+    // Assuming the array values are duration in the format [hours, minutes, seconds, milliseconds]
+    return Array.isArray(value) ? convertToMinute(value) : value
+  }
 
-    // Finally, draw the chart
-    chart_div = document.getElementById('{{ chart_elem_id }}');
-    var chart = new google.visualization.LineChart(chart_div);
-    google.visualization.events.addListener(chart, 'ready', function () {
-      //chart_div = document.getElementById('{{ chart_elem_id }}');
-      //chart_div.innerHTML = '<img src="' + chart.getImageURI() + '">';
-      png_div = document.getElementById('{{ chart_elem_id }}_png');
-      png_div.outerHTML = '<a id="{{ chart_elem_id }}_png" href="' + chart.getImageURI() + '">PNG</a>';
-      console.log("CHART READY: {{ chart_elem_id }}");
-      chartsDrawing -= 1;
-      if (chartsDrawing == 0)
-        console.log("ALL CHARTS READY");
+  // Convert raw data to the format: [time, value]
+  const data = rawData.map(([commit, value, time]) => {
+    return [
+      // The Date object takes values in milliseconds rather than seconds. So to use a Unix timestamp we have to multiply it by 1000.
+      new Date(time * 1000).getTime(),
+      // Assuming the array values are duration in the format [hours, minutes, seconds, milliseconds]
+      updateValue(value)
+    ]
+  });
+
+  // Set chart options
+  const option = {
+    tooltip: {
+      trigger: 'axis',
+      enterable: true,
+      position: function (point, params, dom, rect, size) {
+        return [point[0]-150, '10%'];
+      },
+      formatter: function (param) {
+        const value = param[0].value[1]
+        const sample  = rawData.filter(([commit, dataValue]) => updateValue(dataValue) === value)
+        // Add commit hash to the tooltip as a link
+        const commitLink = `https://git.yoctoproject.org/poky/commit/?id=${sample[0][3]}`
+        if ('{{ measurement.value_type.quantity }}' == 'time') {
+          const hours = Math.floor(value/60)
+          const minutes = Math.floor(value % 60)
+          const seconds = Math.floor((value * 60) % 60)
+          return `<strong>Duration:</strong> ${hours}:${minutes}:${seconds}, <br/> <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>`
+        }
+        return `<strong>Size:</strong> ${value.toFixed(2)} MB, <br/> <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>`
+      ;}
+    },
+    xAxis: {
+      type: 'time',
+    },
+    yAxis: {
+      name: '{{ measurement.value_type.quantity }}' == 'time' ? 'Duration in minutes' : 'Disk size in MB',
+      type: 'value',
+      min: function(value) {
+        return Math.round(value.min - 0.5);
+      },
+      max: function(value) {
+        return Math.round(value.max + 0.5);
+      }
+    },
+    dataZoom: [
+      {
+        type: 'slider',
+        xAxisIndex: 0,
+        filterMode: 'none'
+      },
+    ],
+    series: [
+      {
+        name: '{{ measurement.value_type.quantity }}',
+        type: 'line',
+        step: 'start',
+        symbol: 'none',
+        data: data
+      }
+    ]
+  };
+
+  // Draw chart
+  const chart_div = document.getElementById('{{ chart_elem_id }}');
+  // Set dark mode
+  let measurement_chart
+  if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
+      measurement_chart= echarts.init(chart_div, 'dark', {
+      height: 320
     });
-    chart.draw(data, options);
-}
+  } else {
+      measurement_chart= echarts.init(chart_div, null, {
+      height: 320
+    });
+  }
+  // Change chart size with browser resize
+  window.addEventListener('resize', function() {
+    measurement_chart.resize();
+  });
+  measurement_chart.setOption(option);
 </script>
-
diff --git a/scripts/lib/build_perf/html/report.html b/scripts/lib/build_perf/html/report.html
index d1ba6f2578..537ed3ee52 100644
--- a/scripts/lib/build_perf/html/report.html
+++ b/scripts/lib/build_perf/html/report.html
@@ -3,11 +3,7 @@
 <head>
 {# Scripts, for visualization#}
 <!--START-OF-SCRIPTS-->
-<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
-<script type="text/javascript">
-google.charts.load('current', {'packages':['corechart']});
-var chartsDrawing = 0;
-</script>
+<script src=" https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js "></script>
 
 {# Render measurement result charts #}
 {% for test in test_data %}
@@ -23,28 +19,29 @@ var chartsDrawing = 0;
 
 {# Styles #}
 <style>
+:root {
+  --text: #000;
+  --bg: #fff;
+  --h2heading: #707070;
+  --link: #0000EE;
+  --trtopborder: #9ca3af;
+  --trborder: #e5e7eb;
+  --chartborder: #f0f0f0;
+  }
 .meta-table {
   font-size: 14px;
   text-align: left;
   border-collapse: collapse;
 }
-.meta-table tr:nth-child(even){background-color: #f2f2f2}
-meta-table th, .meta-table td {
-  padding: 4px;
-}
 .summary {
-  margin: 0;
   font-size: 14px;
   text-align: left;
   border-collapse: collapse;
 }
-summary th, .meta-table td {
-  padding: 4px;
-}
 .measurement {
   padding: 8px 0px 8px 8px;
-  border: 2px solid #f0f0f0;
-  margin-bottom: 10px;
+  border: 2px solid var(--chartborder);
+  margin: 1.5rem 0;
 }
 .details {
   margin: 0;
@@ -64,18 +61,71 @@ summary th, .meta-table td {
   background-color: #f0f0f0;
   margin-left: 10px;
 }
-hr {
-  color: #f0f0f0;
+.card-container {
+  border-bottom-width: 1px;
+  padding: 1.25rem 3rem;
+  box-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
+  border-radius: 0.25rem;
+}
+body {
+  font-family: 'Helvetica', sans-serif;
+  margin: 3rem 8rem;
+  background-color: var(--bg);
+  color: var(--text);
+}
+h1 {
+  text-align: center;
 }
 h2 {
-  font-size: 20px;
+  font-size: 1.5rem;
   margin-bottom: 0px;
-  color: #707070;
+  color: var(--h2heading);
+  padding-top: 1.5rem;
 }
 h3 {
-  font-size: 16px;
+  font-size: 1.3rem;
   margin: 0px;
-  color: #707070;
+  color: var(--h2heading);
+  padding: 1.5rem 0;
+}
+h4 {
+  font-size: 14px;
+  font-weight: lighter;
+  line-height: 1.2rem;
+  margin: auto;
+  padding-top: 1rem;
+}
+table {
+  margin-top: 1.5rem;
+  line-height: 2rem;
+}
+tr {
+  border-bottom: 1px solid var(--trborder);
+}
+tr:first-child {
+  border-bottom: 1px solid var(--trtopborder);
+}
+tr:last-child {
+  border-bottom: none;
+}
+a {
+  text-decoration: none;
+  font-weight: bold;
+  color: var(--link);
+}
+a:hover {
+  color: #8080ff;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --text: #e9e8fa;
+    --bg: #0F0C28;
+    --h2heading: #B8B7CB;
+    --link: #87cefa;
+    --trtopborder: #394150;
+    --trborder: #212936;
+    --chartborder: #b1b0bf;
+  }
 }
 </style>
 
@@ -83,13 +133,14 @@ h3 {
 </head>
 
 {% macro poky_link(commit) -%}
-    <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a>
+  <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a>
 {%- endmacro %}
 
-<body><div style="width: 700px">
+<body><div>
+  <h1 style="text-align: center;">Performance Test Report</h1>
   {# Test metadata #}
   <h2>General</h2>
-  <hr>
+  <h4>The table provides an overview of the comparison between two selected commits from the same branch.</h4>
   <table class="meta-table" style="width: 100%">
     <tr>
       <th></th>
@@ -112,19 +163,21 @@ h3 {
 
   {# Test result summary #}
   <h2>Test result summary</h2>
-  <hr>
+  <h4>The test summary presents a thorough breakdown of each test conducted on the branch, including details such as build time and disk space consumption. Additionally, it gives insights into the average time taken for test execution, along with absolute and relative values for a better understanding.</h4>
   <table class="summary" style="width: 100%">
+    <tr>
+      <th>Test name</th>
+      <th>Measurement description</th>
+      <th>Mean value</th>
+      <th>Absolute difference</th>
+      <th>Relative difference</th>
+    </tr>
     {% for test in test_data %}
-      {% if loop.index is even %}
-        {% set row_style = 'style="background-color: #f2f2f2"' %}
-      {% else %}
-        {% set row_style = 'style="background-color: #ffffff"' %}
-      {% endif %}
       {% if test.status == 'SUCCESS' %}
         {% for measurement in test.measurements %}
           <tr {{ row_style }}>
             {% if loop.index == 1 %}
-              <td>{{ test.name }}: {{ test.description }}</td>
+              <td><a href=#{{test.name}}>{{ test.name }}: {{ test.description }}</a></td>
             {% else %}
               {# add empty cell in place of the test name#}
               <td></td>
@@ -153,10 +206,12 @@ h3 {
   </table>
 
   {# Detailed test results #}
+  <h2>Test details</h2>
+  <h4>The following section provides details of each test, accompanied by charts representing build time and disk usage over time or by commit number.</h4>
   {% for test in test_data %}
-  <h2>{{ test.name }}: {{ test.description }}</h2>
-  <hr>
+  <h3 style="color: #000;" id={{test.name}}>{{ test.name }}: {{ test.description }}</h3>
     {% if test.status == 'SUCCESS' %}
+    <div class="card-container">
       {% for measurement in test.measurements %}
         <div class="measurement">
           <h3>{{ measurement.description }}</h3>
@@ -275,7 +330,8 @@ h3 {
             {% endif %}
           {% endif %}
         </div>
-      {% endfor %}
+        {% endfor %}
+      </div>
     {# Unsuccessful test #}
     {% else %}
       <span style="font-size: 150%; font-weight: bold; color: red;">{{ test.status }}
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py
index ab77424cc7..f4e6a92e09 100644
--- a/scripts/lib/build_perf/report.py
+++ b/scripts/lib/build_perf/report.py
@@ -294,7 +294,7 @@ class SizeVal(MeasurementVal):
             return "null"
         return self / 1024
 
-def measurement_stats(meas, prefix=''):
+def measurement_stats(meas, prefix='', time=0):
     """Get statistics of a measurement"""
     if not meas:
         return {prefix + 'sample_cnt': 0,
@@ -319,6 +319,8 @@ def measurement_stats(meas, prefix=''):
     stats['quantity'] = val_cls.quantity
     stats[prefix + 'sample_cnt'] = len(values)
 
+    # Add start time for both type sysres and disk usage
+    start_time = time 
     mean_val = val_cls(mean(values))
     min_val = val_cls(min(values))
     max_val = val_cls(max(values))
@@ -334,6 +336,7 @@ def measurement_stats(meas, prefix=''):
     stats[prefix + 'max'] = max_val
     stats[prefix + 'minus'] = val_cls(mean_val - min_val)
     stats[prefix + 'plus'] = val_cls(max_val - mean_val)
+    stats[prefix + 'start_time'] = start_time
 
     return stats
 
diff --git a/scripts/lib/devtool/ide_sdk.py b/scripts/lib/devtool/ide_sdk.py
index 7807b322b3..65873b088d 100755
--- a/scripts/lib/devtool/ide_sdk.py
+++ b/scripts/lib/devtool/ide_sdk.py
@@ -1052,7 +1052,7 @@ def register_commands(subparsers, context):
     parser_ide_sdk.add_argument(
         '-I', '--key', help='Specify ssh private key for connection to the target')
     parser_ide_sdk.add_argument(
-        '--skip-bitbake', help='Generate IDE configuration but skip calling bibtake to update the SDK.', action='store_true')
+        '--skip-bitbake', help='Generate IDE configuration but skip calling bitbake to update the SDK', action='store_true')
     parser_ide_sdk.add_argument(
         '-k', '--bitbake-k', help='Pass -k parameter to bitbake', action='store_true')
     parser_ide_sdk.add_argument(
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 6674e67267..1d0fe13788 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -387,6 +387,19 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
                 ret[split[3]] = split[0:3]
     return ret
 
+def _git_modified(repodir):
+    """List the difference between HEAD and the index"""
+    import bb
+    cmd = ['git', 'status', '--porcelain']
+    out, _ = bb.process.run(cmd, cwd=repodir)
+    ret = []
+    if out:
+        for line in out.split("\n"):
+            if line and not line.startswith('??'):
+                ret.append(line[3:])
+    return ret
+
+
 def _git_exclude_path(srctree, path):
     """Return pathspec (list of paths) that excludes certain path"""
     # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
@@ -460,32 +473,6 @@ def sync(args, config, basepath, workspace):
     finally:
         tinfoil.shutdown()
 
-def symlink_oelocal_files_srctree(rd, srctree):
-    import oe.patch
-    if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
-        # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
-        # (otherwise the recipe won't build as expected)
-        local_files_dir = os.path.join(srctree, 'oe-local-files')
-        addfiles = []
-        for root, _, files in os.walk(local_files_dir):
-            relpth = os.path.relpath(root, local_files_dir)
-            if relpth != '.':
-                bb.utils.mkdirhier(os.path.join(srctree, relpth))
-            for fn in files:
-                if fn == '.gitignore':
-                    continue
-                destpth = os.path.join(srctree, relpth, fn)
-                if os.path.exists(destpth):
-                    os.unlink(destpth)
-                if relpth != '.':
-                    back_relpth = os.path.relpath(local_files_dir, root)
-                    os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth)
-                else:
-                    os.symlink('oe-local-files/%s' % fn, destpth)
-                addfiles.append(os.path.join(relpth, fn))
-        if addfiles:
-            oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd)
-
 def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
     """Extract sources of a recipe"""
     import oe.recipeutils
@@ -657,35 +644,22 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
             elif not os.path.exists(workshareddir):
                 oe.path.copyhardlinktree(srcsubdir, workshareddir)
 
-        tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
-        srctree_localdir = os.path.join(srctree, 'oe-local-files')
-
         if sync:
-            bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
-
-            # Move the oe-local-files directory to srctree.
-            # As oe-local-files is not part of the constructed git tree,
-            # removing it directly during the synchronization might surprise
-            # the user.  Instead, we move it to oe-local-files.bak and remind
-            # the user in the log message.
-            if os.path.exists(srctree_localdir + '.bak'):
-                shutil.rmtree(srctree_localdir + '.bak')
-
-            if os.path.exists(srctree_localdir):
-                logger.info('Backing up current local file directory %s' % srctree_localdir)
-                shutil.move(srctree_localdir, srctree_localdir + '.bak')
-
-            if os.path.exists(tempdir_localdir):
-                logger.info('Syncing local source files to srctree...')
-                shutil.copytree(tempdir_localdir, srctree_localdir)
-        else:
-            # Move oe-local-files directory to srctree
-            if os.path.exists(tempdir_localdir):
-                logger.info('Adding local source files to srctree...')
-                shutil.move(tempdir_localdir, srcsubdir)
+            try:
+                logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch))
+                bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree)
+
+                # Use git fetch to update the source with the current recipe
+                # To be able to update the currently checked out branch with
+                # possibly new history (no fast-forward) git needs to be told
+                # that's ok
+                logger.info('Syncing source files including patches to git branch: %s' % devbranch)
+                bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
+            except bb.process.ExecutionError as e:
+                raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e))
 
+        else:
             shutil.move(srcsubdir, srctree)
-            symlink_oelocal_files_srctree(d, srctree)
 
         if is_kernel_yocto:
             logger.info('Copying kernel config to srctree')
@@ -841,34 +815,22 @@ def modify(args, config, basepath, workspace):
             if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
                 oe.path.copyhardlinktree(srcdir, srctree)
                 workdir = rd.getVar('WORKDIR')
+                unpackdir = rd.getVar('UNPACKDIR')
                 srcsubdir = rd.getVar('S')
                 localfilesdir = os.path.join(srctree, 'oe-local-files')
-                # Move local source files into separate subdir
-                recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
-                local_files = oe.recipeutils.get_recipe_local_files(rd)
 
-                for key in local_files.copy():
-                    if key.endswith('scc'):
-                        sccfile = open(local_files[key], 'r')
-                        for l in sccfile:
-                            line = l.split()
-                            if line and line[0] in ('kconf', 'patch'):
-                                cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
-                                if not cfg in local_files.values():
-                                    local_files[line[-1]] = cfg
-                                    shutil.copy2(cfg, workdir)
-                        sccfile.close()
-
-                # Ignore local files with subdir={BP}
+                # Add locally copied files to gitignore as we add back to the metadata directly
+                local_files = oe.recipeutils.get_recipe_local_files(rd)
                 srcabspath = os.path.abspath(srcsubdir)
-                local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and  (srcabspath == workdir or not  os.path.join(workdir, fname).startswith(srcabspath + os.sep))]
+                local_files = [fname for fname in local_files if
+                               os.path.exists(os.path.join(unpackdir, fname)) and
+                               srcabspath == unpackdir]
                 if local_files:
-                    for fname in local_files:
-                        _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
-                    with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
-                        f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n')
-
-                symlink_oelocal_files_srctree(rd, srctree)
+                    with open(os.path.join(srctree, '.gitignore'), 'a+') as f:
+                        f.write('# Ignore local files, by default. Remove following lines'
+                                'if you want to commit the directory to Git\n')
+                        for fname in local_files:
+                            f.write('%s\n' % fname)
 
                 task = 'do_configure'
                 res = tinfoil.build_targets(pn, task, handle_events=True)
@@ -893,7 +855,10 @@ def modify(args, config, basepath, workspace):
                 (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree)
                 commits["."] = stdout.split()
                 check_commits = True
-                (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
+                try:
+                    (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
+                except bb.process.ExecutionError:
+                    stdout = ""
                 for line in stdout.splitlines():
                     (rev, submodule_path) = line.split()
                     submodule = os.path.relpath(submodule_path, srctree)
@@ -1452,8 +1417,10 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
          1. updated - files that already exist in SRCURI
          2. added - new files files that don't exist in SRCURI
          3  removed - files that exist in SRCURI but not in exported files
-      In each dict the key is the 'basepath' of the URI and value is the
-      absolute path to the existing file in recipe space (if any).
+       In each dict the key is the 'basepath' of the URI and value is:
+         - for updated and added dicts, a dict with 1 optionnal key:
+           - 'path': the absolute path to the existing file in recipe space (if any)
+         - for removed dict, the absolute path to the existing file in recipe space
     """
     import oe.recipeutils
 
@@ -1462,6 +1429,7 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
     # Instead they are directly copied over the original source files (in
     # recipe space).
     existing_files = oe.recipeutils.get_recipe_local_files(rd)
+
     new_set = None
     updated = OrderedDict()
     added = OrderedDict()
@@ -1478,24 +1446,28 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
     if branchname.startswith(override_branch_prefix):
         return (updated, added, removed)
 
-    local_files_dir = os.path.join(srctreebase, 'oe-local-files')
-    git_files = _git_ls_tree(srctree)
-    if 'oe-local-files' in git_files:
-        # If tracked by Git, take the files from srctree HEAD. First get
-        # the tree object of the directory
-        tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
-        tree = git_files['oe-local-files'][2]
-        bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
-                        env=dict(os.environ, GIT_WORK_TREE=destdir,
-                                 GIT_INDEX_FILE=tmp_index))
-        new_set = list(_git_ls_tree(srctree, tree, True).keys())
-    elif os.path.isdir(local_files_dir):
-        # If not tracked by Git, just copy from working copy
-        new_set = _ls_tree(local_files_dir)
-        bb.process.run(['cp', '-ax',
-                        os.path.join(local_files_dir, '.'), destdir])
-    else:
-        new_set = []
+    files = _git_modified(srctree)
+        #if not files:
+        #    files = _ls_tree(srctree)
+    for f in files:
+        fullfile = os.path.join(srctree, f)
+        if os.path.exists(os.path.join(fullfile, ".git")):
+            # submodules handled elsewhere
+            continue
+        if f not in existing_files:
+            added[f] = {}
+            if os.path.isdir(os.path.join(srctree, f)):
+                shutil.copytree(fullfile, os.path.join(destdir, f))
+            else:
+                shutil.copy2(fullfile, os.path.join(destdir, f))
+        elif not os.path.exists(fullfile):
+            removed[f] = existing_files[f]
+        elif f in existing_files:
+            updated[f] = {'path' : existing_files[f]}
+            if os.path.isdir(os.path.join(srctree, f)):
+                shutil.copytree(fullfile, os.path.join(destdir, f))
+            else:
+                shutil.copy2(fullfile, os.path.join(destdir, f))
 
     # Special handling for kernel config
     if bb.data.inherits_class('kernel-yocto', rd):
@@ -1503,17 +1475,14 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
         fragment_path = os.path.join(destdir, fragment_fn)
         if _create_kconfig_diff(srctree, rd, fragment_path):
             if os.path.exists(fragment_path):
-                if fragment_fn not in new_set:
-                    new_set.append(fragment_fn)
-                # Copy fragment to local-files
-                if os.path.isdir(local_files_dir):
-                    shutil.copy2(fragment_path, local_files_dir)
+                if fragment_fn in removed:
+                    del removed[fragment_fn]
+                if fragment_fn not in updated and fragment_fn not in added:
+                    added[fragment_fn] = {}
             else:
-                if fragment_fn in new_set:
-                    new_set.remove(fragment_fn)
-                # Remove fragment from local-files
-                if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
-                    os.unlink(os.path.join(local_files_dir, fragment_fn))
+                if fragment_fn in updated:
+                    revoved[fragment_fn] = updated[fragment_fn]
+                    del updated[fragment_fn]
 
     # Special handling for cml1, ccmake, etc bbclasses that generated
     # configuration fragment files that are consumed as source files
@@ -1521,42 +1490,13 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
         if bb.data.inherits_class(frag_class, rd):
             srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
             if os.path.exists(srcpath):
-                if frag_name not in new_set:
-                    new_set.append(frag_name)
+                if frag_name in removed:
+                    del removed[frag_name]
+                if frag_name not in updated:
+                    added[frag_name] = {}
                 # copy fragment into destdir
                 shutil.copy2(srcpath, destdir)
-                # copy fragment into local files if exists
-                if os.path.isdir(local_files_dir):
-                    shutil.copy2(srcpath, local_files_dir)
-
-    if new_set is not None:
-        for fname in new_set:
-            if fname in existing_files:
-                origpath = existing_files.pop(fname)
-                workpath = os.path.join(local_files_dir, fname)
-                if not filecmp.cmp(origpath, workpath):
-                    updated[fname] = origpath
-            elif fname != '.gitignore':
-                added[fname] = None
-
-        workdir = rd.getVar('WORKDIR')
-        s = rd.getVar('S')
-        if not s.endswith(os.sep):
-            s += os.sep
-
-        if workdir != s:
-            # Handle files where subdir= was specified
-            for fname in list(existing_files.keys()):
-                # FIXME handle both subdir starting with BP and not?
-                fworkpath = os.path.join(workdir, fname)
-                if fworkpath.startswith(s):
-                    fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
-                    if os.path.exists(fpath):
-                        origpath = existing_files.pop(fname)
-                        if not filecmp.cmp(origpath, fpath):
-                            updated[fpath] = origpath
-
-        removed = existing_files
+
     return (updated, added, removed)
 
 
@@ -1640,7 +1580,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                     redirect_output=dry_run_outdir)
         else:
             files_dir = _determine_files_dir(rd)
-            for basepath, path in upd_f.items():
+            for basepath, param in upd_f.items():
+                path = param['path']
                 logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
@@ -1650,7 +1591,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                     _move_file(os.path.join(local_files_dir, basepath), path,
                                dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 update_srcuri= True
-            for basepath, path in new_f.items():
+            for basepath, param in new_f.items():
+                path = param['path']
                 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
                            os.path.join(files_dir, basepath),
@@ -1772,7 +1714,8 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
         else:
             # Update existing files
             files_dir = _determine_files_dir(rd)
-            for basepath, path in upd_f.items():
+            for basepath, param in upd_f.items():
+                path = param['path']
                 logger.info('Updating file %s' % basepath)
                 if os.path.isabs(basepath):
                     # Original file (probably with subdir pointing inside source tree)
@@ -1806,7 +1749,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                            dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                 updatefiles = True
             # Add any new files
-            for basepath, path in new_f.items():
+            for basepath, param in new_f.items():
                 logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                 _move_file(os.path.join(local_files_dir, basepath),
                            os.path.join(files_dir, basepath),
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index fa5b8ef3c7..a8130ed23f 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -32,7 +32,7 @@ def _run(cmd, cwd=''):
 
 def _get_srctree(tmpdir):
     srctree = tmpdir
-    dirs = scriptutils.filter_src_subdirs(tmpdir)
+    dirs = os.listdir(tmpdir)
     if len(dirs) == 1:
         srctree = os.path.join(tmpdir, dirs[0])
     else:
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index 341e893305..10945d6008 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -101,7 +101,7 @@ def determine_file_source(targetpath, rd):
     import oe.recipeutils
 
     # See if it's in do_install for the recipe
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
     src_uri = rd.getVar('SRC_URI')
     srcfile = ''
     modpatches = []
@@ -113,9 +113,9 @@ def determine_file_source(targetpath, rd):
         if not srcpath.startswith('/'):
             # Handle non-absolute path
             srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
-        if srcpath.startswith(workdir):
+        if srcpath.startswith(unpackdir):
             # OK, now we have the source file name, look for it in SRC_URI
-            workdirfile = os.path.relpath(srcpath, workdir)
+            workdirfile = os.path.relpath(srcpath, unpackdir)
             # FIXME this is where we ought to have some code in the fetcher, because this is naive
             for item in src_uri.split():
                 localpath = bb.fetch2.localpath(item, rd)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 8e9ff38db6..066366e34f 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -528,7 +528,7 @@ def create_recipe(args):
         if ftmpdir and args.keep_temp:
             logger.info('Fetch temp directory is %s' % ftmpdir)
 
-        dirlist = scriptutils.filter_src_subdirs(srctree)
+        dirlist = os.listdir(srctree)
         logger.debug('Directory listing (excluding filtered out):\n  %s' % '\n  '.join(dirlist))
         if len(dirlist) == 1:
             singleitem = os.path.join(srctree, dirlist[0])
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index f23e53cba9..81f0b01fa5 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -179,6 +179,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
                 f.write('SRCREV = "%s"\n' % srcrev)
                 f.write('PV = "0.0+"\n')
                 f.write('WORKDIR = "%s"\n' % tmpworkdir)
+                f.write('UNPACKDIR = "%s"\n' % destdir)
+
                 # Set S out of the way so it doesn't get created under the workdir
                 f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
                 if not mirrors:
@@ -232,10 +234,6 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
                 if e.errno != errno.ENOTEMPTY:
                     raise
 
-        bb.utils.mkdirhier(destdir)
-        for fn in os.listdir(tmpworkdir):
-            shutil.move(os.path.join(tmpworkdir, fn), destdir)
-
     finally:
         if not preserve_tmp:
             shutil.rmtree(tmpdir)
@@ -271,12 +269,3 @@ def is_src_url(param):
         return True
     return False
 
-def filter_src_subdirs(pth):
-    """
-    Filter out subdirectories of initial unpacked source trees that we do not care about.
-    Used by devtool and recipetool.
-    """
-    dirlist = os.listdir(pth)
-    filterout = ['git.indirectionsymlink', 'source-date-epoch', 'sstate-install-recipe_qa']
-    dirlist = [x for x in dirlist if x not in filterout]
-    return dirlist
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 674ccfc244..ce7e6c5d75 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -359,7 +359,7 @@ class Disk:
         Remove files/dirs and their contents from the partition.
         This only applies to ext* partition.
         """
-        abs_path = re.sub('\/\/+', '/', path)
+        abs_path = re.sub(r'\/\/+', '/', path)
         cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
                                             self._get_part_image(pnum),
                                             abs_path)
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg-efi.py
index 13a9cddf4e..7cc5131541 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg-efi.py
@@ -428,10 +428,10 @@ class BootimgEFIPlugin(SourcePlugin):
             elif source_params['loader'] == 'uefi-kernel':
                 kernel = get_bitbake_var("KERNEL_IMAGETYPE")
                 if not kernel:
-                    raise WicError("Empty KERNEL_IMAGETYPE %s\n" % target)
+                    raise WicError("Empty KERNEL_IMAGETYPE")
                 target = get_bitbake_var("TARGET_SYS")
                 if not target:
-                    raise WicError("Unknown arch (TARGET_SYS) %s\n" % target)
+                    raise WicError("Empty TARGET_SYS")
 
                 if re.match("x86_64", target):
                     kernel_efi_image = "bootx64.efi"
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index e29f3a4c2f..c990143c0d 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -43,7 +43,7 @@ class RootfsPlugin(SourcePlugin):
         # directory, or modify a directory outside OpenEmbedded).
         full_path = os.path.realpath(os.path.join(rootfs_dir, path))
         if not full_path.startswith(os.path.realpath(rootfs_dir)):
-            logger.error("%s: Must point inside the rootfs:" % (cmd, path))
+            logger.error("%s: Must point inside the rootfs: %s" % (cmd, path))
             sys.exit(1)
 
         return full_path
diff --git a/scripts/oe-build-perf-report b/scripts/oe-build-perf-report
index 7812ea4540..6c3c726ee3 100755
--- a/scripts/oe-build-perf-report
+++ b/scripts/oe-build-perf-report
@@ -336,8 +336,12 @@ def print_html_report(data, id_comp, buildstats):
                 test_i = test_data['tests'][test]
                 meas_i = test_i['measurements'][meas]
                 commit_num = get_data_item(meta, 'layers.meta.commit_count')
-                samples.append(measurement_stats(meas_i))
+                commit = get_data_item(meta, 'layers.meta.commit')
+                # Add start_time for both test measurement types of sysres and disk usage
+                start_time = test_i['start_time'][0]
+                samples.append(measurement_stats(meas_i, '', start_time))
                 samples[-1]['commit_num'] = commit_num
+                samples[-1]['commit'] = commit
 
             absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
             reldiff = absdiff * 100 / samples[id_comp]['mean']
@@ -473,7 +477,7 @@ Examine build performance test results from a Git repository"""
     group.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
     group.add_argument('--branch2', help="Branch to find comparision revisions in")
     group.add_argument('--machine', default='qemux86')
-    group.add_argument('--history-length', default=25, type=int,
+    group.add_argument('--history-length', default=300, type=int,
                        help="Number of tested revisions to plot in html report")
     group.add_argument('--commit',
                        help="Revision to search for")
diff --git a/scripts/oe-debuginfod b/scripts/oe-debuginfod
index b525310225..5e70d37b8b 100755
--- a/scripts/oe-debuginfod
+++ b/scripts/oe-debuginfod
@@ -15,14 +15,29 @@ scriptpath.add_bitbake_lib_path()
 
 import bb.tinfoil
 import subprocess
+import argparse
 
 if __name__ == "__main__":
+    p = argparse.ArgumentParser()
+    p.add_argument("-d", action='store_true', \
+                         help="store debuginfod files in project sub-directory")
+    
+    args = p.parse_args()
+    
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
         package_classes_var = "DEPLOY_DIR_" + tinfoil.config_data.getVar("PACKAGE_CLASSES").split()[0].replace("package_", "").upper()
         feed_dir = tinfoil.config_data.getVar(package_classes_var, expand=True)
 
+    opts = [ '--verbose', '-R', '-U', feed_dir ]
+    
+    if args.d:
+        fdir = os.path.join(os.getcwd(), 'oedid-files')
+        os.makedirs(fdir, exist_ok=True)
+        opts += [ '-d', os.path.join(fdir, 'did.sqlite') ]
+        
     subprocess.call(['bitbake', '-c', 'addto_recipe_sysroot', 'elfutils-native'])
 
-    subprocess.call(['oe-run-native', 'elfutils-native', 'debuginfod', '--verbose', '-R', '-U', feed_dir])
+    subprocess.call(['oe-run-native', 'elfutils-native', 'debuginfod'] + opts)
+    # we should not get here
     print("\nTo use the debuginfod server please ensure that this variable PACKAGECONFIG:pn-elfutils-native = \"debuginfod libdebuginfod\" is set in the local.conf")
diff --git a/scripts/oe-setup-build b/scripts/oe-setup-build
index 5364f2b481..c0476992a2 100755
--- a/scripts/oe-setup-build
+++ b/scripts/oe-setup-build
@@ -91,7 +91,16 @@ def setup_build_env(args):
     builddir = args.b if args.b else template["buildpath"]
     no_shell = args.no_shell
     coredir = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
-    cmd = "TEMPLATECONF={} . {} {}".format(template["templatepath"], os.path.join(coredir, 'oe-init-build-env'), builddir)
+    cmd_base = ". {} {}".format(os.path.join(coredir, 'oe-init-build-env'), os.path.abspath(builddir))
+
+    initbuild = os.path.join(builddir, 'init-build-env')
+    if not os.path.exists(initbuild):
+        os.makedirs(builddir, exist_ok=True)
+        with open(initbuild, 'w') as f:
+            f.write(cmd_base)
+    print("\nRun '. {}' to initialize the build in a current shell session.\n".format(initbuild))
+
+    cmd = "TEMPLATECONF={} {}".format(template["templatepath"], cmd_base)
     if not no_shell:
         cmd = cmd + " && {}".format(os.environ['SHELL'])
     print("Running:", cmd)