67 files changed, 5946 insertions, 1632 deletions
diff --git a/scripts/lib/argparse_oe.py b/scripts/lib/argparse_oe.py
index 94a4ac5011..176b732bbc 100644
--- a/scripts/lib/argparse_oe.py
+++ b/scripts/lib/argparse_oe.py
@@ -1,4 +1,6 @@
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
diff --git a/scripts/lib/build_perf/html/measurement_chart.html b/scripts/lib/build_perf/html/measurement_chart.html
index 65f1a227ad..86435273cf 100644
--- a/scripts/lib/build_perf/html/measurement_chart.html
+++ b/scripts/lib/build_perf/html/measurement_chart.html
@@ -1,50 +1,168 @@
-<script type="text/javascript">
+<script type="module">
-  chartsDrawing += 1;
+  // Get raw data
-  google.charts.setOnLoadCallback(drawChart_{{ chart_elem_id }});
+  const rawData = [
-  function drawChart_{{ chart_elem_id }}() {
+    {% for sample in measurement.samples %}
-    var data = new google.visualization.DataTable();
+      [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}, {{ sample.start_time }}, '{{sample.commit}}'],
+    {% endfor %}
-    // Chart options
+  ];
-    var options = {
-      theme : 'material',
+  const convertToMinute = (time) => {
-      legend: 'none',
+    return time[0]*60 + time[1] + time[2]/60 + time[3]/3600;
-      hAxis: { format: '', title: 'Commit number',
+  }
-               minValue: {{ chart_opts.haxis.min }},
-               maxValue: {{ chart_opts.haxis.max }} },
+  // Update value format to either minutes or leave as size value
-      {% if measurement.type == 'time' %}
+  const updateValue = (value) => {
-      vAxis: { format: 'h:mm:ss' },
+    // Assuming the array values are duration in the format [hours, minutes, seconds, milliseconds]
-      {% else %}
+    return Array.isArray(value) ? convertToMinute(value) : value
-      vAxis: { format: '' },
+  }
-      {% endif %}
-      pointSize: 5,
+  // Convert raw data to the format: [time, value]
-      chartArea: { left: 80, right: 15 },
+  const data = rawData.map(([commit, value, time]) => {
-    };
+    return [
+      // The Date object takes values in milliseconds rather than seconds. So to use a Unix timestamp we have to multiply it by 1000.
-    // Define data columns
+      new Date(time * 1000).getTime(),
-    data.addColumn('number', 'Commit');
+      // Assuming the array values are duration in the format [hours, minutes, seconds, milliseconds]
-    data.addColumn('{{ measurement.value_type.gv_data_type }}',
+      updateValue(value)
-                   '{{ measurement.value_type.quantity }}');
+    ]
-    // Add data rows
+  });
-    data.addRows([
-      {% for sample in measurement.samples %}
+  const commitCountList = rawData.map(([commit, value, time]) => {
-        [{{ sample.commit_num }}, {{ sample.mean.gv_value() }}],
+    return commit
-      {% endfor %}
+  });
-    ]);
+  const commitCountData = rawData.map(([commit, value, time]) => {
-    // Finally, draw the chart
+    return updateValue(value)
-    chart_div = document.getElementById('{{ chart_elem_id }}');
+  });
-    var chart = new google.visualization.LineChart(chart_div);
-    google.visualization.events.addListener(chart, 'ready', function () {
+  // Set chart options
-      //chart_div = document.getElementById('{{ chart_elem_id }}');
+  const option_start_time = {
-      //chart_div.innerHTML = '<img src="' + chart.getImageURI() + '">';
+    tooltip: {
-      png_div = document.getElementById('{{ chart_elem_id }}_png');
+      trigger: 'axis',
-      png_div.outerHTML = '<a id="{{ chart_elem_id }}_png" href="' + chart.getImageURI() + '">PNG</a>';
+      enterable: true,
-      console.log("CHART READY: {{ chart_elem_id }}");
+      position: function (point, params, dom, rect, size) {
-      chartsDrawing -= 1;
+        return [point[0], '0%'];
-      if (chartsDrawing == 0)
+      },
-        console.log("ALL CHARTS READY");
+      formatter: function (param) {
+        const value = param[0].value[1]
+        const sample  = rawData.filter(([commit, dataValue]) => updateValue(dataValue) === value)
+        const formattedDate = new Date(sample[0][2] * 1000).toString().replace(/GMT[+-]\d{4}/, '').replace(/\(.*\)/, '(CEST)');
+        // Add commit hash to the tooltip as a link
+        const commitLink = `https://git.yoctoproject.org/poky/commit/?id=${sample[0][3]}`
+        if ('{{ measurement.value_type.quantity }}' == 'time') {
+          const hours = Math.floor(value/60)
+          const minutes = Math.floor(value % 60)
+          const seconds = Math.floor((value * 60) % 60)
+          return `<strong>Duration:</strong> ${hours}:${minutes}:${seconds}, <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>, <br/> <strong>Start time:</strong> ${formattedDate}`
+        }
+        return `<strong>Size:</strong> ${value.toFixed(2)} MB, <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>, <br/> <strong>Start time:</strong> ${formattedDate}`
+      ;}
+    },
+    xAxis: {
+      type: 'time',
+    },
+    yAxis: {
+      name: '{{ measurement.value_type.quantity }}' == 'time' ? 'Duration in minutes' : 'Disk size in MB',
+      type: 'value',
+      min: function(value) {
+        return Math.round(value.min - 0.5);
+      },
+      max: function(value) {
+        return Math.round(value.max + 0.5);
+      }
+    },
+    dataZoom: [
+      {
+        type: 'slider',
+        xAxisIndex: 0,
+        filterMode: 'none'
+      },
+    ],
+    series: [
+      {
+        name: '{{ measurement.value_type.quantity }}',
+        type: 'line',
+        symbol: 'none',
+        data: data
+      }
+    ]
+  };
+  const option_commit_count = {
+    tooltip: {
+      trigger: 'axis',
+      enterable: true,
+      position: function (point, params, dom, rect, size) {
+        return [point[0], '0%'];
+      },
+      formatter: function (param) {
+        const value = param[0].value
+        const sample  = rawData.filter(([commit, dataValue]) => updateValue(dataValue) === value)
+        const formattedDate = new Date(sample[0][2] * 1000).toString().replace(/GMT[+-]\d{4}/, '').replace(/\(.*\)/, '(CEST)');
+        // Add commit hash to the tooltip as a link
+        const commitLink = `https://git.yoctoproject.org/poky/commit/?id=${sample[0][3]}`
+        if ('{{ measurement.value_type.quantity }}' == 'time') {
+          const hours = Math.floor(value/60)
+          const minutes = Math.floor(value % 60)
+          const seconds = Math.floor((value * 60) % 60)
+          return `<strong>Duration:</strong> ${hours}:${minutes}:${seconds}, <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>, <br/> <strong>Start time:</strong> ${formattedDate}`
+        }
+        return `<strong>Size:</strong> ${value.toFixed(2)} MB, <strong>Commit number:</strong> <a href="${commitLink}" target="_blank" rel="noreferrer noopener">${sample[0][0]}</a>, <br/> <strong>Start time:</strong> ${formattedDate}`
+      ;}
+    },
+    xAxis: {
+      name: 'Commit count',
+      type: 'category',
+      data: commitCountList
+    },
+    yAxis: {
+      name: '{{ measurement.value_type.quantity }}' == 'time' ? 'Duration in minutes' : 'Disk size in MB',
+      type: 'value',
+      min: function(value) {
+        return Math.round(value.min - 0.5);
+      },
+      max: function(value) {
+        return Math.round(value.max + 0.5);
+      }
+    },
+    dataZoom: [
+      {
+        type: 'slider',
+        xAxisIndex: 0,
+        filterMode: 'none'
+      },
+    ],
+    series: [
+      {
+        name: '{{ measurement.value_type.quantity }}',
+        type: 'line',
+        symbol: 'none',
+        data: commitCountData
+      }
+    ]
+  };
+  // Draw chart
+  const draw_chart = (chart_id, option) => {
+    let chart_name
+    const chart_div = document.getElementById(chart_id);
+    // Set dark mode
+    if (window.matchMedia('(prefers-color-scheme: dark)').matches) {
+        chart_name= echarts.init(chart_div, 'dark', {
+        height: 320
+      });
+    } else {
+        chart_name= echarts.init(chart_div, null, {
+        height: 320
+      });
+    }
+    // Change chart size with browser resize
+    window.addEventListener('resize', function() {
+      chart_name.resize();
    });
-    chart.draw(data, options);
+    return chart_name.setOption(option);
-}
+  }
-</script>
+  draw_chart('{{ chart_elem_start_time_id }}', option_start_time)
+  draw_chart('{{ chart_elem_commit_count_id }}', option_commit_count)
+</script>
diff --git a/scripts/lib/build_perf/html/report.html b/scripts/lib/build_perf/html/report.html
index d1ba6f2578..28cd80e738 100644
--- a/scripts/lib/build_perf/html/report.html
+++ b/scripts/lib/build_perf/html/report.html
@@ -3,17 +3,14 @@
 <head>
 {# Scripts, for visualization#}
 <!--START-OF-SCRIPTS-->
-<script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+<script src=" https://cdn.jsdelivr.net/npm/echarts@5.5.0/dist/echarts.min.js "></script>
-<script type="text/javascript">
-google.charts.load('current', {'packages':['corechart']});
-var chartsDrawing = 0;
-</script>
 {# Render measurement result charts #}
 {% for test in test_data %}
  {% if test.status == 'SUCCESS' %}
    {% for measurement in test.measurements %}
-      {% set chart_elem_id = test.name + '_' + measurement.name + '_chart' %}
+      {% set chart_elem_start_time_id = test.name + '_' + measurement.name + '_chart_start_time' %}
+      {% set chart_elem_commit_count_id = test.name + '_' + measurement.name + '_chart_commit_count' %}
      {% include 'measurement_chart.html' %}
    {% endfor %}
  {% endif %}
@@ -23,28 +20,29 @@ var chartsDrawing = 0;
 {# Styles #}
 <style>
+:root {
+  --text: #000;
+  --bg: #fff;
+  --h2heading: #707070;
+  --link: #0000EE;
+  --trtopborder: #9ca3af;
+  --trborder: #e5e7eb;
+  --chartborder: #f0f0f0;
+  }
 .meta-table {
  font-size: 14px;
  text-align: left;
  border-collapse: collapse;
 }
-.meta-table tr:nth-child(even){background-color: #f2f2f2}
-meta-table th, .meta-table td {
-  padding: 4px;
-}
 .summary {
-  margin: 0;
  font-size: 14px;
  text-align: left;
  border-collapse: collapse;
 }
-summary th, .meta-table td {
-  padding: 4px;
-}
 .measurement {
  padding: 8px 0px 8px 8px;
-  border: 2px solid #f0f0f0;
+  border: 2px solid var(--chartborder);
-  margin-bottom: 10px;
+  margin: 1.5rem 0;
 }
 .details {
  margin: 0;
@@ -64,18 +62,97 @@ summary th, .meta-table td {
  background-color: #f0f0f0;
  margin-left: 10px;
 }
-hr {
+.card-container {
-  color: #f0f0f0;
+  border-bottom-width: 1px;
+  padding: 1.25rem 3rem;
+  box-shadow: 0 1px 3px 0 rgb(0 0 0 / 0.1), 0 1px 2px -1px rgb(0 0 0 / 0.1);
+  border-radius: 0.25rem;
+}
+body {
+  font-family: 'Helvetica', sans-serif;
+  margin: 3rem 8rem;
+  background-color: var(--bg);
+  color: var(--text);
+}
+h1 {
+  text-align: center;
 }
 h2 {
-  font-size: 20px;
+  font-size: 1.5rem;
  margin-bottom: 0px;
-  color: #707070;
+  color: var(--h2heading);
+  padding-top: 1.5rem;
 }
 h3 {
-  font-size: 16px;
+  font-size: 1.3rem;
  margin: 0px;
-  color: #707070;
+  color: var(--h2heading);
+  padding: 1.5rem 0;
+}
+h4 {
+  font-size: 14px;
+  font-weight: lighter;
+  line-height: 1.2rem;
+  margin: auto;
+  padding-top: 1rem;
+}
+table {
+  margin-top: 1.5rem;
+  line-height: 2rem;
+}
+tr {
+  border-bottom: 1px solid var(--trborder);
+}
+tr:first-child {
+  border-bottom: 1px solid var(--trtopborder);
+}
+tr:last-child {
+  border-bottom: none;
+}
+a {
+  text-decoration: none;
+  font-weight: bold;
+  color: var(--link);
+}
+a:hover {
+  color: #8080ff;
+}
+button {
+  background-color: #F3F4F6;
+  border: none;
+  outline: none;
+  cursor: pointer;
+  padding: 10px 12px;
+  transition: 0.3s;
+  border-radius: 8px;
+  color: #3A4353;
+}
+button:hover {
+  background-color: #d6d9e0;
+}
+.tab button.active {
+  background-color: #d6d9e0;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --text: #e9e8fa;
+    --bg: #0F0C28;
+    --h2heading: #B8B7CB;
+    --link: #87cefa;
+    --trtopborder: #394150;
+    --trborder: #212936;
+    --chartborder: #b1b0bf;
+  }
+  button {
+    background-color: #28303E;
+    color: #fff;
+  }
+  button:hover {
+    background-color: #545a69;
+  }
+  .tab button.active {
+    background-color: #545a69;
+  }
 }
 </style>
@@ -83,13 +160,14 @@ h3 {
 </head>
 {% macro poky_link(commit) -%}
-    <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a>
+  <a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?id={{ commit }}">{{ commit[0:11] }}</a>
 {%- endmacro %}
-<body><div style="width: 700px">
+<body><div>
+  <h1 style="text-align: center;">Performance Test Report</h1>
  {# Test metadata #}
  <h2>General</h2>
-  <hr>
+  <h4>The table provides an overview of the comparison between two selected commits from the same branch.</h4>
  <table class="meta-table" style="width: 100%">
    <tr>
      <th></th>
@@ -112,19 +190,21 @@ h3 {
  {# Test result summary #}
  <h2>Test result summary</h2>
-  <hr>
+  <h4>The test summary presents a thorough breakdown of each test conducted on the branch, including details such as build time and disk space consumption. Additionally, it gives insights into the average time taken for test execution, along with absolute and relative values for a better understanding.</h4>
  <table class="summary" style="width: 100%">
+    <tr>
+      <th>Test name</th>
+      <th>Measurement description</th>
+      <th>Mean value</th>
+      <th>Absolute difference</th>
+      <th>Relative difference</th>
+    </tr>
    {% for test in test_data %}
-      {% if loop.index is even %}
-        {% set row_style = 'style="background-color: #f2f2f2"' %}
-      {% else %}
-        {% set row_style = 'style="background-color: #ffffff"' %}
-      {% endif %}
      {% if test.status == 'SUCCESS' %}
        {% for measurement in test.measurements %}
          <tr {{ row_style }}>
            {% if loop.index == 1 %}
-              <td>{{ test.name }}: {{ test.description }}</td>
+              <td><a href=#{{test.name}}>{{ test.name }}: {{ test.description }}</a></td>
            {% else %}
              {# add empty cell in place of the test name#}
              <td></td>
@@ -153,10 +233,12 @@ h3 {
  </table>
  {# Detailed test results #}
+  <h2>Test details</h2>
+  <h4>The following section provides details of each test, accompanied by charts representing build time and disk usage over time or by commit number.</h4>
  {% for test in test_data %}
-  <h2>{{ test.name }}: {{ test.description }}</h2>
+  <h3 style="color: #000;" id={{test.name}}>{{ test.name }}: {{ test.description }}</h3>
-  <hr>
    {% if test.status == 'SUCCESS' %}
+    <div class="card-container">
      {% for measurement in test.measurements %}
        <div class="measurement">
          <h3>{{ measurement.description }}</h3>
@@ -178,7 +260,18 @@ h3 {
            <tr>
              <td style="width: 75%">
                {# Linechart #}
-                <div id="{{ test.name }}_{{ measurement.name }}_chart"></div>
+                <div class="tab {{ test.name }}_{{ measurement.name }}_tablinks">
+                  <button class="tablinks active" onclick="openChart(event, '{{ test.name }}_{{ measurement.name }}_start_time', '{{ test.name }}_{{ measurement.name }}')">Chart with start time</button>
+                  <button class="tablinks" onclick="openChart(event, '{{ test.name }}_{{ measurement.name }}_commit_count', '{{ test.name }}_{{ measurement.name }}')">Chart with commit count</button>
+                </div>
+                <div class="{{ test.name }}_{{ measurement.name }}_tabcontent">
+                  <div id="{{ test.name }}_{{ measurement.name }}_start_time" class="tabcontent" style="display: block;">
+                    <div id="{{ test.name }}_{{ measurement.name }}_chart_start_time"></div>
+                  </div>
+                  <div id="{{ test.name }}_{{ measurement.name }}_commit_count" class="tabcontent" style="display: none;">
+                    <div id="{{ test.name }}_{{ measurement.name }}_chart_commit_count"></div>
+                  </div>
+                </div>
              </td>
              <td>
                {# Measurement statistics #}
@@ -275,7 +368,8 @@ h3 {
            {% endif %}
          {% endif %}
        </div>
-      {% endfor %}
+        {% endfor %}
+      </div>
    {# Unsuccessful test #}
    {% else %}
      <span style="font-size: 150%; font-weight: bold; color: red;">{{ test.status }}
@@ -284,6 +378,31 @@ h3 {
      <div class="preformatted">{{ test.message }}</div>
    {% endif %}
  {% endfor %}
-</div></body>
+</div>
-</html>
+<script>
+function openChart(event, chartType, chartName) {
+  let i, tabcontents, tablinks
+  tabcontents = document.querySelectorAll(`.${chartName}_tabcontent > .tabcontent`);
+  tabcontents.forEach((tabcontent) => {
+    tabcontent.style.display = "none";
+  });
+  tablinks = document.querySelectorAll(`.${chartName}_tablinks > .tablinks`);
+  tablinks.forEach((tabLink) => {
+    tabLink.classList.remove('active');
+  });
+  const targetTab = document.getElementById(chartType)
+  targetTab.style.display = "block";
+  // Call resize on the ECharts instance to redraw the chart
+  const chartContainer = targetTab.querySelector('div')
+  echarts.init(chartContainer).resize();
+  event.currentTarget.classList.add('active');
+}
+</script>
+</body>
+</html>
diff --git a/scripts/lib/build_perf/report.py b/scripts/lib/build_perf/report.py
index 4e8e2a8a93..f4e6a92e09 100644
--- a/scripts/lib/build_perf/report.py
+++ b/scripts/lib/build_perf/report.py
@@ -4,7 +4,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """Handling of build perf test reports"""
-from collections import OrderedDict, Mapping, namedtuple
+from collections import OrderedDict, namedtuple
+from collections.abc import Mapping
 from datetime import datetime, timezone
 from numbers import Number
 from statistics import mean, stdev, variance
@@ -293,7 +294,7 @@ class SizeVal(MeasurementVal):
            return "null"
        return self / 1024
-def measurement_stats(meas, prefix=''):
+def measurement_stats(meas, prefix='', time=0):
    """Get statistics of a measurement"""
    if not meas:
        return {prefix + 'sample_cnt': 0,
@@ -318,6 +319,8 @@ def measurement_stats(meas, prefix=''):
    stats['quantity'] = val_cls.quantity
    stats[prefix + 'sample_cnt'] = len(values)
+    # Add start time for both type sysres and disk usage
+    start_time = time 
    mean_val = val_cls(mean(values))
    min_val = val_cls(min(values))
    max_val = val_cls(max(values))
@@ -333,6 +336,7 @@ def measurement_stats(meas, prefix=''):
    stats[prefix + 'max'] = max_val
    stats[prefix + 'minus'] = val_cls(mean_val - min_val)
    stats[prefix + 'plus'] = val_cls(max_val - mean_val)
+    stats[prefix + 'start_time'] = start_time
    return stats
diff --git a/scripts/lib/buildstats.py b/scripts/lib/buildstats.py
index c69b5bf4d7..6db60d5bcf 100644
--- a/scripts/lib/buildstats.py
+++ b/scripts/lib/buildstats.py
@@ -8,7 +8,7 @@ import json
 import logging
 import os
 import re
-from collections import namedtuple,OrderedDict
+from collections import namedtuple
 from statistics import mean
@@ -79,8 +79,8 @@ class BSTask(dict):
            return self['rusage']['ru_oublock']
    @classmethod
-    def from_file(cls, buildstat_file):
+    def from_file(cls, buildstat_file, fallback_end=0):
-        """Read buildstat text file"""
+        """Read buildstat text file. fallback_end is an optional end time for tasks that are not recorded as finishing."""
        bs_task = cls()
        log.debug("Reading task buildstats from %s", buildstat_file)
        end_time = None
@@ -108,7 +108,10 @@ class BSTask(dict):
                    bs_task[ru_type][ru_key] = val
                elif key == 'Status':
                    bs_task['status'] = val
-        if end_time is not None and start_time is not None:
+        # If the task didn't finish, fill in the fallback end time if specified
+        if start_time and not end_time and fallback_end:
+            end_time = fallback_end
+        if start_time and end_time:
            bs_task['elapsed_time'] = end_time - start_time
        else:
            raise BSError("{} looks like a invalid buildstats file".format(buildstat_file))
@@ -226,25 +229,44 @@ class BuildStats(dict):
        epoch = match.group('epoch')
        return name, epoch, version, revision
+    @staticmethod
+    def parse_top_build_stats(path):
+        """
+        Parse the top-level build_stats file for build-wide start and duration.
+        """
+        start = elapsed = 0
+        with open(path) as fobj:
+            for line in fobj.readlines():
+                key, val = line.split(':', 1)
+                val = val.strip()
+                if key == 'Build Started':
+                    start = float(val)
+                elif key == "Elapsed time":
+                    elapsed = float(val.split()[0])
+        return start, elapsed
    @classmethod
    def from_dir(cls, path):
        """Load buildstats from a buildstats directory"""
-        if not os.path.isfile(os.path.join(path, 'build_stats')):
+        top_stats = os.path.join(path, 'build_stats')
+        if not os.path.isfile(top_stats):
            raise BSError("{} does not look like a buildstats directory".format(path))
        log.debug("Reading buildstats directory %s", path)
        buildstats = cls()
+        build_started, build_elapsed = buildstats.parse_top_build_stats(top_stats)
+        build_end = build_started + build_elapsed
        subdirs = os.listdir(path)
        for dirname in subdirs:
            recipe_dir = os.path.join(path, dirname)
-            if not os.path.isdir(recipe_dir):
+            if dirname == "reduced_proc_pressure" or not os.path.isdir(recipe_dir):
                continue
            name, epoch, version, revision = cls.split_nevr(dirname)
            bsrecipe = BSRecipe(name, epoch, version, revision)
            for task in os.listdir(recipe_dir):
                bsrecipe.tasks[task] = BSTask.from_file(
-                    os.path.join(recipe_dir, task))
+                    os.path.join(recipe_dir, task), build_end)
            if name in buildstats:
                raise BSError("Cannot handle multiple versions of the same "
                              "package ({})".format(name))
diff --git a/scripts/lib/checklayer/__init__.py b/scripts/lib/checklayer/__init__.py
index fe545607bb..86aadf39a6 100644
--- a/scripts/lib/checklayer/__init__.py
+++ b/scripts/lib/checklayer/__init__.py
@@ -16,6 +16,7 @@ class LayerType(Enum):
    BSP = 0
    DISTRO = 1
    SOFTWARE = 2
+    CORE = 3
    ERROR_NO_LAYER_CONF = 98
    ERROR_BSP_DISTRO = 99
@@ -43,7 +44,7 @@ def _get_layer_collections(layer_path, lconf=None, data=None):
    ldata.setVar('LAYERDIR', layer_path)
    try:
-        ldata = bb.parse.handle(lconf, ldata, include=True)
+        ldata = bb.parse.handle(lconf, ldata, include=True, baseconfig=True)
    except:
        raise RuntimeError("Parsing of layer.conf from layer: %s failed" % layer_path)
    ldata.expandVarref('LAYERDIR')
@@ -106,7 +107,13 @@ def _detect_layer(layer_path):
        if distros:
            is_distro = True
-    if is_bsp and is_distro:
+    layer['collections'] = _get_layer_collections(layer['path'])
+    if layer_name == "meta" and "core" in layer['collections']:
+        layer['type'] = LayerType.CORE
+        layer['conf']['machines'] = machines
+        layer['conf']['distros'] = distros
+    elif is_bsp and is_distro:
        layer['type'] = LayerType.ERROR_BSP_DISTRO
    elif is_bsp:
        layer['type'] = LayerType.BSP
@@ -117,8 +124,6 @@ def _detect_layer(layer_path):
    else:
        layer['type'] = LayerType.SOFTWARE
-    layer['collections'] = _get_layer_collections(layer['path'])
    return layer
 def detect_layers(layer_directories, no_auto):
@@ -146,7 +151,7 @@ def detect_layers(layer_directories, no_auto):
    return layers
-def _find_layer_depends(depend, layers):
+def _find_layer(depend, layers):
    for layer in layers:
        if 'collections' not in layer:
            continue
@@ -156,7 +161,28 @@ def _find_layer_depends(depend, layers):
                return layer
    return None
-def add_layer_dependencies(bblayersconf, layer, layers, logger):
+def sanity_check_layers(layers, logger):
+    """
+    Check that we didn't find duplicate collection names, as the layer that will
+    be used is non-deterministic. The precise check is duplicate collections
+    with different patterns, as the same pattern being repeated won't cause
+    problems.
+    """
+    import collections
+    passed = True
+    seen = collections.defaultdict(set)
+    for layer in layers:
+        for name, data in layer.get("collections", {}).items():
+            seen[name].add(data["pattern"])
+    for name, patterns in seen.items():
+        if len(patterns) > 1:
+            passed = False
+            logger.error("Collection %s found multiple times: %s" % (name, ", ".join(patterns)))
+    return passed
+def get_layer_dependencies(layer, layers, logger):
    def recurse_dependencies(depends, layer, layers, logger, ret = []):
        logger.debug('Processing dependencies %s for layer %s.' % \
                    (depends, layer['name']))
@@ -166,7 +192,7 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
            if depend == 'core':
                continue
-            layer_depend = _find_layer_depends(depend, layers)
+            layer_depend = _find_layer(depend, layers)
            if not layer_depend:
                logger.error('Layer %s depends on %s and isn\'t found.' % \
                        (layer['name'], depend))
@@ -203,6 +229,11 @@ def add_layer_dependencies(bblayersconf, layer, layers, logger):
        layer_depends = recurse_dependencies(depends, layer, layers, logger, layer_depends)
    # Note: [] (empty) is allowed, None is not!
+    return layer_depends
+def add_layer_dependencies(bblayersconf, layer, layers, logger):
+    layer_depends = get_layer_dependencies(layer, layers, logger)
    if layer_depends is None:
        return False
    else:
@@ -256,7 +287,7 @@ def check_command(error_msg, cmd, cwd=None):
        raise RuntimeError(msg)
    return output
-def get_signatures(builddir, failsafe=False, machine=None):
+def get_signatures(builddir, failsafe=False, machine=None, extravars=None):
    import re
    # some recipes needs to be excluded like meta-world-pkgdata
@@ -267,13 +298,16 @@ def get_signatures(builddir, failsafe=False, machine=None):
    sigs = {}
    tune2tasks = {}
-    cmd = 'BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
+    cmd = 'BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS BB_SIGNATURE_HANDLER" BB_SIGNATURE_HANDLER="OEBasicHash" '
+    if extravars:
+        cmd += extravars
+        cmd += ' '
    if machine:
        cmd += 'MACHINE=%s ' % machine
    cmd += 'bitbake '
    if failsafe:
        cmd += '-k '
-    cmd += '-S none world'
+    cmd += '-S lockedsigs world'
    sigs_file = os.path.join(builddir, 'locked-sigs.inc')
    if os.path.exists(sigs_file):
        os.unlink(sigs_file)
@@ -290,8 +324,8 @@ def get_signatures(builddir, failsafe=False, machine=None):
        else:
            raise
-    sig_regex = re.compile("^(?P<task>.*:.*):(?P<hash>.*) .$")
+    sig_regex = re.compile(r"^(?P<task>.*:.*):(?P<hash>.*) .$")
-    tune_regex = re.compile("(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
+    tune_regex = re.compile(r"(^|\s)SIGGEN_LOCKEDSIGS_t-(?P<tune>\S*)\s*=\s*")
    current_tune = None
    with open(sigs_file, 'r') as f:
        for line in f.readlines():
@@ -418,3 +452,15 @@ def compare_signatures(old_sigs, curr_sigs):
            msg.extend(['      ' + line for line in output.splitlines()])
            msg.append('')
    return '\n'.join(msg)
+def get_git_toplevel(directory):
+    """
+    Try and find the top of the git repository that directory might be in.
+    Returns the top-level directory, or None.
+    """
+    cmd = ["git", "-C", directory, "rev-parse", "--show-toplevel"]
+    try:
+        return subprocess.check_output(cmd, text=True).strip()
+    except:
+        return None
diff --git a/scripts/lib/checklayer/cases/bsp.py b/scripts/lib/checklayer/cases/bsp.py
index 7fd56f5d36..b76163fb56 100644
--- a/scripts/lib/checklayer/cases/bsp.py
+++ b/scripts/lib/checklayer/cases/bsp.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
 class BSPCheckLayer(OECheckLayerTestCase):
    @classmethod
    def setUpClass(self):
-        if self.tc.layer['type'] != LayerType.BSP:
+        if self.tc.layer['type'] not in (LayerType.BSP, LayerType.CORE):
            raise unittest.SkipTest("BSPCheckLayer: Layer %s isn't BSP one." %\
                self.tc.layer['name'])
@@ -153,7 +153,7 @@ class BSPCheckLayer(OECheckLayerTestCase):
                # do_build can be ignored: it is know to have
                # different signatures in some cases, for example in
                # the allarch ca-certificates due to RDEPENDS=openssl.
-                # That particular dependency is whitelisted via
+                # That particular dependency is marked via
                # SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS, but still shows up
                # in the sstate signature hash because filtering it
                # out would be hard and running do_build multiple
diff --git a/scripts/lib/checklayer/cases/common.py b/scripts/lib/checklayer/cases/common.py
index b82304e361..ddead69a7b 100644
--- a/scripts/lib/checklayer/cases/common.py
+++ b/scripts/lib/checklayer/cases/common.py
@@ -6,15 +6,19 @@
 import glob
 import os
 import unittest
-from checklayer import get_signatures, LayerType, check_command, get_depgraph, compare_signatures
+import re
+from checklayer import get_signatures, LayerType, check_command, compare_signatures, get_git_toplevel
 from checklayer.case import OECheckLayerTestCase
 class CommonCheckLayer(OECheckLayerTestCase):
    def test_readme(self):
+        if self.tc.layer['type'] == LayerType.CORE:
+            raise unittest.SkipTest("Core layer's README is top level")
        # The top-level README file may have a suffix (like README.rst or README.txt).
        readme_files = glob.glob(os.path.join(self.tc.layer['path'], '[Rr][Ee][Aa][Dd][Mm][Ee]*'))
        self.assertTrue(len(readme_files) > 0,
-                        msg="Layer doesn't contains README file.")
+                        msg="Layer doesn't contain a README file.")
        # There might be more than one file matching the file pattern above
        # (for example, README.rst and README-COPYING.rst). The one with the shortest
@@ -26,6 +30,48 @@ class CommonCheckLayer(OECheckLayerTestCase):
        self.assertTrue(data,
                msg="Layer contains a README file but it is empty.")
+        # If a layer's README references another README, then the checks below are not valid
+        if re.search('README', data, re.IGNORECASE):
+            return
+        self.assertIn('maintainer', data.lower())
+        self.assertIn('patch', data.lower())
+        # Check that there is an email address in the README
+        email_regex = re.compile(r"[^@]+@[^@]+")
+        self.assertTrue(email_regex.match(data))
+    def find_file_by_name(self, globs):
+        """
+        Utility function to find a file that matches the specified list of
+        globs, in either the layer directory itself or the repository top-level
+        directory.
+        """
+        directories = [self.tc.layer["path"]]
+        toplevel = get_git_toplevel(directories[0])
+        if toplevel:
+            directories.append(toplevel)
+        for path in directories:
+            for name in globs:
+                files = glob.glob(os.path.join(path, name))
+                if files:
+                    return sorted(files)[0]
+        return None
+    def test_security(self):
+        """
+        Test that the layer has a SECURITY.md (or similar) file, either in the
+        layer itself or at the top of the containing git repository.
+        """
+        if self.tc.layer["type"] == LayerType.CORE:
+            raise unittest.SkipTest("Core layer's SECURITY is top level")
+        filename = self.find_file_by_name(("SECURITY", "SECURITY.*"))
+        self.assertTrue(filename, msg="Layer doesn't contain a SECURITY.md file.")
+        size = os.path.getsize(filename)
+        self.assertGreater(size, 0, msg=f"{filename} has no content.")
    def test_parse(self):
        check_command('Layer %s failed to parse.' % self.tc.layer['name'],
                      'bitbake -p')
@@ -43,6 +89,35 @@ class CommonCheckLayer(OECheckLayerTestCase):
        '''
        get_signatures(self.td['builddir'], failsafe=False)
+    def test_world_inherit_class(self):
+        '''
+        This also does "bitbake -S none world" along with inheriting "yocto-check-layer"
+        class, which can do additional per-recipe test cases.
+        '''
+        msg = []
+        try:
+            get_signatures(self.td['builddir'], failsafe=False, machine=None, extravars='BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS INHERIT" INHERIT="yocto-check-layer"')
+        except RuntimeError as ex:
+            msg.append(str(ex))
+        if msg:
+            msg.insert(0, 'Layer %s failed additional checks from yocto-check-layer.bbclass\nSee below log for specific recipe parsing errors:\n' % \
+                self.tc.layer['name'])
+            self.fail('\n'.join(msg))
+    def test_patches_upstream_status(self):
+        import sys
+        sys.path.append(os.path.join(sys.path[0], '../../../../meta/lib/'))
+        import oe.qa
+        patches = []
+        for dirpath, dirs, files in os.walk(self.tc.layer['path']):
+            for filename in files:
+                if filename.endswith(".patch"):
+                    ppath = os.path.join(dirpath, filename)
+                    if oe.qa.check_upstream_status(ppath):
+                        patches.append(ppath)
+        self.assertEqual(len(patches), 0 , \
+                msg="Found following patches with malformed or missing upstream status:\n%s" % '\n'.join([str(patch) for patch in patches]))
    def test_signatures(self):
        if self.tc.layer['type'] == LayerType.SOFTWARE and \
           not self.tc.test_software_layer_signatures:
diff --git a/scripts/lib/checklayer/cases/distro.py b/scripts/lib/checklayer/cases/distro.py
index f0bee5493c..a35332451c 100644
--- a/scripts/lib/checklayer/cases/distro.py
+++ b/scripts/lib/checklayer/cases/distro.py
@@ -11,7 +11,7 @@ from checklayer.case import OECheckLayerTestCase
 class DistroCheckLayer(OECheckLayerTestCase):
    @classmethod
    def setUpClass(self):
-        if self.tc.layer['type'] != LayerType.DISTRO:
+        if self.tc.layer['type'] not in (LayerType.DISTRO, LayerType.CORE):
            raise unittest.SkipTest("DistroCheckLayer: Layer %s isn't Distro one." %\
                self.tc.layer['name'])
diff --git a/scripts/lib/devtool/__init__.py b/scripts/lib/devtool/__init__.py
index 702db669de..fa6e1a34fd 100644
--- a/scripts/lib/devtool/__init__.py
+++ b/scripts/lib/devtool/__init__.py
@@ -78,12 +78,15 @@ def exec_fakeroot(d, cmd, **kwargs):
    """Run a command under fakeroot (pseudo, in fact) so that it picks up the appropriate file permissions"""
    # Grab the command and check it actually exists
    fakerootcmd = d.getVar('FAKEROOTCMD')
+    fakerootenv = d.getVar('FAKEROOTENV')
+    exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, kwargs)
+def exec_fakeroot_no_d(fakerootcmd, fakerootenv, cmd, **kwargs):
    if not os.path.exists(fakerootcmd):
        logger.error('pseudo executable %s could not be found - have you run a build yet? pseudo-native should install this and if you have run any build then that should have been built')
        return 2
    # Set up the appropriate environment
    newenv = dict(os.environ)
-    fakerootenv = d.getVar('FAKEROOTENV')
    for varvalue in fakerootenv.split():
        if '=' in varvalue:
            splitval = varvalue.split('=', 1)
@@ -231,7 +234,29 @@ def setup_git_repo(repodir, version, devbranch, basetag='devtool-base', d=None):
            f.write(line)
    bb.process.run('git checkout -b %s' % devbranch, cwd=repodir)
-    bb.process.run('git tag -f %s' % basetag, cwd=repodir)
+    bb.process.run('git tag -f --no-sign %s' % basetag, cwd=repodir)
+    # if recipe unpacks another git repo inside S, we need to declare it as a regular git submodule now,
+    # so we will be able to tag branches on it and extract patches when doing finish/update on the recipe
+    stdout, _ = bb.process.run("git status --porcelain", cwd=repodir)
+    found = False
+    for line in stdout.splitlines():
+        if line.endswith("/"):
+            new_dir = line.split()[1]
+            for root, dirs, files in os.walk(os.path.join(repodir, new_dir)):
+                if ".git" in dirs + files:
+                    (stdout, _) = bb.process.run('git remote', cwd=root)
+                    remote = stdout.splitlines()[0]
+                    (stdout, _) = bb.process.run('git remote get-url %s' % remote, cwd=root)
+                    remote_url = stdout.splitlines()[0]
+                    logger.error(os.path.relpath(os.path.join(root, ".."), root))
+                    bb.process.run('git submodule add %s %s' % (remote_url, os.path.relpath(root, os.path.join(root, ".."))), cwd=os.path.join(root, ".."))
+                    found = True
+                if found:
+                    oe.patch.GitApplyTree.commitIgnored("Add additional submodule from SRC_URI", dir=os.path.join(root, ".."), d=d)
+                    found = False
+    if os.path.exists(os.path.join(repodir, '.gitmodules')):
+        bb.process.run('git submodule foreach --recursive  "git tag -f --no-sign %s"' % basetag, cwd=repodir)
 def recipe_to_append(recipefile, config, wildcard=False):
    """
diff --git a/scripts/lib/devtool/build.py b/scripts/lib/devtool/build.py
index 935ffab46c..0b2c3d33dc 100644
--- a/scripts/lib/devtool/build.py
+++ b/scripts/lib/devtool/build.py
@@ -49,7 +49,7 @@ def build(args, config, basepath, workspace):
        rd = parse_recipe(config, tinfoil, args.recipename, appends=True, filter_workspace=False)
        if not rd:
            return 1
-        deploytask = 'do_deploy' in rd.getVar('__BBTASKS')
+        deploytask = 'do_deploy' in bb.build.listtasks(rd)
    finally:
        tinfoil.shutdown()
diff --git a/scripts/lib/devtool/build_image.py b/scripts/lib/devtool/build_image.py
index 9388abbacf..980f90ddd6 100644
--- a/scripts/lib/devtool/build_image.py
+++ b/scripts/lib/devtool/build_image.py
@@ -113,7 +113,7 @@ def build_image_task(config, basepath, workspace, image, add_packages=None, task
                with open(appendfile, 'w') as afile:
                    if packages:
                        # include packages from workspace recipes into the image
-                        afile.write('IMAGE_INSTALL_append = " %s"\n' % ' '.join(packages))
+                        afile.write('IMAGE_INSTALL:append = " %s"\n' % ' '.join(packages))
                        if not task:
                            logger.info('Building image %s with the following '
                                        'additional packages: %s', image, ' '.join(packages))
diff --git a/scripts/lib/devtool/build_sdk.py b/scripts/lib/devtool/build_sdk.py
index 6fe02fff2a..990303982c 100644
--- a/scripts/lib/devtool/build_sdk.py
+++ b/scripts/lib/devtool/build_sdk.py
@@ -5,15 +5,8 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
-import os
-import subprocess
 import logging
-import glob
+from devtool import DevtoolError
-import shutil
-import errno
-import sys
-import tempfile
-from devtool import exec_build_env_command, setup_tinfoil, parse_recipe, DevtoolError
 from devtool import build_image
 logger = logging.getLogger('devtool')
diff --git a/scripts/lib/devtool/deploy.py b/scripts/lib/devtool/deploy.py
index e5af2c95ae..b5ca8f2c2f 100644
--- a/scripts/lib/devtool/deploy.py
+++ b/scripts/lib/devtool/deploy.py
@@ -16,7 +16,7 @@ import bb.utils
 import argparse_oe
 import oe.types
-from devtool import exec_fakeroot, setup_tinfoil, check_workspace_recipe, DevtoolError
+from devtool import exec_fakeroot_no_d, setup_tinfoil, check_workspace_recipe, DevtoolError
 logger = logging.getLogger('devtool')
@@ -133,16 +133,38 @@ def _prepare_remote_script(deploy, verbose=False, dryrun=False, undeployall=Fals
    return '\n'.join(lines)
 def deploy(args, config, basepath, workspace):
    """Entry point for the devtool 'deploy' subcommand"""
-    import math
+    import oe.utils
-    import oe.recipeutils
-    import oe.package
    check_workspace_recipe(workspace, args.recipename, checksrc=False)
+    tinfoil = setup_tinfoil(basepath=basepath)
+    try:
+        try:
+            rd = tinfoil.parse_recipe(args.recipename)
+        except Exception as e:
+            raise DevtoolError('Exception parsing recipe %s: %s' %
+                            (args.recipename, e))
+        
+        srcdir = rd.getVar('D')
+        workdir = rd.getVar('WORKDIR')
+        path = rd.getVar('PATH')
+        strip_cmd = rd.getVar('STRIP')
+        libdir = rd.getVar('libdir')
+        base_libdir = rd.getVar('base_libdir')
+        max_process = oe.utils.get_bb_number_threads(rd)
+        fakerootcmd = rd.getVar('FAKEROOTCMD')
+        fakerootenv = rd.getVar('FAKEROOTENV')
+    finally:
+        tinfoil.shutdown()
+    return deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args)
+def deploy_no_d(srcdir, workdir, path, strip_cmd, libdir, base_libdir, max_process, fakerootcmd, fakerootenv, args):
+    import math
+    import oe.package
    try:
        host, destdir = args.target.split(':')
    except ValueError:
@@ -152,118 +174,108 @@ def deploy(args, config, basepath, workspace):
    if not destdir.endswith('/'):
        destdir += '/'
-    tinfoil = setup_tinfoil(basepath=basepath)
+    recipe_outdir = srcdir
-    try:
+    if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
-        try:
+        raise DevtoolError('No files to deploy - have you built the %s '
-            rd = tinfoil.parse_recipe(args.recipename)
+                        'recipe? If so, the install step has not installed '
-        except Exception as e:
+                        'any files.' % args.recipename)
-            raise DevtoolError('Exception parsing recipe %s: %s' %
-                            (args.recipename, e))
+    if args.strip and not args.dry_run:
-        recipe_outdir = rd.getVar('D')
+        # Fakeroot copy to new destination
-        if not os.path.exists(recipe_outdir) or not os.listdir(recipe_outdir):
+        srcdir = recipe_outdir
-            raise DevtoolError('No files to deploy - have you built the %s '
+        recipe_outdir = os.path.join(workdir, 'devtool-deploy-target-stripped')
-                            'recipe? If so, the install step has not installed '
+        if os.path.isdir(recipe_outdir):
-                            'any files.' % args.recipename)
+            exec_fakeroot_no_d(fakerootcmd, fakerootenv, "rm -rf %s" % recipe_outdir, shell=True)
+        exec_fakeroot_no_d(fakerootcmd, fakerootenv, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
-        if args.strip and not args.dry_run:
+        os.environ['PATH'] = ':'.join([os.environ['PATH'], path or ''])
-            # Fakeroot copy to new destination
+        oe.package.strip_execs(args.recipename, recipe_outdir, strip_cmd, libdir, base_libdir, max_process)
-            srcdir = recipe_outdir
-            recipe_outdir = os.path.join(rd.getVar('WORKDIR'), 'deploy-target-stripped')
+    filelist = []
-            if os.path.isdir(recipe_outdir):
+    inodes = set({})
-                bb.utils.remove(recipe_outdir, True)
+    ftotalsize = 0
-            exec_fakeroot(rd, "cp -af %s %s" % (os.path.join(srcdir, '.'), recipe_outdir), shell=True)
+    for root, _, files in os.walk(recipe_outdir):
-            os.environ['PATH'] = ':'.join([os.environ['PATH'], rd.getVar('PATH') or ''])
+        for fn in files:
-            oe.package.strip_execs(args.recipename, recipe_outdir, rd.getVar('STRIP'), rd.getVar('libdir'),
+            fstat = os.lstat(os.path.join(root, fn))
-                        rd.getVar('base_libdir'), rd)
+            # Get the size in kiB (since we'll be comparing it to the output of du -k)
+            # MUST use lstat() here not stat() or getfilesize() since we don't want to
-        filelist = []
+            # dereference symlinks
-        inodes = set({})
+            if fstat.st_ino in inodes:
-        ftotalsize = 0
+                fsize = 0
-        for root, _, files in os.walk(recipe_outdir):
+            else:
-            for fn in files:
+                fsize = int(math.ceil(float(fstat.st_size)/1024))
-                fstat = os.lstat(os.path.join(root, fn))
+            inodes.add(fstat.st_ino)
-                # Get the size in kiB (since we'll be comparing it to the output of du -k)
+            ftotalsize += fsize
-                # MUST use lstat() here not stat() or getfilesize() since we don't want to
+            # The path as it would appear on the target
-                # dereference symlinks
+            fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
-                if fstat.st_ino in inodes:
+            filelist.append((fpath, fsize))
-                    fsize = 0
-                else:
+    if args.dry_run:
-                    fsize = int(math.ceil(float(fstat.st_size)/1024))
+        print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
-                inodes.add(fstat.st_ino)
+        for item, _ in filelist:
-                ftotalsize += fsize
+            print('  %s' % item)
-                # The path as it would appear on the target
+        return 0
-                fpath = os.path.join(destdir, os.path.relpath(root, recipe_outdir), fn)
-                filelist.append((fpath, fsize))
-        if args.dry_run:
-            print('Files to be deployed for %s on target %s:' % (args.recipename, args.target))
-            for item, _ in filelist:
-                print('  %s' % item)
-            return 0
-        extraoptions = ''
-        if args.no_host_check:
-            extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-        if not args.show_status:
-            extraoptions += ' -q'
-        scp_sshexec = ''
-        ssh_sshexec = 'ssh'
-        if args.ssh_exec:
-            scp_sshexec = "-S %s" % args.ssh_exec
-            ssh_sshexec = args.ssh_exec
-        scp_port = ''
-        ssh_port = ''
-        if args.port:
-            scp_port = "-P %s" % args.port
-            ssh_port = "-p %s" % args.port
-        if args.key:
-            extraoptions += ' -i %s' % args.key
-        # In order to delete previously deployed files and have the manifest file on
-        # the target, we write out a shell script and then copy it to the target
-        # so we can then run it (piping tar output to it).
-        # (We cannot use scp here, because it doesn't preserve symlinks.)
-        tmpdir = tempfile.mkdtemp(prefix='devtool')
-        try:
-            tmpscript = '/tmp/devtool_deploy.sh'
-            tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
-            shellscript = _prepare_remote_script(deploy=True,
-                                                verbose=args.show_status,
-                                                nopreserve=args.no_preserve,
-                                                nocheckspace=args.no_check_space)
-            # Write out the script to a file
-            with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
-                f.write(shellscript)
-            # Write out the file list
-            with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
-                f.write('%d\n' % ftotalsize)
-                for fpath, fsize in filelist:
-                    f.write('%s %d\n' % (fpath, fsize))
-            # Copy them to the target
-            ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
-            if ret != 0:
-                raise DevtoolError('Failed to copy script to %s - rerun with -s to '
-                                'get a complete error message' % args.target)
-        finally:
-            shutil.rmtree(tmpdir)
-        # Now run the script
+    extraoptions = ''
-        ret = exec_fakeroot(rd, 'tar cf - . | %s  %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
+    if args.no_host_check:
-        if ret != 0:
+        extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
-            raise DevtoolError('Deploy failed - rerun with -s to get a complete '
+    if not args.show_status:
-                            'error message')
+        extraoptions += ' -q'
-        logger.info('Successfully deployed %s' % recipe_outdir)
+    scp_sshexec = ''
+    ssh_sshexec = 'ssh'
+    if args.ssh_exec:
+        scp_sshexec = "-S %s" % args.ssh_exec
+        ssh_sshexec = args.ssh_exec
+    scp_port = ''
+    ssh_port = ''
+    if args.port:
+        scp_port = "-P %s" % args.port
+        ssh_port = "-p %s" % args.port
+    if args.key:
+        extraoptions += ' -i %s' % args.key
-        files_list = []
+    # In order to delete previously deployed files and have the manifest file on
-        for root, _, files in os.walk(recipe_outdir):
+    # the target, we write out a shell script and then copy it to the target
-            for filename in files:
+    # so we can then run it (piping tar output to it).
-                filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
+    # (We cannot use scp here, because it doesn't preserve symlinks.)
-                files_list.append(os.path.join(destdir, filename))
+    tmpdir = tempfile.mkdtemp(prefix='devtool')
+    try:
+        tmpscript = '/tmp/devtool_deploy.sh'
+        tmpfilelist = os.path.join(os.path.dirname(tmpscript), 'devtool_deploy.list')
+        shellscript = _prepare_remote_script(deploy=True,
+                                            verbose=args.show_status,
+                                            nopreserve=args.no_preserve,
+                                            nocheckspace=args.no_check_space)
+        # Write out the script to a file
+        with open(os.path.join(tmpdir, os.path.basename(tmpscript)), 'w') as f:
+            f.write(shellscript)
+        # Write out the file list
+        with open(os.path.join(tmpdir, os.path.basename(tmpfilelist)), 'w') as f:
+            f.write('%d\n' % ftotalsize)
+            for fpath, fsize in filelist:
+                f.write('%s %d\n' % (fpath, fsize))
+        # Copy them to the target
+        ret = subprocess.call("scp %s %s %s %s/* %s:%s" % (scp_sshexec, scp_port, extraoptions, tmpdir, args.target, os.path.dirname(tmpscript)), shell=True)
+        if ret != 0:
+            raise DevtoolError('Failed to copy script to %s - rerun with -s to '
+                            'get a complete error message' % args.target)
    finally:
-        tinfoil.shutdown()
+        shutil.rmtree(tmpdir)
+    # Now run the script
+    ret = exec_fakeroot_no_d(fakerootcmd, fakerootenv, 'tar cf - . | %s  %s %s %s \'sh %s %s %s %s\'' % (ssh_sshexec, ssh_port, extraoptions, args.target, tmpscript, args.recipename, destdir, tmpfilelist), cwd=recipe_outdir, shell=True)
+    if ret != 0:
+        raise DevtoolError('Deploy failed - rerun with -s to get a complete '
+                        'error message')
+    logger.info('Successfully deployed %s' % recipe_outdir)
+    files_list = []
+    for root, _, files in os.walk(recipe_outdir):
+        for filename in files:
+            filename = os.path.relpath(os.path.join(root, filename), recipe_outdir)
+            files_list.append(os.path.join(destdir, filename))
    return 0
diff --git a/scripts/lib/devtool/ide_plugins/__init__.py b/scripts/lib/devtool/ide_plugins/__init__.py
new file mode 100644
index 0000000000..19c2f61c5f
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/__init__.py
@@ -0,0 +1,282 @@
+#
+# Copyright (C) 2023-2024 Siemens AG
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""Devtool ide-sdk IDE plugin interface definition and helper functions"""
+import errno
+import json
+import logging
+import os
+import stat
+from enum import Enum, auto
+from devtool import DevtoolError
+from bb.utils import mkdirhier
+logger = logging.getLogger('devtool')
+class BuildTool(Enum):
+    UNDEFINED = auto()
+    CMAKE = auto()
+    MESON = auto()
+    @property
+    def is_c_ccp(self):
+        if self is BuildTool.CMAKE:
+            return True
+        if self is BuildTool.MESON:
+            return True
+        return False
+class GdbCrossConfig:
+    """Base class defining the GDB configuration generator interface
+    Generate a GDB configuration for a binary on the target device.
+    Only one instance per binary is allowed. This allows to assign unique port
+    numbers for all gdbserver instances.
+    """
+    _gdbserver_port_next = 1234
+    _binaries = []
+    def __init__(self, image_recipe, modified_recipe, binary, gdbserver_multi=True):
+        self.image_recipe = image_recipe
+        self.modified_recipe = modified_recipe
+        self.gdb_cross = modified_recipe.gdb_cross
+        self.binary = binary
+        if binary in GdbCrossConfig._binaries:
+            raise DevtoolError(
+                "gdbserver config for binary %s is already generated" % binary)
+        GdbCrossConfig._binaries.append(binary)
+        self.script_dir = modified_recipe.ide_sdk_scripts_dir
+        self.gdbinit_dir = os.path.join(self.script_dir, 'gdbinit')
+        self.gdbserver_multi = gdbserver_multi
+        self.binary_pretty = self.binary.replace(os.sep, '-').lstrip('-')
+        self.gdbserver_port = GdbCrossConfig._gdbserver_port_next
+        GdbCrossConfig._gdbserver_port_next += 1
+        self.id_pretty = "%d_%s" % (self.gdbserver_port, self.binary_pretty)
+        # gdbserver start script
+        gdbserver_script_file = 'gdbserver_' + self.id_pretty
+        if self.gdbserver_multi:
+            gdbserver_script_file += "_m"
+        self.gdbserver_script = os.path.join(
+            self.script_dir, gdbserver_script_file)
+        # gdbinit file
+        self.gdbinit = os.path.join(
+            self.gdbinit_dir, 'gdbinit_' + self.id_pretty)
+        # gdb start script
+        self.gdb_script = os.path.join(
+            self.script_dir, 'gdb_' + self.id_pretty)
+    def _gen_gdbserver_start_script(self):
+        """Generate a shell command starting the gdbserver on the remote device via ssh
+        GDB supports two modes:
+        multi: gdbserver remains running over several debug sessions
+        once: gdbserver terminates after the debugged process terminates
+        """
+        cmd_lines = ['#!/bin/sh']
+        if self.gdbserver_multi:
+            temp_dir = "TEMP_DIR=/tmp/gdbserver_%s; " % self.id_pretty
+            gdbserver_cmd_start = temp_dir
+            gdbserver_cmd_start += "test -f \\$TEMP_DIR/pid && exit 0; "
+            gdbserver_cmd_start += "mkdir -p \\$TEMP_DIR; "
+            gdbserver_cmd_start += "%s --multi :%s > \\$TEMP_DIR/log 2>&1 & " % (
+                self.gdb_cross.gdbserver_path, self.gdbserver_port)
+            gdbserver_cmd_start += "echo \\$! > \\$TEMP_DIR/pid;"
+            gdbserver_cmd_stop = temp_dir
+            gdbserver_cmd_stop += "test -f \\$TEMP_DIR/pid && kill \\$(cat \\$TEMP_DIR/pid); "
+            gdbserver_cmd_stop += "rm -rf \\$TEMP_DIR; "
+            gdbserver_cmd_l = []
+            gdbserver_cmd_l.append('if [ "$1" = "stop" ]; then')
+            gdbserver_cmd_l.append('  shift')
+            gdbserver_cmd_l.append("  %s %s %s %s 'sh -c \"%s\"'" % (
+                self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_stop))
+            gdbserver_cmd_l.append('else')
+            gdbserver_cmd_l.append("  %s %s %s %s 'sh -c \"%s\"'" % (
+                self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start))
+            gdbserver_cmd_l.append('fi')
+            gdbserver_cmd = os.linesep.join(gdbserver_cmd_l)
+        else:
+            gdbserver_cmd_start = "%s --once :%s %s" % (
+                self.gdb_cross.gdbserver_path, self.gdbserver_port, self.binary)
+            gdbserver_cmd = "%s %s %s %s 'sh -c \"%s\"'" % (
+                self.gdb_cross.target_device.ssh_sshexec, self.gdb_cross.target_device.ssh_port, self.gdb_cross.target_device.extraoptions, self.gdb_cross.target_device.target, gdbserver_cmd_start)
+        cmd_lines.append(gdbserver_cmd)
+        GdbCrossConfig.write_file(self.gdbserver_script, cmd_lines, True)
+    def _gen_gdbinit_config(self):
+        """Generate a gdbinit file for this binary and the corresponding gdbserver configuration"""
+        gdbinit_lines = ['# This file is generated by devtool ide-sdk']
+        if self.gdbserver_multi:
+            target_help = '#   gdbserver --multi :%d' % self.gdbserver_port
+            remote_cmd = 'target extended-remote'
+        else:
+            target_help = '#   gdbserver :%d %s' % (
+                self.gdbserver_port, self.binary)
+            remote_cmd = 'target remote'
+        gdbinit_lines.append('# On the remote target:')
+        gdbinit_lines.append(target_help)
+        gdbinit_lines.append('# On the build machine:')
+        gdbinit_lines.append('#   cd ' + self.modified_recipe.real_srctree)
+        gdbinit_lines.append(
+            '#   ' + self.gdb_cross.gdb + ' -ix ' + self.gdbinit)
+        gdbinit_lines.append('set sysroot ' + self.modified_recipe.d)
+        gdbinit_lines.append('set substitute-path "/usr/include" "' +
+                             os.path.join(self.modified_recipe.recipe_sysroot, 'usr', 'include') + '"')
+        # Disable debuginfod for now, the IDE configuration uses rootfs-dbg from the image workdir.
+        gdbinit_lines.append('set debuginfod enabled off')
+        if self.image_recipe.rootfs_dbg:
+            gdbinit_lines.append(
+                'set solib-search-path "' + self.modified_recipe.solib_search_path_str(self.image_recipe) + '"')
+            # First: Search for sources of this recipe in the workspace folder
+            if self.modified_recipe.pn in self.modified_recipe.target_dbgsrc_dir:
+                gdbinit_lines.append('set substitute-path "%s" "%s"' %
+                                     (self.modified_recipe.target_dbgsrc_dir, self.modified_recipe.real_srctree))
+            else:
+                logger.error(
+                    "TARGET_DBGSRC_DIR must contain the recipe name PN.")
+            # Second: Search for sources of other recipes in the rootfs-dbg
+            if self.modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"):
+                gdbinit_lines.append('set substitute-path "/usr/src/debug" "%s"' % os.path.join(
+                    self.image_recipe.rootfs_dbg, "usr", "src", "debug"))
+            else:
+                logger.error(
+                    "TARGET_DBGSRC_DIR must start with /usr/src/debug.")
+        else:
+            logger.warning(
+                "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.")
+        gdbinit_lines.append(
+            '%s %s:%d' % (remote_cmd, self.gdb_cross.host, self.gdbserver_port))
+        gdbinit_lines.append('set remote exec-file ' + self.binary)
+        gdbinit_lines.append(
+            'run ' + os.path.join(self.modified_recipe.d, self.binary))
+        GdbCrossConfig.write_file(self.gdbinit, gdbinit_lines)
+    def _gen_gdb_start_script(self):
+        """Generate a script starting GDB with the corresponding gdbinit configuration."""
+        cmd_lines = ['#!/bin/sh']
+        cmd_lines.append('cd ' + self.modified_recipe.real_srctree)
+        cmd_lines.append(self.gdb_cross.gdb + ' -ix ' +
+                         self.gdbinit + ' "$@"')
+        GdbCrossConfig.write_file(self.gdb_script, cmd_lines, True)
+    def initialize(self):
+        self._gen_gdbserver_start_script()
+        self._gen_gdbinit_config()
+        self._gen_gdb_start_script()
+    @staticmethod
+    def write_file(script_file, cmd_lines, executable=False):
+        script_dir = os.path.dirname(script_file)
+        mkdirhier(script_dir)
+        with open(script_file, 'w') as script_f:
+            script_f.write(os.linesep.join(cmd_lines))
+            script_f.write(os.linesep)
+        if executable:
+            st = os.stat(script_file)
+            os.chmod(script_file, st.st_mode | stat.S_IEXEC)
+        logger.info("Created: %s" % script_file)
+class IdeBase:
+    """Base class defining the interface for IDE plugins"""
+    def __init__(self):
+        self.ide_name = 'undefined'
+        self.gdb_cross_configs = []
+    @classmethod
+    def ide_plugin_priority(cls):
+        """Used to find the default ide handler if --ide is not passed"""
+        return 10
+    def setup_shared_sysroots(self, shared_env):
+        logger.warn("Shared sysroot mode is not supported for IDE %s" %
+                    self.ide_name)
+    def setup_modified_recipe(self, args, image_recipe, modified_recipe):
+        logger.warn("Modified recipe mode is not supported for IDE %s" %
+                    self.ide_name)
+    def initialize_gdb_cross_configs(self, image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfig):
+        binaries = modified_recipe.find_installed_binaries()
+        for binary in binaries:
+            gdb_cross_config = gdb_cross_config_class(
+                image_recipe, modified_recipe, binary)
+            gdb_cross_config.initialize()
+            self.gdb_cross_configs.append(gdb_cross_config)
+    @staticmethod
+    def gen_oe_scrtips_sym_link(modified_recipe):
+        # create a sym-link from sources to the scripts directory
+        if os.path.isdir(modified_recipe.ide_sdk_scripts_dir):
+            IdeBase.symlink_force(modified_recipe.ide_sdk_scripts_dir,
+                                  os.path.join(modified_recipe.real_srctree, 'oe-scripts'))
+    @staticmethod
+    def update_json_file(json_dir, json_file, update_dict):
+        """Update a json file
+        By default it uses the dict.update function. If this is not sutiable
+        the update function might be passed via update_func parameter.
+        """
+        json_path = os.path.join(json_dir, json_file)
+        logger.info("Updating IDE config file: %s (%s)" %
+                    (json_file, json_path))
+        if not os.path.exists(json_dir):
+            os.makedirs(json_dir)
+        try:
+            with open(json_path) as f:
+                orig_dict = json.load(f)
+        except json.decoder.JSONDecodeError:
+            logger.info(
+                "Decoding %s failed. Probably because of comments in the json file" % json_path)
+            orig_dict = {}
+        except FileNotFoundError:
+            orig_dict = {}
+        orig_dict.update(update_dict)
+        with open(json_path, 'w') as f:
+            json.dump(orig_dict, f, indent=4)
+    @staticmethod
+    def symlink_force(tgt, dst):
+        try:
+            os.symlink(tgt, dst)
+        except OSError as err:
+            if err.errno == errno.EEXIST:
+                if os.readlink(dst) != tgt:
+                    os.remove(dst)
+                    os.symlink(tgt, dst)
+            else:
+                raise err
+def get_devtool_deploy_opts(args):
+    """Filter args for devtool deploy-target args"""
+    if not args.target:
+        return None
+    devtool_deploy_opts = [args.target]
+    if args.no_host_check:
+        devtool_deploy_opts += ["-c"]
+    if args.show_status:
+        devtool_deploy_opts += ["-s"]
+    if args.no_preserve:
+        devtool_deploy_opts += ["-p"]
+    if args.no_check_space:
+        devtool_deploy_opts += ["--no-check-space"]
+    if args.ssh_exec:
+        devtool_deploy_opts += ["-e", args.ssh.exec]
+    if args.port:
+        devtool_deploy_opts += ["-P", args.port]
+    if args.key:
+        devtool_deploy_opts += ["-I", args.key]
+    if args.strip is False:
+        devtool_deploy_opts += ["--no-strip"]
+    return devtool_deploy_opts
diff --git a/scripts/lib/devtool/ide_plugins/ide_code.py b/scripts/lib/devtool/ide_plugins/ide_code.py
new file mode 100644
index 0000000000..ee5bb57265
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/ide_code.py
@@ -0,0 +1,462 @@
+#
+# Copyright (C) 2023-2024 Siemens AG
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""Devtool ide-sdk IDE plugin for VSCode and VSCodium"""
+import json
+import logging
+import os
+import shutil
+from devtool.ide_plugins import BuildTool, IdeBase, GdbCrossConfig, get_devtool_deploy_opts
+logger = logging.getLogger('devtool')
+class GdbCrossConfigVSCode(GdbCrossConfig):
+    def __init__(self, image_recipe, modified_recipe, binary):
+        super().__init__(image_recipe, modified_recipe, binary, False)
+    def initialize(self):
+        self._gen_gdbserver_start_script()
+class IdeVSCode(IdeBase):
+    """Manage IDE configurations for VSCode
+    Modified recipe mode:
+    - cmake: use the cmake-preset generated by devtool ide-sdk
+    - meson: meson is called via a wrapper script generated by devtool ide-sdk
+    Shared sysroot mode:
+    In shared sysroot mode, the cross tool-chain is exported to the user's global configuration.
+    A workspace cannot be created because there is no recipe that defines how a workspace could
+    be set up.
+    - cmake: adds a cmake-kit to .local/share/CMakeTools/cmake-tools-kits.json
+             The cmake-kit uses the environment script and the tool-chain file
+             generated by meta-ide-support.
+    - meson: Meson needs manual workspace configuration.
+    """
+    @classmethod
+    def ide_plugin_priority(cls):
+        """If --ide is not passed this is the default plugin"""
+        if shutil.which('code'):
+            return 100
+        return 0
+    def setup_shared_sysroots(self, shared_env):
+        """Expose the toolchain of the shared sysroots SDK"""
+        datadir = shared_env.ide_support.datadir
+        deploy_dir_image = shared_env.ide_support.deploy_dir_image
+        real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys
+        standalone_sysroot_native = shared_env.build_sysroots.standalone_sysroot_native
+        vscode_ws_path = os.path.join(
+            os.environ['HOME'], '.local', 'share', 'CMakeTools')
+        cmake_kits_path = os.path.join(vscode_ws_path, 'cmake-tools-kits.json')
+        oecmake_generator = "Ninja"
+        env_script = os.path.join(
+            deploy_dir_image, 'environment-setup-' + real_multimach_target_sys)
+        if not os.path.isdir(vscode_ws_path):
+            os.makedirs(vscode_ws_path)
+        cmake_kits_old = []
+        if os.path.exists(cmake_kits_path):
+            with open(cmake_kits_path, 'r', encoding='utf-8') as cmake_kits_file:
+                cmake_kits_old = json.load(cmake_kits_file)
+        cmake_kits = cmake_kits_old.copy()
+        cmake_kit_new = {
+            "name": "OE " + real_multimach_target_sys,
+            "environmentSetupScript": env_script,
+            "toolchainFile": standalone_sysroot_native + datadir + "/cmake/OEToolchainConfig.cmake",
+            "preferredGenerator": {
+                "name": oecmake_generator
+            }
+        }
+        def merge_kit(cmake_kits, cmake_kit_new):
+            i = 0
+            while i < len(cmake_kits):
+                if 'environmentSetupScript' in cmake_kits[i] and \
+                        cmake_kits[i]['environmentSetupScript'] == cmake_kit_new['environmentSetupScript']:
+                    cmake_kits[i] = cmake_kit_new
+                    return
+                i += 1
+            cmake_kits.append(cmake_kit_new)
+        merge_kit(cmake_kits, cmake_kit_new)
+        if cmake_kits != cmake_kits_old:
+            logger.info("Updating: %s" % cmake_kits_path)
+            with open(cmake_kits_path, 'w', encoding='utf-8') as cmake_kits_file:
+                json.dump(cmake_kits, cmake_kits_file, indent=4)
+        else:
+            logger.info("Already up to date: %s" % cmake_kits_path)
+        cmake_native = os.path.join(
+            shared_env.build_sysroots.standalone_sysroot_native, 'usr', 'bin', 'cmake')
+        if os.path.isfile(cmake_native):
+            logger.info('cmake-kits call cmake by default. If the cmake provided by this SDK should be used, please add the following line to ".vscode/settings.json" file: "cmake.cmakePath": "%s"' % cmake_native)
+        else:
+            logger.error("Cannot find cmake native at: %s" % cmake_native)
+    def dot_code_dir(self, modified_recipe):
+        return os.path.join(modified_recipe.srctree, '.vscode')
+    def __vscode_settings_meson(self, settings_dict, modified_recipe):
+        if modified_recipe.build_tool is not BuildTool.MESON:
+            return
+        settings_dict["mesonbuild.mesonPath"] = modified_recipe.meson_wrapper
+        confopts = modified_recipe.mesonopts.split()
+        confopts += modified_recipe.meson_cross_file.split()
+        confopts += modified_recipe.extra_oemeson.split()
+        settings_dict["mesonbuild.configureOptions"] = confopts
+        settings_dict["mesonbuild.buildFolder"] = modified_recipe.b
+    def __vscode_settings_cmake(self, settings_dict, modified_recipe):
+        """Add cmake specific settings to settings.json.
+        Note: most settings are passed to the cmake preset.
+        """
+        if modified_recipe.build_tool is not BuildTool.CMAKE:
+            return
+        settings_dict["cmake.configureOnOpen"] = True
+        settings_dict["cmake.sourceDirectory"] = modified_recipe.real_srctree
+    def vscode_settings(self, modified_recipe, image_recipe):
+        files_excludes = {
+            "**/.git/**": True,
+            "**/oe-logs/**": True,
+            "**/oe-workdir/**": True,
+            "**/source-date-epoch/**": True
+        }
+        python_exclude = [
+            "**/.git/**",
+            "**/oe-logs/**",
+            "**/oe-workdir/**",
+            "**/source-date-epoch/**"
+        ]
+        files_readonly = {
+            modified_recipe.recipe_sysroot + '/**': True,
+            modified_recipe.recipe_sysroot_native + '/**': True,
+        }
+        if image_recipe.rootfs_dbg is not None:
+            files_readonly[image_recipe.rootfs_dbg + '/**'] = True
+        settings_dict = {
+            "files.watcherExclude": files_excludes,
+            "files.exclude": files_excludes,
+            "files.readonlyInclude": files_readonly,
+            "python.analysis.exclude": python_exclude
+        }
+        self.__vscode_settings_cmake(settings_dict, modified_recipe)
+        self.__vscode_settings_meson(settings_dict, modified_recipe)
+        settings_file = 'settings.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), settings_file, settings_dict)
+    def __vscode_extensions_cmake(self, modified_recipe, recommendations):
+        if modified_recipe.build_tool is not BuildTool.CMAKE:
+            return
+        recommendations += [
+            "ms-vscode.cmake-tools",
+            "ms-vscode.cpptools",
+            "ms-vscode.cpptools-extension-pack",
+            "ms-vscode.cpptools-themes"
+        ]
+    def __vscode_extensions_meson(self, modified_recipe, recommendations):
+        if modified_recipe.build_tool is not BuildTool.MESON:
+            return
+        recommendations += [
+            'mesonbuild.mesonbuild',
+            "ms-vscode.cpptools",
+            "ms-vscode.cpptools-extension-pack",
+            "ms-vscode.cpptools-themes"
+        ]
+    def vscode_extensions(self, modified_recipe):
+        recommendations = []
+        self.__vscode_extensions_cmake(modified_recipe, recommendations)
+        self.__vscode_extensions_meson(modified_recipe, recommendations)
+        extensions_file = 'extensions.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), extensions_file, {"recommendations": recommendations})
+    def vscode_c_cpp_properties(self, modified_recipe):
+        properties_dict = {
+            "name": modified_recipe.recipe_id_pretty,
+        }
+        if modified_recipe.build_tool is BuildTool.CMAKE:
+            properties_dict["configurationProvider"] = "ms-vscode.cmake-tools"
+        elif modified_recipe.build_tool is BuildTool.MESON:
+            properties_dict["configurationProvider"] = "mesonbuild.mesonbuild"
+            properties_dict["compilerPath"] = os.path.join(modified_recipe.staging_bindir_toolchain, modified_recipe.cxx.split()[0])
+        else:  # no C/C++ build
+            return
+        properties_dicts = {
+            "configurations": [
+                properties_dict
+            ],
+            "version": 4
+        }
+        prop_file = 'c_cpp_properties.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), prop_file, properties_dicts)
+    def vscode_launch_bin_dbg(self, gdb_cross_config):
+        modified_recipe = gdb_cross_config.modified_recipe
+        launch_config = {
+            "name": gdb_cross_config.id_pretty,
+            "type": "cppdbg",
+            "request": "launch",
+            "program": os.path.join(modified_recipe.d, gdb_cross_config.binary.lstrip('/')),
+            "stopAtEntry": True,
+            "cwd": "${workspaceFolder}",
+            "environment": [],
+            "externalConsole": False,
+            "MIMode": "gdb",
+            "preLaunchTask": gdb_cross_config.id_pretty,
+            "miDebuggerPath": modified_recipe.gdb_cross.gdb,
+            "miDebuggerServerAddress": "%s:%d" % (modified_recipe.gdb_cross.host, gdb_cross_config.gdbserver_port)
+        }
+        # Search for header files in recipe-sysroot.
+        src_file_map = {
+            "/usr/include": os.path.join(modified_recipe.recipe_sysroot, "usr", "include")
+        }
+        # First of all search for not stripped binaries in the image folder.
+        # These binaries are copied (and optionally stripped) by deploy-target
+        setup_commands = [
+            {
+                "description": "sysroot",
+                "text": "set sysroot " + modified_recipe.d
+            }
+        ]
+        if gdb_cross_config.image_recipe.rootfs_dbg:
+            launch_config['additionalSOLibSearchPath'] = modified_recipe.solib_search_path_str(
+                gdb_cross_config.image_recipe)
+            # First: Search for sources of this recipe in the workspace folder
+            if modified_recipe.pn in modified_recipe.target_dbgsrc_dir:
+                src_file_map[modified_recipe.target_dbgsrc_dir] = "${workspaceFolder}"
+            else:
+                logger.error(
+                    "TARGET_DBGSRC_DIR must contain the recipe name PN.")
+            # Second: Search for sources of other recipes in the rootfs-dbg
+            if modified_recipe.target_dbgsrc_dir.startswith("/usr/src/debug"):
+                src_file_map["/usr/src/debug"] = os.path.join(
+                    gdb_cross_config.image_recipe.rootfs_dbg, "usr", "src", "debug")
+            else:
+                logger.error(
+                    "TARGET_DBGSRC_DIR must start with /usr/src/debug.")
+        else:
+            logger.warning(
+                "Cannot setup debug symbols configuration for GDB. IMAGE_GEN_DEBUGFS is not enabled.")
+        launch_config['sourceFileMap'] = src_file_map
+        launch_config['setupCommands'] = setup_commands
+        return launch_config
+    def vscode_launch(self, modified_recipe):
+        """GDB Launch configuration for binaries (elf files)"""
+        configurations = []
+        for gdb_cross_config in self.gdb_cross_configs:
+            if gdb_cross_config.modified_recipe is modified_recipe:
+                configurations.append(self.vscode_launch_bin_dbg(gdb_cross_config))
+        launch_dict = {
+            "version": "0.2.0",
+            "configurations": configurations
+        }
+        launch_file = 'launch.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), launch_file, launch_dict)
+    def vscode_tasks_cpp(self, args, modified_recipe):
+        run_install_deploy = modified_recipe.gen_install_deploy_script(args)
+        install_task_name = "install && deploy-target %s" % modified_recipe.recipe_id_pretty
+        tasks_dict = {
+            "version": "2.0.0",
+            "tasks": [
+                {
+                    "label": install_task_name,
+                    "type": "shell",
+                    "command": run_install_deploy,
+                    "problemMatcher": []
+                }
+            ]
+        }
+        for gdb_cross_config in self.gdb_cross_configs:
+            if gdb_cross_config.modified_recipe is not modified_recipe:
+                continue
+            tasks_dict['tasks'].append(
+                {
+                    "label": gdb_cross_config.id_pretty,
+                    "type": "shell",
+                    "isBackground": True,
+                    "dependsOn": [
+                        install_task_name
+                    ],
+                    "command": gdb_cross_config.gdbserver_script,
+                    "problemMatcher": [
+                        {
+                            "pattern": [
+                                {
+                                    "regexp": ".",
+                                    "file": 1,
+                                    "location": 2,
+                                    "message": 3
+                                }
+                            ],
+                            "background": {
+                                "activeOnStart": True,
+                                "beginsPattern": ".",
+                                "endsPattern": ".",
+                            }
+                        }
+                    ]
+                })
+        tasks_file = 'tasks.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), tasks_file, tasks_dict)
+    def vscode_tasks_fallback(self, args, modified_recipe):
+        oe_init_dir = modified_recipe.oe_init_dir
+        oe_init = ". %s %s > /dev/null && " % (modified_recipe.oe_init_build_env, modified_recipe.topdir)
+        dt_build = "devtool build "
+        dt_build_label = dt_build + modified_recipe.recipe_id_pretty
+        dt_build_cmd = dt_build + modified_recipe.bpn
+        clean_opt = " --clean"
+        dt_build_clean_label = dt_build + modified_recipe.recipe_id_pretty + clean_opt
+        dt_build_clean_cmd = dt_build + modified_recipe.bpn + clean_opt
+        dt_deploy = "devtool deploy-target "
+        dt_deploy_label = dt_deploy + modified_recipe.recipe_id_pretty
+        dt_deploy_cmd = dt_deploy + modified_recipe.bpn
+        dt_build_deploy_label = "devtool build & deploy-target %s" % modified_recipe.recipe_id_pretty
+        deploy_opts = ' '.join(get_devtool_deploy_opts(args))
+        tasks_dict = {
+            "version": "2.0.0",
+            "tasks": [
+                {
+                    "label": dt_build_label,
+                    "type": "shell",
+                    "command": "bash",
+                    "linux": {
+                        "options": {
+                            "cwd": oe_init_dir
+                        }
+                    },
+                    "args": [
+                        "--login",
+                        "-c",
+                        "%s%s" % (oe_init, dt_build_cmd)
+                    ],
+                    "problemMatcher": []
+                },
+                {
+                    "label": dt_deploy_label,
+                    "type": "shell",
+                    "command": "bash",
+                    "linux": {
+                        "options": {
+                            "cwd": oe_init_dir
+                        }
+                    },
+                    "args": [
+                        "--login",
+                        "-c",
+                        "%s%s %s" % (
+                            oe_init, dt_deploy_cmd, deploy_opts)
+                    ],
+                    "problemMatcher": []
+                },
+                {
+                    "label": dt_build_deploy_label,
+                    "dependsOrder": "sequence",
+                    "dependsOn": [
+                        dt_build_label,
+                        dt_deploy_label
+                    ],
+                    "problemMatcher": [],
+                    "group": {
+                        "kind": "build",
+                        "isDefault": True
+                    }
+                },
+                {
+                    "label": dt_build_clean_label,
+                    "type": "shell",
+                    "command": "bash",
+                    "linux": {
+                        "options": {
+                            "cwd": oe_init_dir
+                        }
+                    },
+                    "args": [
+                        "--login",
+                        "-c",
+                        "%s%s" % (oe_init, dt_build_clean_cmd)
+                    ],
+                    "problemMatcher": []
+                }
+            ]
+        }
+        if modified_recipe.gdb_cross:
+            for gdb_cross_config in self.gdb_cross_configs:
+                if gdb_cross_config.modified_recipe is not modified_recipe:
+                    continue
+                tasks_dict['tasks'].append(
+                    {
+                        "label": gdb_cross_config.id_pretty,
+                        "type": "shell",
+                        "isBackground": True,
+                        "dependsOn": [
+                            dt_build_deploy_label
+                        ],
+                        "command": gdb_cross_config.gdbserver_script,
+                        "problemMatcher": [
+                            {
+                                "pattern": [
+                                    {
+                                        "regexp": ".",
+                                        "file": 1,
+                                        "location": 2,
+                                        "message": 3
+                                    }
+                                ],
+                                "background": {
+                                    "activeOnStart": True,
+                                    "beginsPattern": ".",
+                                    "endsPattern": ".",
+                                }
+                            }
+                        ]
+                    })
+        tasks_file = 'tasks.json'
+        IdeBase.update_json_file(
+            self.dot_code_dir(modified_recipe), tasks_file, tasks_dict)
+    def vscode_tasks(self, args, modified_recipe):
+        if modified_recipe.build_tool.is_c_ccp:
+            self.vscode_tasks_cpp(args, modified_recipe)
+        else:
+            self.vscode_tasks_fallback(args, modified_recipe)
+    def setup_modified_recipe(self, args, image_recipe, modified_recipe):
+        self.vscode_settings(modified_recipe, image_recipe)
+        self.vscode_extensions(modified_recipe)
+        self.vscode_c_cpp_properties(modified_recipe)
+        if args.target:
+            self.initialize_gdb_cross_configs(
+                image_recipe, modified_recipe, gdb_cross_config_class=GdbCrossConfigVSCode)
+            self.vscode_launch(modified_recipe)
+            self.vscode_tasks(args, modified_recipe)
+def register_ide_plugin(ide_plugins):
+    ide_plugins['code'] = IdeVSCode
diff --git a/scripts/lib/devtool/ide_plugins/ide_none.py b/scripts/lib/devtool/ide_plugins/ide_none.py
new file mode 100644
index 0000000000..f106c5a026
--- /dev/null
+++ b/scripts/lib/devtool/ide_plugins/ide_none.py
@@ -0,0 +1,53 @@
+#
+# Copyright (C) 2023-2024 Siemens AG
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""Devtool ide-sdk generic IDE plugin"""
+import os
+import logging
+from devtool.ide_plugins import IdeBase, GdbCrossConfig
+logger = logging.getLogger('devtool')
+class IdeNone(IdeBase):
+    """Generate some generic helpers for other IDEs
+    Modified recipe mode:
+    Generate some helper scripts for remote debugging with GDB
+    Shared sysroot mode:
+    A wrapper for bitbake meta-ide-support and bitbake build-sysroots
+    """
+    def __init__(self):
+        super().__init__()
+    def setup_shared_sysroots(self, shared_env):
+        real_multimach_target_sys = shared_env.ide_support.real_multimach_target_sys
+        deploy_dir_image = shared_env.ide_support.deploy_dir_image
+        env_script = os.path.join(
+            deploy_dir_image, 'environment-setup-' + real_multimach_target_sys)
+        logger.info(
+            "To use this SDK please source this: %s" % env_script)
+    def setup_modified_recipe(self, args, image_recipe, modified_recipe):
+        """generate some helper scripts and config files
+        - Execute the do_install task
+        - Execute devtool deploy-target
+        - Generate a gdbinit file per executable
+        - Generate the oe-scripts sym-link
+        """
+        script_path = modified_recipe.gen_install_deploy_script(args)
+        logger.info("Created: %s" % script_path)
+        self.initialize_gdb_cross_configs(image_recipe, modified_recipe)
+        IdeBase.gen_oe_scrtips_sym_link(modified_recipe)
+def register_ide_plugin(ide_plugins):
+    ide_plugins['none'] = IdeNone
diff --git a/scripts/lib/devtool/ide_sdk.py b/scripts/lib/devtool/ide_sdk.py
new file mode 100755
index 0000000000..931408fa74
--- /dev/null
+++ b/scripts/lib/devtool/ide_sdk.py
@@ -0,0 +1,1009 @@
+# Development tool - ide-sdk command plugin
+#
+# Copyright (C) 2023-2024 Siemens AG
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+"""Devtool ide-sdk plugin"""
+import json
+import logging
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
+from argparse import RawTextHelpFormatter
+from enum import Enum
+import scriptutils
+import bb
+from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, DevtoolError, parse_recipe
+from devtool.standard import get_real_srctree
+from devtool.ide_plugins import BuildTool
+logger = logging.getLogger('devtool')
+# dict of classes derived from IdeBase
+ide_plugins = {}
+class DevtoolIdeMode(Enum):
+    """Different modes are supported by the ide-sdk plugin.
+    The enum might be extended by more advanced modes in the future. Some ideas:
+    - auto: modified if all recipes are modified, shared if none of the recipes is modified.
+    - mixed: modified mode for modified recipes, shared mode for all other recipes.
+    """
+    modified = 'modified'
+    shared = 'shared'
+class TargetDevice:
+    """SSH remote login parameters"""
+    def __init__(self, args):
+        self.extraoptions = ''
+        if args.no_host_check:
+            self.extraoptions += '-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
+        self.ssh_sshexec = 'ssh'
+        if args.ssh_exec:
+            self.ssh_sshexec = args.ssh_exec
+        self.ssh_port = ''
+        if args.port:
+            self.ssh_port = "-p %s" % args.port
+        if args.key:
+            self.extraoptions += ' -i %s' % args.key
+        self.target = args.target
+        target_sp = args.target.split('@')
+        if len(target_sp) == 1:
+            self.login = ""
+            self.host = target_sp[0]
+        elif len(target_sp) == 2:
+            self.login = target_sp[0]
+            self.host = target_sp[1]
+        else:
+            logger.error("Invalid target argument: %s" % args.target)
+class RecipeNative:
+    """Base class for calling bitbake to provide a -native recipe"""
+    def __init__(self, name, target_arch=None):
+        self.name = name
+        self.target_arch = target_arch
+        self.bootstrap_tasks = [self.name + ':do_addto_recipe_sysroot']
+        self.staging_bindir_native = None
+        self.target_sys = None
+        self.__native_bin = None
+    def _initialize(self, config, workspace, tinfoil):
+        """Get the parsed recipe"""
+        recipe_d = parse_recipe(
+            config, tinfoil, self.name, appends=True, filter_workspace=False)
+        if not recipe_d:
+            raise DevtoolError("Parsing %s recipe failed" % self.name)
+        self.staging_bindir_native = os.path.realpath(
+            recipe_d.getVar('STAGING_BINDIR_NATIVE'))
+        self.target_sys = recipe_d.getVar('TARGET_SYS')
+        return recipe_d
+    def initialize(self, config, workspace, tinfoil):
+        """Basic initialization that can be overridden by a derived class"""
+        self._initialize(config, workspace, tinfoil)
+    @property
+    def native_bin(self):
+        if not self.__native_bin:
+            raise DevtoolError("native binary name is not defined.")
+        return self.__native_bin
+class RecipeGdbCross(RecipeNative):
+    """Handle handle gdb-cross on the host and the gdbserver on the target device"""
+    def __init__(self, args, target_arch, target_device):
+        super().__init__('gdb-cross-' + target_arch, target_arch)
+        self.target_device = target_device
+        self.gdb = None
+        self.gdbserver_port_next = int(args.gdbserver_port_start)
+        self.config_db = {}
+    def __find_gdbserver(self, config, tinfoil):
+        """Absolute path of the gdbserver"""
+        recipe_d_gdb = parse_recipe(
+            config, tinfoil, 'gdb', appends=True, filter_workspace=False)
+        if not recipe_d_gdb:
+            raise DevtoolError("Parsing gdb recipe failed")
+        return os.path.join(recipe_d_gdb.getVar('bindir'), 'gdbserver')
+    def initialize(self, config, workspace, tinfoil):
+        super()._initialize(config, workspace, tinfoil)
+        gdb_bin = self.target_sys + '-gdb'
+        gdb_path = os.path.join(
+            self.staging_bindir_native, self.target_sys, gdb_bin)
+        self.gdb = gdb_path
+        self.gdbserver_path = self.__find_gdbserver(config, tinfoil)
+    @property
+    def host(self):
+        return self.target_device.host
+class RecipeImage:
+    """Handle some image recipe related properties
+    Most workflows require firmware that runs on the target device.
+    This firmware must be consistent with the setup of the host system.
+    In particular, the debug symbols must be compatible. For this, the
+    rootfs must be created as part of the SDK.
+    """
+    def __init__(self, name):
+        self.combine_dbg_image = False
+        self.gdbserver_missing = False
+        self.name = name
+        self.rootfs = None
+        self.__rootfs_dbg = None
+        self.bootstrap_tasks = [self.name + ':do_build']
+    def initialize(self, config, tinfoil):
+        image_d = parse_recipe(
+            config, tinfoil, self.name, appends=True, filter_workspace=False)
+        if not image_d:
+            raise DevtoolError(
+                "Parsing image recipe %s failed" % self.name)
+        self.combine_dbg_image = bb.data.inherits_class(
+            'image-combined-dbg', image_d)
+        workdir = image_d.getVar('WORKDIR')
+        self.rootfs = os.path.join(workdir, 'rootfs')
+        if image_d.getVar('IMAGE_GEN_DEBUGFS') == "1":
+            self.__rootfs_dbg = os.path.join(workdir, 'rootfs-dbg')
+        self.gdbserver_missing = 'gdbserver' not in image_d.getVar(
+            'IMAGE_INSTALL') and 'tools-debug' not in image_d.getVar('IMAGE_FEATURES')
+    @property
+    def debug_support(self):
+        return bool(self.rootfs_dbg)
+    @property
+    def rootfs_dbg(self):
+        if self.__rootfs_dbg and os.path.isdir(self.__rootfs_dbg):
+            return self.__rootfs_dbg
+        return None
+class RecipeMetaIdeSupport:
+    """For the shared sysroots mode meta-ide-support is needed
+    For use cases where just a cross tool-chain is required but
+    no recipe is used, devtool ide-sdk abstracts calling bitbake meta-ide-support
+    and bitbake build-sysroots. This also allows to expose the cross-toolchains
+    to IDEs. For example VSCode support different tool-chains with e.g. cmake-kits.
+    """
+    def __init__(self):
+        self.bootstrap_tasks = ['meta-ide-support:do_build']
+        self.topdir = None
+        self.datadir = None
+        self.deploy_dir_image = None
+        self.build_sys = None
+        # From toolchain-scripts
+        self.real_multimach_target_sys = None
+    def initialize(self, config, tinfoil):
+        meta_ide_support_d = parse_recipe(
+            config, tinfoil, 'meta-ide-support', appends=True, filter_workspace=False)
+        if not meta_ide_support_d:
+            raise DevtoolError("Parsing meta-ide-support recipe failed")
+        self.topdir = meta_ide_support_d.getVar('TOPDIR')
+        self.datadir = meta_ide_support_d.getVar('datadir')
+        self.deploy_dir_image = meta_ide_support_d.getVar(
+            'DEPLOY_DIR_IMAGE')
+        self.build_sys = meta_ide_support_d.getVar('BUILD_SYS')
+        self.real_multimach_target_sys = meta_ide_support_d.getVar(
+            'REAL_MULTIMACH_TARGET_SYS')
+class RecipeBuildSysroots:
+    """For the shared sysroots mode build-sysroots is needed"""
+    def __init__(self):
+        self.standalone_sysroot = None
+        self.standalone_sysroot_native = None
+        self.bootstrap_tasks = [
+            'build-sysroots:do_build_target_sysroot',
+            'build-sysroots:do_build_native_sysroot'
+        ]
+    def initialize(self, config, tinfoil):
+        build_sysroots_d = parse_recipe(
+            config, tinfoil, 'build-sysroots', appends=True, filter_workspace=False)
+        if not build_sysroots_d:
+            raise DevtoolError("Parsing build-sysroots recipe failed")
+        self.standalone_sysroot = build_sysroots_d.getVar(
+            'STANDALONE_SYSROOT')
+        self.standalone_sysroot_native = build_sysroots_d.getVar(
+            'STANDALONE_SYSROOT_NATIVE')
+class SharedSysrootsEnv:
+    """Handle the shared sysroots based workflow
+    Support the workflow with just a tool-chain without a recipe.
+    It's basically like:
+      bitbake some-dependencies
+      bitbake meta-ide-support
+      bitbake build-sysroots
+      Use the environment-* file found in the deploy folder
+    """
+    def __init__(self):
+        self.ide_support = None
+        self.build_sysroots = None
+    def initialize(self, ide_support, build_sysroots):
+        self.ide_support = ide_support
+        self.build_sysroots = build_sysroots
+    def setup_ide(self, ide):
+        ide.setup(self)
+class RecipeNotModified:
+    """Handling of recipes added to the Direct DSK shared sysroots."""
+    def __init__(self, name):
+        self.name = name
+        self.bootstrap_tasks = [name + ':do_populate_sysroot']
+class RecipeModified:
+    """Handling of recipes in the workspace created by devtool modify"""
+    OE_INIT_BUILD_ENV = 'oe-init-build-env'
+    VALID_BASH_ENV_NAME_CHARS = re.compile(r"^[a-zA-Z0-9_]*$")
+    def __init__(self, name):
+        self.name = name
+        self.bootstrap_tasks = [name + ':do_install']
+        self.gdb_cross = None
+        # workspace
+        self.real_srctree = None
+        self.srctree = None
+        self.ide_sdk_dir = None
+        self.ide_sdk_scripts_dir = None
+        self.bbappend = None
+        # recipe variables from d.getVar
+        self.b = None
+        self.base_libdir = None
+        self.bblayers = None
+        self.bpn = None
+        self.d = None
+        self.debug_build = None
+        self.fakerootcmd = None
+        self.fakerootenv = None
+        self.libdir = None
+        self.max_process = None
+        self.package_arch = None
+        self.package_debug_split_style = None
+        self.path = None
+        self.pn = None
+        self.recipe_sysroot = None
+        self.recipe_sysroot_native = None
+        self.staging_incdir = None
+        self.strip_cmd = None
+        self.target_arch = None
+        self.target_dbgsrc_dir = None
+        self.topdir = None
+        self.workdir = None
+        self.recipe_id = None
+        # replicate bitbake build environment
+        self.exported_vars = None
+        self.cmd_compile = None
+        self.__oe_init_dir = None
+        # main build tool used by this recipe
+        self.build_tool = BuildTool.UNDEFINED
+        # build_tool = cmake
+        self.oecmake_generator = None
+        self.cmake_cache_vars = None
+        # build_tool = meson
+        self.meson_buildtype = None
+        self.meson_wrapper = None
+        self.mesonopts = None
+        self.extra_oemeson = None
+        self.meson_cross_file = None
+    def initialize(self, config, workspace, tinfoil):
+        recipe_d = parse_recipe(
+            config, tinfoil, self.name, appends=True, filter_workspace=False)
+        if not recipe_d:
+            raise DevtoolError("Parsing %s recipe failed" % self.name)
+        # Verify this recipe is built as externalsrc setup by devtool modify
+        workspacepn = check_workspace_recipe(
+            workspace, self.name, bbclassextend=True)
+        self.srctree = workspace[workspacepn]['srctree']
+        # Need to grab this here in case the source is within a subdirectory
+        self.real_srctree = get_real_srctree(
+            self.srctree, recipe_d.getVar('S'), recipe_d.getVar('UNPACKDIR'))
+        self.bbappend = workspace[workspacepn]['bbappend']
+        self.ide_sdk_dir = os.path.join(
+            config.workspace_path, 'ide-sdk', self.name)
+        if os.path.exists(self.ide_sdk_dir):
+            shutil.rmtree(self.ide_sdk_dir)
+        self.ide_sdk_scripts_dir = os.path.join(self.ide_sdk_dir, 'scripts')
+        self.b = recipe_d.getVar('B')
+        self.base_libdir = recipe_d.getVar('base_libdir')
+        self.bblayers = recipe_d.getVar('BBLAYERS').split()
+        self.bpn = recipe_d.getVar('BPN')
+        self.cxx = recipe_d.getVar('CXX')
+        self.d = recipe_d.getVar('D')
+        self.debug_build = recipe_d.getVar('DEBUG_BUILD')
+        self.fakerootcmd = recipe_d.getVar('FAKEROOTCMD')
+        self.fakerootenv = recipe_d.getVar('FAKEROOTENV')
+        self.libdir = recipe_d.getVar('libdir')
+        self.max_process = int(recipe_d.getVar(
+            "BB_NUMBER_THREADS") or os.cpu_count() or 1)
+        self.package_arch = recipe_d.getVar('PACKAGE_ARCH')
+        self.package_debug_split_style = recipe_d.getVar(
+            'PACKAGE_DEBUG_SPLIT_STYLE')
+        self.path = recipe_d.getVar('PATH')
+        self.pn = recipe_d.getVar('PN')
+        self.recipe_sysroot = os.path.realpath(
+            recipe_d.getVar('RECIPE_SYSROOT'))
+        self.recipe_sysroot_native = os.path.realpath(
+            recipe_d.getVar('RECIPE_SYSROOT_NATIVE'))
+        self.staging_bindir_toolchain = os.path.realpath(
+            recipe_d.getVar('STAGING_BINDIR_TOOLCHAIN'))
+        self.staging_incdir = os.path.realpath(
+            recipe_d.getVar('STAGING_INCDIR'))
+        self.strip_cmd = recipe_d.getVar('STRIP')
+        self.target_arch = recipe_d.getVar('TARGET_ARCH')
+        self.target_dbgsrc_dir = recipe_d.getVar('TARGET_DBGSRC_DIR')
+        self.topdir = recipe_d.getVar('TOPDIR')
+        self.workdir = os.path.realpath(recipe_d.getVar('WORKDIR'))
+        self.__init_exported_variables(recipe_d)
+        if bb.data.inherits_class('cmake', recipe_d):
+            self.oecmake_generator = recipe_d.getVar('OECMAKE_GENERATOR')
+            self.__init_cmake_preset_cache(recipe_d)
+            self.build_tool = BuildTool.CMAKE
+        elif bb.data.inherits_class('meson', recipe_d):
+            self.meson_buildtype = recipe_d.getVar('MESON_BUILDTYPE')
+            self.mesonopts = recipe_d.getVar('MESONOPTS')
+            self.extra_oemeson = recipe_d.getVar('EXTRA_OEMESON')
+            self.meson_cross_file = recipe_d.getVar('MESON_CROSS_FILE')
+            self.build_tool = BuildTool.MESON
+        # Recipe ID is the identifier for IDE config sections
+        self.recipe_id = self.bpn + "-" + self.package_arch
+        self.recipe_id_pretty = self.bpn + ": " + self.package_arch
+    @staticmethod
+    def is_valid_shell_variable(var):
+        """Skip strange shell variables like systemd
+        prevent from strange bugs because of strange variables which
+        are not used in this context but break various tools.
+        """
+        if RecipeModified.VALID_BASH_ENV_NAME_CHARS.match(var):
+            bb.debug(1, "ignoring variable: %s" % var)
+            return True
+        return False
+    def solib_search_path(self, image):
+        """Search for debug symbols in the rootfs and rootfs-dbg
+        The debug symbols of shared libraries which are provided by other packages
+        are grabbed from the -dbg packages in the rootfs-dbg.
+        But most cross debugging tools like gdb, perf, and systemtap need to find
+        executable/library first and through it debuglink note find corresponding
+        symbols file. Therefore the library paths from the rootfs are added as well.
+        Note: For the devtool modified recipe compiled from the IDE, the debug
+        symbols are taken from the unstripped binaries in the image folder.
+        Also, devtool deploy-target takes the files from the image folder.
+        debug symbols in the image folder refer to the corresponding source files
+        with absolute paths of the build machine. Debug symbols found in the
+        rootfs-dbg are relocated and contain paths which refer to the source files
+        installed on the target device e.g. /usr/src/...
+        """
+        base_libdir = self.base_libdir.lstrip('/')
+        libdir = self.libdir.lstrip('/')
+        so_paths = [
+            # debug symbols for package_debug_split_style: debug-with-srcpkg or .debug
+            os.path.join(image.rootfs_dbg, base_libdir, ".debug"),
+            os.path.join(image.rootfs_dbg, libdir, ".debug"),
+            # debug symbols for package_debug_split_style: debug-file-directory
+            os.path.join(image.rootfs_dbg, "usr", "lib", "debug"),
+            # The binaries are required as well, the debug packages are not enough
+            # With image-combined-dbg.bbclass the binaries are copied into rootfs-dbg
+            os.path.join(image.rootfs_dbg, base_libdir),
+            os.path.join(image.rootfs_dbg, libdir),
+            # Without image-combined-dbg.bbclass the binaries are only in rootfs.
+            # Note: Stepping into source files located in rootfs-dbg does not
+            #       work without image-combined-dbg.bbclass yet.
+            os.path.join(image.rootfs, base_libdir),
+            os.path.join(image.rootfs, libdir)
+        ]
+        return so_paths
+    def solib_search_path_str(self, image):
+        """Return a : separated list of paths usable by GDB's set solib-search-path"""
+        return ':'.join(self.solib_search_path(image))
+    def __init_exported_variables(self, d):
+        """Find all variables with export flag set.
+        This allows to generate IDE configurations which compile with the same
+        environment as bitbake does. That's at least a reasonable default behavior.
+        """
+        exported_vars = {}
+        vars = (key for key in d.keys() if not key.startswith(
+            "__") and not d.getVarFlag(key, "func", False))
+        for var in sorted(vars):
+            func = d.getVarFlag(var, "func", False)
+            if d.getVarFlag(var, 'python', False) and func:
+                continue
+            export = d.getVarFlag(var, "export", False)
+            unexport = d.getVarFlag(var, "unexport", False)
+            if not export and not unexport and not func:
+                continue
+            if unexport:
+                continue
+            val = d.getVar(var)
+            if val is None:
+                continue
+            if set(var) & set("-.{}+"):
+                logger.warn(
+                    "Warning: Found invalid character in variable name %s", str(var))
+                continue
+            varExpanded = d.expand(var)
+            val = str(val)
+            if not RecipeModified.is_valid_shell_variable(varExpanded):
+                continue
+            if func:
+                code_line = "line: {0}, file: {1}\n".format(
+                    d.getVarFlag(var, "lineno", False),
+                    d.getVarFlag(var, "filename", False))
+                val = val.rstrip('\n')
+                logger.warn("Warning: exported shell function %s() is not exported (%s)" %
+                            (varExpanded, code_line))
+                continue
+            if export:
+                exported_vars[varExpanded] = val.strip()
+                continue
+        self.exported_vars = exported_vars
+    def __init_cmake_preset_cache(self, d):
+        """Get the arguments passed to cmake
+        Replicate the cmake configure arguments with all details to
+        share on build folder between bitbake and SDK.
+        """
+        site_file = os.path.join(self.workdir, 'site-file.cmake')
+        if os.path.exists(site_file):
+            print("Warning: site-file.cmake is not supported")
+        cache_vars = {}
+        oecmake_args = d.getVar('OECMAKE_ARGS').split()
+        extra_oecmake = d.getVar('EXTRA_OECMAKE').split()
+        for param in sorted(oecmake_args + extra_oecmake):
+            d_pref = "-D"
+            if param.startswith(d_pref):
+                param = param[len(d_pref):]
+            else:
+                print("Error: expected a -D")
+            param_s = param.split('=', 1)
+            param_nt = param_s[0].split(':', 1)
+            def handle_undefined_variable(var):
+                if var.startswith('${') and var.endswith('}'):
+                    return ''
+                else:
+                    return var
+            # Example: FOO=ON
+            if len(param_nt) == 1:
+                cache_vars[param_s[0]] = handle_undefined_variable(param_s[1])
+            # Example: FOO:PATH=/tmp
+            elif len(param_nt) == 2:
+                cache_vars[param_nt[0]] = {
+                    "type": param_nt[1],
+                    "value": handle_undefined_variable(param_s[1]),
+                }
+            else:
+                print("Error: cannot parse %s" % param)
+        self.cmake_cache_vars = cache_vars
+    def cmake_preset(self):
+        """Create a preset for cmake that mimics how bitbake calls cmake"""
+        toolchain_file = os.path.join(self.workdir, 'toolchain.cmake')
+        cmake_executable = os.path.join(
+            self.recipe_sysroot_native, 'usr', 'bin', 'cmake')
+        self.cmd_compile = cmake_executable + " --build --preset " + self.recipe_id
+        preset_dict_configure = {
+            "name": self.recipe_id,
+            "displayName": self.recipe_id_pretty,
+            "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
+            "binaryDir": self.b,
+            "generator": self.oecmake_generator,
+            "toolchainFile": toolchain_file,
+            "cacheVariables": self.cmake_cache_vars,
+            "environment": self.exported_vars,
+            "cmakeExecutable": cmake_executable
+        }
+        preset_dict_build = {
+            "name": self.recipe_id,
+            "displayName": self.recipe_id_pretty,
+            "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
+            "configurePreset": self.recipe_id,
+            "inheritConfigureEnvironment": True
+        }
+        preset_dict_test = {
+            "name": self.recipe_id,
+            "displayName": self.recipe_id_pretty,
+            "description": "Bitbake build environment for the recipe %s compiled for %s" % (self.bpn, self.package_arch),
+            "configurePreset": self.recipe_id,
+            "inheritConfigureEnvironment": True
+        }
+        preset_dict = {
+            "version": 3,  # cmake 3.21, backward compatible with kirkstone
+            "configurePresets": [preset_dict_configure],
+            "buildPresets": [preset_dict_build],
+            "testPresets": [preset_dict_test]
+        }
+        # Finally write the json file
+        json_file = 'CMakeUserPresets.json'
+        json_path = os.path.join(self.real_srctree, json_file)
+        logger.info("Updating CMake preset: %s (%s)" % (json_file, json_path))
+        if not os.path.exists(self.real_srctree):
+            os.makedirs(self.real_srctree)
+        try:
+            with open(json_path) as f:
+                orig_dict = json.load(f)
+        except json.decoder.JSONDecodeError:
+            logger.info(
+                "Decoding %s failed. Probably because of comments in the json file" % json_path)
+            orig_dict = {}
+        except FileNotFoundError:
+            orig_dict = {}
+        # Add or update the presets for the recipe and keep other presets
+        for k, v in preset_dict.items():
+            if isinstance(v, list):
+                update_preset = v[0]
+                preset_added = False
+                if k in orig_dict:
+                    for index, orig_preset in enumerate(orig_dict[k]):
+                        if 'name' in orig_preset:
+                            if orig_preset['name'] == update_preset['name']:
+                                logger.debug("Updating preset: %s" %
+                                             orig_preset['name'])
+                                orig_dict[k][index] = update_preset
+                                preset_added = True
+                                break
+                            else:
+                                logger.debug("keeping preset: %s" %
+                                             orig_preset['name'])
+                        else:
+                            logger.warn("preset without a name found")
+                if not preset_added:
+                    if not k in orig_dict:
+                        orig_dict[k] = []
+                    orig_dict[k].append(update_preset)
+                    logger.debug("Added preset: %s" %
+                                 update_preset['name'])
+            else:
+                orig_dict[k] = v
+        with open(json_path, 'w') as f:
+            json.dump(orig_dict, f, indent=4)
+    def gen_meson_wrapper(self):
+        """Generate a wrapper script to call meson with the cross environment"""
+        bb.utils.mkdirhier(self.ide_sdk_scripts_dir)
+        meson_wrapper = os.path.join(self.ide_sdk_scripts_dir, 'meson')
+        meson_real = os.path.join(
+            self.recipe_sysroot_native, 'usr', 'bin', 'meson.real')
+        with open(meson_wrapper, 'w') as mwrap:
+            mwrap.write("#!/bin/sh" + os.linesep)
+            for var, val in self.exported_vars.items():
+                mwrap.write('export %s="%s"' % (var, val) + os.linesep)
+            mwrap.write("unset CC CXX CPP LD AR NM STRIP" + os.linesep)
+            private_temp = os.path.join(self.b, "meson-private", "tmp")
+            mwrap.write('mkdir -p "%s"' % private_temp + os.linesep)
+            mwrap.write('export TMPDIR="%s"' % private_temp + os.linesep)
+            mwrap.write('exec "%s" "$@"' % meson_real + os.linesep)
+        st = os.stat(meson_wrapper)
+        os.chmod(meson_wrapper, st.st_mode | stat.S_IEXEC)
+        self.meson_wrapper = meson_wrapper
+        self.cmd_compile = meson_wrapper + " compile -C " + self.b
+    def which(self, executable):
+        bin_path = shutil.which(executable, path=self.path)
+        if not bin_path:
+            raise DevtoolError(
+                'Cannot find %s. Probably the recipe %s is not built yet.' % (executable, self.bpn))
+        return bin_path
+    @staticmethod
+    def is_elf_file(file_path):
+        with open(file_path, "rb") as f:
+            data = f.read(4)
+        if data == b'\x7fELF':
+            return True
+        return False
+    def find_installed_binaries(self):
+        """find all executable elf files in the image directory"""
+        binaries = []
+        d_len = len(self.d)
+        re_so = re.compile(r'.*\.so[.0-9]*$')
+        for root, _, files in os.walk(self.d, followlinks=False):
+            for file in files:
+                if os.path.islink(file):
+                    continue
+                if re_so.match(file):
+                    continue
+                abs_name = os.path.join(root, file)
+                if os.access(abs_name, os.X_OK) and RecipeModified.is_elf_file(abs_name):
+                    binaries.append(abs_name[d_len:])
+        return sorted(binaries)
+    def gen_deploy_target_script(self, args):
+        """Generate a script which does what devtool deploy-target does
+        This script is much quicker than devtool target-deploy. Because it
+        does not need to start a bitbake server. All information from tinfoil
+        is hard-coded in the generated script.
+        """
+        cmd_lines = ['#!%s' % str(sys.executable)]
+        cmd_lines.append('import sys')
+        cmd_lines.append('devtool_sys_path = %s' % str(sys.path))
+        cmd_lines.append('devtool_sys_path.reverse()')
+        cmd_lines.append('for p in devtool_sys_path:')
+        cmd_lines.append('    if p not in sys.path:')
+        cmd_lines.append('        sys.path.insert(0, p)')
+        cmd_lines.append('from devtool.deploy import deploy_no_d')
+        args_filter = ['debug', 'dry_run', 'key', 'no_check_space', 'no_host_check',
+                       'no_preserve', 'port', 'show_status', 'ssh_exec', 'strip', 'target']
+        filtered_args_dict = {key: value for key, value in vars(
+            args).items() if key in args_filter}
+        cmd_lines.append('filtered_args_dict = %s' % str(filtered_args_dict))
+        cmd_lines.append('class Dict2Class(object):')
+        cmd_lines.append('    def __init__(self, my_dict):')
+        cmd_lines.append('        for key in my_dict:')
+        cmd_lines.append('            setattr(self, key, my_dict[key])')
+        cmd_lines.append('filtered_args = Dict2Class(filtered_args_dict)')
+        cmd_lines.append(
+            'setattr(filtered_args, "recipename", "%s")' % self.bpn)
+        cmd_lines.append('deploy_no_d("%s", "%s", "%s", "%s", "%s", "%s", %d, "%s", "%s", filtered_args)' %
+                         (self.d, self.workdir, self.path, self.strip_cmd,
+                          self.libdir, self.base_libdir, self.max_process,
+                          self.fakerootcmd, self.fakerootenv))
+        return self.write_script(cmd_lines, 'deploy_target')
+    def gen_install_deploy_script(self, args):
+        """Generate a script which does install and deploy"""
+        cmd_lines = ['#!/bin/bash']
+        # . oe-init-build-env $BUILDDIR
+        # Note: Sourcing scripts with arguments requires bash
+        cmd_lines.append('cd "%s" || { echo "cd %s failed"; exit 1; }' % (
+            self.oe_init_dir, self.oe_init_dir))
+        cmd_lines.append('. "%s" "%s" || { echo ". %s %s failed"; exit 1; }' % (
+            self.oe_init_build_env, self.topdir, self.oe_init_build_env, self.topdir))
+        # bitbake -c install
+        cmd_lines.append(
+            'bitbake %s -c install --force || { echo "bitbake %s -c install --force failed"; exit 1; }' % (self.bpn, self.bpn))
+        # Self contained devtool deploy-target
+        cmd_lines.append(self.gen_deploy_target_script(args))
+        return self.write_script(cmd_lines, 'install_and_deploy')
+    def write_script(self, cmd_lines, script_name):
+        bb.utils.mkdirhier(self.ide_sdk_scripts_dir)
+        script_name_arch = script_name + '_' + self.recipe_id
+        script_file = os.path.join(self.ide_sdk_scripts_dir, script_name_arch)
+        with open(script_file, 'w') as script_f:
+            script_f.write(os.linesep.join(cmd_lines))
+        st = os.stat(script_file)
+        os.chmod(script_file, st.st_mode | stat.S_IEXEC)
+        return script_file
+    @property
+    def oe_init_build_env(self):
+        """Find the oe-init-build-env used for this setup"""
+        oe_init_dir = self.oe_init_dir
+        if oe_init_dir:
+            return os.path.join(oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV)
+        return None
+    @property
+    def oe_init_dir(self):
+        """Find the directory where the oe-init-build-env is located
+        Assumption: There might be a layer with higher priority than poky
+        which provides to oe-init-build-env in the layer's toplevel folder.
+        """
+        if not self.__oe_init_dir:
+            for layer in reversed(self.bblayers):
+                result = subprocess.run(
+                    ['git', 'rev-parse', '--show-toplevel'], cwd=layer, capture_output=True)
+                if result.returncode == 0:
+                    oe_init_dir = result.stdout.decode('utf-8').strip()
+                    oe_init_path = os.path.join(
+                        oe_init_dir, RecipeModified.OE_INIT_BUILD_ENV)
+                    if os.path.exists(oe_init_path):
+                        logger.debug("Using %s from: %s" % (
+                            RecipeModified.OE_INIT_BUILD_ENV, oe_init_path))
+                        self.__oe_init_dir = oe_init_dir
+                        break
+            if not self.__oe_init_dir:
+                logger.error("Cannot find the bitbake top level folder")
+        return self.__oe_init_dir
+def ide_setup(args, config, basepath, workspace):
+    """Generate the IDE configuration for the workspace"""
+    # Explicitely passing some special recipes does not make sense
+    for recipe in args.recipenames:
+        if recipe in ['meta-ide-support', 'build-sysroots']:
+            raise DevtoolError("Invalid recipe: %s." % recipe)
+    # Collect information about tasks which need to be bitbaked
+    bootstrap_tasks = []
+    bootstrap_tasks_late = []
+    tinfoil = setup_tinfoil(config_only=False, basepath=basepath)
+    try:
+        # define mode depending on recipes which need to be processed
+        recipes_image_names = []
+        recipes_modified_names = []
+        recipes_other_names = []
+        for recipe in args.recipenames:
+            try:
+                check_workspace_recipe(
+                    workspace, recipe, bbclassextend=True)
+                recipes_modified_names.append(recipe)
+            except DevtoolError:
+                recipe_d = parse_recipe(
+                    config, tinfoil, recipe, appends=True, filter_workspace=False)
+                if not recipe_d:
+                    raise DevtoolError("Parsing recipe %s failed" % recipe)
+                if bb.data.inherits_class('image', recipe_d):
+                    recipes_image_names.append(recipe)
+                else:
+                    recipes_other_names.append(recipe)
+        invalid_params = False
+        if args.mode == DevtoolIdeMode.shared:
+            if len(recipes_modified_names):
+                logger.error("In shared sysroots mode modified recipes %s cannot be handled." % str(
+                    recipes_modified_names))
+                invalid_params = True
+        if args.mode == DevtoolIdeMode.modified:
+            if len(recipes_other_names):
+                logger.error("Only in shared sysroots mode not modified recipes %s can be handled." % str(
+                    recipes_other_names))
+                invalid_params = True
+            if len(recipes_image_names) != 1:
+                logger.error(
+                    "One image recipe is required as the rootfs for the remote development.")
+                invalid_params = True
+            for modified_recipe_name in recipes_modified_names:
+                if modified_recipe_name.startswith('nativesdk-') or modified_recipe_name.endswith('-native'):
+                    logger.error(
+                        "Only cross compiled recipes are support. %s is not cross." % modified_recipe_name)
+                    invalid_params = True
+        if invalid_params:
+            raise DevtoolError("Invalid parameters are passed.")
+        # For the shared sysroots mode, add all dependencies of all the images to the sysroots
+        # For the modified mode provide one rootfs and the corresponding debug symbols via rootfs-dbg
+        recipes_images = []
+        for recipes_image_name in recipes_image_names:
+            logger.info("Using image: %s" % recipes_image_name)
+            recipe_image = RecipeImage(recipes_image_name)
+            recipe_image.initialize(config, tinfoil)
+            bootstrap_tasks += recipe_image.bootstrap_tasks
+            recipes_images.append(recipe_image)
+        # Provide a Direct SDK with shared sysroots
+        recipes_not_modified = []
+        if args.mode == DevtoolIdeMode.shared:
+            ide_support = RecipeMetaIdeSupport()
+            ide_support.initialize(config, tinfoil)
+            bootstrap_tasks += ide_support.bootstrap_tasks
+            logger.info("Adding %s to the Direct SDK sysroots." %
+                        str(recipes_other_names))
+            for recipe_name in recipes_other_names:
+                recipe_not_modified = RecipeNotModified(recipe_name)
+                bootstrap_tasks += recipe_not_modified.bootstrap_tasks
+                recipes_not_modified.append(recipe_not_modified)
+            build_sysroots = RecipeBuildSysroots()
+            build_sysroots.initialize(config, tinfoil)
+            bootstrap_tasks_late += build_sysroots.bootstrap_tasks
+            shared_env = SharedSysrootsEnv()
+            shared_env.initialize(ide_support, build_sysroots)
+        recipes_modified = []
+        if args.mode == DevtoolIdeMode.modified:
+            logger.info("Setting up workspaces for modified recipe: %s" %
+                        str(recipes_modified_names))
+            gdbs_cross = {}
+            for recipe_name in recipes_modified_names:
+                recipe_modified = RecipeModified(recipe_name)
+                recipe_modified.initialize(config, workspace, tinfoil)
+                bootstrap_tasks += recipe_modified.bootstrap_tasks
+                recipes_modified.append(recipe_modified)
+                if recipe_modified.target_arch not in gdbs_cross:
+                    target_device = TargetDevice(args)
+                    gdb_cross = RecipeGdbCross(
+                        args, recipe_modified.target_arch, target_device)
+                    gdb_cross.initialize(config, workspace, tinfoil)
+                    bootstrap_tasks += gdb_cross.bootstrap_tasks
+                    gdbs_cross[recipe_modified.target_arch] = gdb_cross
+                recipe_modified.gdb_cross = gdbs_cross[recipe_modified.target_arch]
+    finally:
+        tinfoil.shutdown()
+    if not args.skip_bitbake:
+        bb_cmd = 'bitbake '
+        if args.bitbake_k:
+            bb_cmd += "-k "
+        bb_cmd_early = bb_cmd + ' '.join(bootstrap_tasks)
+        exec_build_env_command(
+            config.init_path, basepath, bb_cmd_early, watch=True)
+        if bootstrap_tasks_late:
+            bb_cmd_late = bb_cmd + ' '.join(bootstrap_tasks_late)
+            exec_build_env_command(
+                config.init_path, basepath, bb_cmd_late, watch=True)
+    for recipe_image in recipes_images:
+        if (recipe_image.gdbserver_missing):
+            logger.warning(
+                "gdbserver not installed in image %s. Remote debugging will not be available" % recipe_image)
+        if recipe_image.combine_dbg_image is False:
+            logger.warning(
+                'IMAGE_CLASSES += "image-combined-dbg" is missing for image %s. Remote debugging will not find debug symbols from rootfs-dbg.' % recipe_image)
+    # Instantiate the active IDE plugin
+    ide = ide_plugins[args.ide]()
+    if args.mode == DevtoolIdeMode.shared:
+        ide.setup_shared_sysroots(shared_env)
+    elif args.mode == DevtoolIdeMode.modified:
+        for recipe_modified in recipes_modified:
+            if recipe_modified.build_tool is BuildTool.CMAKE:
+                recipe_modified.cmake_preset()
+            if recipe_modified.build_tool is BuildTool.MESON:
+                recipe_modified.gen_meson_wrapper()
+            ide.setup_modified_recipe(
+                args, recipe_image, recipe_modified)
+            if recipe_modified.debug_build != '1':
+                logger.warn(
+                    'Recipe %s is compiled with release build configuration. '
+                    'You might want to add DEBUG_BUILD = "1" to %s. '
+                    'Note that devtool modify --debug-build can do this automatically.',
+                    recipe_modified.name, recipe_modified.bbappend)
+    else:
+        raise DevtoolError("Must not end up here.")
+def register_commands(subparsers, context):
+    """Register devtool subcommands from this plugin"""
+    # The ide-sdk command bootstraps the SDK from the bitbake environment before the IDE
+    # configuration is generated. In the case of the eSDK, the bootstrapping is performed
+    # during the installation of the eSDK installer. Running the ide-sdk plugin from an
+    # eSDK installer-based setup would require skipping the bootstrapping and probably
+    # taking some other differences into account when generating the IDE configurations.
+    # This would be possible. But it is not implemented.
+    if context.fixed_setup:
+        return
+    global ide_plugins
+    # Search for IDE plugins in all sub-folders named ide_plugins where devtool seraches for plugins.
+    pluginpaths = [os.path.join(path, 'ide_plugins')
+                   for path in context.pluginpaths]
+    ide_plugin_modules = []
+    for pluginpath in pluginpaths:
+        scriptutils.load_plugins(logger, ide_plugin_modules, pluginpath)
+    for ide_plugin_module in ide_plugin_modules:
+        if hasattr(ide_plugin_module, 'register_ide_plugin'):
+            ide_plugin_module.register_ide_plugin(ide_plugins)
+    # Sort plugins according to their priority. The first entry is the default IDE plugin.
+    ide_plugins = dict(sorted(ide_plugins.items(),
+                       key=lambda p: p[1].ide_plugin_priority(), reverse=True))
+    parser_ide_sdk = subparsers.add_parser('ide-sdk', group='working', order=50, formatter_class=RawTextHelpFormatter,
+                                           help='Setup the SDK and configure the IDE')
+    parser_ide_sdk.add_argument(
+        'recipenames', nargs='+', help='Generate an IDE configuration suitable to work on the given recipes.\n'
+        'Depending on the --mode parameter different types of SDKs and IDE configurations are generated.')
+    parser_ide_sdk.add_argument(
+        '-m', '--mode', type=DevtoolIdeMode, default=DevtoolIdeMode.modified,
+        help='Different SDK types are supported:\n'
+        '- "' + DevtoolIdeMode.modified.name + '" (default):\n'
+        '  devtool modify creates a workspace to work on the source code of a recipe.\n'
+        '  devtool ide-sdk builds the SDK and generates the IDE configuration(s) in the workspace directorie(s)\n'
+        '  Usage example:\n'
+        '    devtool modify cmake-example\n'
+        '    devtool ide-sdk cmake-example core-image-minimal\n'
+        '    Start the IDE in the workspace folder\n'
+        '  At least one devtool modified recipe plus one image recipe are required:\n'
+        '  The image recipe is used to generate the target image and the remote debug configuration.\n'
+        '- "' + DevtoolIdeMode.shared.name + '":\n'
+        '  Usage example:\n'
+        '    devtool ide-sdk -m ' + DevtoolIdeMode.shared.name + ' recipe(s)\n'
+        '  This command generates a cross-toolchain as well as the corresponding shared sysroot directories.\n'
+        '  To use this tool-chain the environment-* file found in the deploy..image folder needs to be sourced into a shell.\n'
+        '  In case of VSCode and cmake the tool-chain is also exposed as a cmake-kit')
+    default_ide = list(ide_plugins.keys())[0]
+    parser_ide_sdk.add_argument(
+        '-i', '--ide', choices=ide_plugins.keys(), default=default_ide,
+        help='Setup the configuration for this IDE (default: %s)' % default_ide)
+    parser_ide_sdk.add_argument(
+        '-t', '--target', default='root@192.168.7.2',
+        help='Live target machine running an ssh server: user@hostname.')
+    parser_ide_sdk.add_argument(
+        '-G', '--gdbserver-port-start', default="1234", help='port where gdbserver is listening.')
+    parser_ide_sdk.add_argument(
+        '-c', '--no-host-check', help='Disable ssh host key checking', action='store_true')
+    parser_ide_sdk.add_argument(
+        '-e', '--ssh-exec', help='Executable to use in place of ssh')
+    parser_ide_sdk.add_argument(
+        '-P', '--port', help='Specify ssh port to use for connection to the target')
+    parser_ide_sdk.add_argument(
+        '-I', '--key', help='Specify ssh private key for connection to the target')
+    parser_ide_sdk.add_argument(
+        '--skip-bitbake', help='Generate IDE configuration but skip calling bitbake to update the SDK', action='store_true')
+    parser_ide_sdk.add_argument(
+        '-k', '--bitbake-k', help='Pass -k parameter to bitbake', action='store_true')
+    parser_ide_sdk.add_argument(
+        '--no-strip', help='Do not strip executables prior to deploy', dest='strip', action='store_false')
+    parser_ide_sdk.add_argument(
+        '-n', '--dry-run', help='List files to be undeployed only', action='store_true')
+    parser_ide_sdk.add_argument(
+        '-s', '--show-status', help='Show progress/status output', action='store_true')
+    parser_ide_sdk.add_argument(
+        '-p', '--no-preserve', help='Do not preserve existing files', action='store_true')
+    parser_ide_sdk.add_argument(
+        '--no-check-space', help='Do not check for available space before deploying', action='store_true')
+    parser_ide_sdk.set_defaults(func=ide_setup)
diff --git a/scripts/lib/devtool/menuconfig.py b/scripts/lib/devtool/menuconfig.py
index 95384c5333..1054960551 100644
--- a/scripts/lib/devtool/menuconfig.py
+++ b/scripts/lib/devtool/menuconfig.py
@@ -3,6 +3,8 @@
 # Copyright (C) 2018 Xilinx
 # Written by: Chandana Kalluri <ckalluri@xilinx.com>
 #
+# SPDX-License-Identifier: MIT
+#
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 2 as
 # published by the Free Software Foundation.
@@ -21,9 +23,6 @@
 import os
 import bb
 import logging
-import argparse
-import re
-import glob
 from devtool import setup_tinfoil, parse_recipe, DevtoolError, standard, exec_build_env_command
 from devtool import check_workspace_recipe
 logger = logging.getLogger('devtool')
@@ -32,7 +31,6 @@ def menuconfig(args, config, basepath, workspace):
    """Entry point for the devtool 'menuconfig' subcommand"""
    rd = ""
-    kconfigpath = ""
    pn_src = ""
    localfilesdir = ""
    workspace_dir = ""
@@ -43,13 +41,12 @@ def menuconfig(args, config, basepath, workspace):
            return 1
        check_workspace_recipe(workspace, args.component)
-        pn = rd.getVar('PN', True)
+        pn = rd.getVar('PN')
        if not rd.getVarFlag('do_menuconfig','task'):
            raise DevtoolError("This recipe does not support menuconfig option")
        workspace_dir = os.path.join(config.workspace_path,'sources')
-        kconfigpath = rd.getVar('B')
        pn_src = os.path.join(workspace_dir,pn)
        # add check to see if oe_local_files exists or not
@@ -68,7 +65,7 @@ def menuconfig(args, config, basepath, workspace):
    logger.info('Launching menuconfig')
    exec_build_env_command(config.init_path, basepath, 'bitbake -c menuconfig %s' % pn, watch=True)
    fragment = os.path.join(localfilesdir, 'devtool-fragment.cfg')
-    res = standard._create_kconfig_diff(pn_src,rd,fragment)
+    standard._create_kconfig_diff(pn_src,rd,fragment)
    return 0
diff --git a/scripts/lib/devtool/sdk.py b/scripts/lib/devtool/sdk.py
index 3aa42a1466..9aefd7e354 100644
--- a/scripts/lib/devtool/sdk.py
+++ b/scripts/lib/devtool/sdk.py
@@ -207,7 +207,7 @@ def sdk_update(args, config, basepath, workspace):
        if not sstate_mirrors:
            with open(os.path.join(conf_dir, 'site.conf'), 'a') as f:
                f.write('SCONF_VERSION = "%s"\n' % site_conf_version)
-                f.write('SSTATE_MIRRORS_append = " file://.* %s/sstate-cache/PATH \\n "\n' % updateserver)
+                f.write('SSTATE_MIRRORS:append = " file://.* %s/sstate-cache/PATH"\n' % updateserver)
    finally:
        shutil.rmtree(tmpsdk_dir)
@@ -300,7 +300,8 @@ def sdk_install(args, config, basepath, workspace):
            return 2
        try:
-            exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots', watch=True)
+            exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_native_sysroot', watch=True)
+            exec_build_env_command(config.init_path, basepath, 'bitbake build-sysroots -c build_target_sysroot', watch=True)
        except bb.process.ExecutionError as e:
            raise DevtoolError('Failed to bitbake build-sysroots:\n%s' % (str(e)))
diff --git a/scripts/lib/devtool/search.py b/scripts/lib/devtool/search.py
index d24040df37..70b81cac5e 100644
--- a/scripts/lib/devtool/search.py
+++ b/scripts/lib/devtool/search.py
@@ -62,10 +62,11 @@ def search(args, config, basepath, workspace):
                            with open(os.path.join(pkgdata_dir, 'runtime', pkg), 'r') as f:
                                for line in f:
                                    if ': ' in line:
-                                        splitline = line.split(':', 1)
+                                        splitline = line.split(': ', 1)
                                        key = splitline[0]
                                        value = splitline[1].strip()
-                                    if key in ['PKG_%s' % pkg, 'DESCRIPTION', 'FILES_INFO'] or key.startswith('FILERPROVIDES_'):
+                                    key = key.replace(":" + pkg, "")
+                                    if key in ['PKG', 'DESCRIPTION', 'FILES_INFO', 'FILERPROVIDES']:
                                        if keyword_rc.search(value):
                                            match = True
                                            break
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py
index 7b62b7e7b8..1fd5947c41 100644
--- a/scripts/lib/devtool/standard.py
+++ b/scripts/lib/devtool/standard.py
@@ -18,11 +18,13 @@ import argparse_oe
 import scriptutils
 import errno
 import glob
-import filecmp
 from collections import OrderedDict
 from devtool import exec_build_env_command, setup_tinfoil, check_workspace_recipe, use_external_build, setup_git_repo, recipe_to_append, get_bbclassextend_targets, update_unlockedsigs, check_prerelease_version, check_git_repo_dirty, check_git_repo_op, DevtoolError
 from devtool import parse_recipe
+import bb.utils
 logger = logging.getLogger('devtool')
 override_branch_prefix = 'devtool-override-'
@@ -30,7 +32,8 @@ override_branch_prefix = 'devtool-override-'
 def add(args, config, basepath, workspace):
    """Entry point for the devtool 'add' subcommand"""
-    import bb
+    import bb.data
+    import bb.process
    import oe.recipeutils
    if not args.recipename and not args.srctree and not args.fetch and not args.fetchuri:
@@ -147,6 +150,8 @@ def add(args, config, basepath, workspace):
        extracmdopts += ' -a'
    if args.npm_dev:
        extracmdopts += ' --npm-dev'
+    if args.no_pypi:
+        extracmdopts += ' --no-pypi'
    if args.mirrors:
        extracmdopts += ' --mirrors'
    if args.srcrev:
@@ -204,7 +209,7 @@ def add(args, config, basepath, workspace):
            for fn in os.listdir(tempdir):
                shutil.move(os.path.join(tempdir, fn), recipedir)
        else:
-            raise DevtoolError('Command \'%s\' did not create any recipe file:\n%s' % (e.command, e.stdout))
+            raise DevtoolError(f'Failed to create a recipe file for source {source}')
        attic_recipe = os.path.join(config.workspace_path, 'attic', recipename, os.path.basename(recipefile))
        if os.path.exists(attic_recipe):
            logger.warning('A modified recipe from a previous invocation exists in %s - you may wish to move this over the top of the new recipe if you had changes in it that you want to continue with' % attic_recipe)
@@ -234,10 +239,14 @@ def add(args, config, basepath, workspace):
        if args.fetchuri and not args.no_git:
            setup_git_repo(srctree, args.version, 'devtool', d=tinfoil.config_data)
-        initial_rev = None
+        initial_rev = {}
        if os.path.exists(os.path.join(srctree, '.git')):
            (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
-            initial_rev = stdout.rstrip()
+            initial_rev["."] = stdout.rstrip()
+            (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse HEAD` $PWD\'', cwd=srctree)
+            for line in stdout.splitlines():
+                (rev, submodule) = line.split()
+                initial_rev[os.path.relpath(submodule, srctree)] = rev
        if args.src_subdir:
            srctree = os.path.join(srctree, args.src_subdir)
@@ -251,16 +260,17 @@ def add(args, config, basepath, workspace):
            if b_is_s:
                f.write('EXTERNALSRC_BUILD = "%s"\n' % srctree)
            if initial_rev:
-                f.write('\n# initial_rev: %s\n' % initial_rev)
+                for key, value in initial_rev.items():
+                    f.write('\n# initial_rev %s: %s\n' % (key, value))
            if args.binary:
-                f.write('do_install_append() {\n')
+                f.write('do_install:append() {\n')
                f.write('    rm -rf ${D}/.git\n')
                f.write('    rm -f ${D}/singletask.lock\n')
                f.write('}\n')
            if bb.data.inherits_class('npm', rd):
-                f.write('python do_configure_append() {\n')
+                f.write('python do_configure:append() {\n')
                f.write('    pkgdir = d.getVar("NPM_PACKAGE")\n')
                f.write('    lockfile = os.path.join(pkgdir, "singletask.lock")\n')
                f.write('    bb.utils.remove(lockfile)\n')
@@ -298,6 +308,7 @@ def add(args, config, basepath, workspace):
 def _check_compatible_recipe(pn, d):
    """Check if the recipe is supported by devtool"""
+    import bb.data
    if pn == 'perf':
        raise DevtoolError("The perf recipe does not actually check out "
                           "source and thus cannot be supported by this tool",
@@ -318,10 +329,6 @@ def _check_compatible_recipe(pn, d):
        raise DevtoolError("The %s recipe is a packagegroup, and therefore is "
                           "not supported by this tool" % pn, 4)
-    if bb.data.inherits_class('meta', d):
-        raise DevtoolError("The %s recipe is a meta-recipe, and therefore is "
-                           "not supported by this tool" % pn, 4)
    if bb.data.inherits_class('externalsrc', d) and d.getVar('EXTERNALSRC'):
        # Not an incompatibility error per se, so we don't pass the error code
        raise DevtoolError("externalsrc is currently enabled for the %s "
@@ -357,7 +364,7 @@ def _move_file(src, dst, dry_run_outdir=None, base_outdir=None):
            bb.utils.mkdirhier(dst_d)
        shutil.move(src, dst)
-def _copy_file(src, dst, dry_run_outdir=None):
+def _copy_file(src, dst, dry_run_outdir=None, base_outdir=None):
    """Copy a file. Creates all the directory components of destination path."""
    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
    logger.debug('Copying %s to %s%s' % (src, dst, dry_run_suffix))
@@ -371,7 +378,7 @@ def _copy_file(src, dst, dry_run_outdir=None):
 def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
    """List contents of a git treeish"""
-    import bb
+    import bb.process
    cmd = ['git', 'ls-tree', '-z', treeish]
    if recursive:
        cmd.append('-r')
@@ -384,6 +391,19 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False):
                ret[split[3]] = split[0:3]
    return ret
+def _git_modified(repodir):
+    """List the difference between HEAD and the index"""
+    import bb.process
+    cmd = ['git', 'status', '--porcelain']
+    out, _ = bb.process.run(cmd, cwd=repodir)
+    ret = []
+    if out:
+        for line in out.split("\n"):
+            if line and not line.startswith('??'):
+                ret.append(line[3:])
+    return ret
 def _git_exclude_path(srctree, path):
    """Return pathspec (list of paths) that excludes certain path"""
    # NOTE: "Filtering out" files/paths in this way is not entirely reliable -
@@ -411,8 +431,6 @@ def _ls_tree(directory):
 def extract(args, config, basepath, workspace):
    """Entry point for the devtool 'extract' subcommand"""
-    import bb
    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
    if not tinfoil:
        # Error already shown
@@ -435,8 +453,6 @@ def extract(args, config, basepath, workspace):
 def sync(args, config, basepath, workspace):
    """Entry point for the devtool 'sync' subcommand"""
-    import bb
    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
    if not tinfoil:
        # Error already shown
@@ -457,41 +473,11 @@ def sync(args, config, basepath, workspace):
    finally:
        tinfoil.shutdown()
-def symlink_oelocal_files_srctree(rd,srctree):
-    import oe.patch
-    if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')):
-        # If recipe extracts to ${WORKDIR}, symlink the files into the srctree
-        # (otherwise the recipe won't build as expected)
-        local_files_dir = os.path.join(srctree, 'oe-local-files')
-        addfiles = []
-        for root, _, files in os.walk(local_files_dir):
-            relpth = os.path.relpath(root, local_files_dir)
-            if relpth != '.':
-                bb.utils.mkdirhier(os.path.join(srctree, relpth))
-            for fn in files:
-                if fn == '.gitignore':
-                    continue
-                destpth = os.path.join(srctree, relpth, fn)
-                if os.path.exists(destpth):
-                    os.unlink(destpth)
-                if relpth != '.':
-                    back_relpth = os.path.relpath(local_files_dir, root)
-                    os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth)
-                else:
-                    os.symlink('oe-local-files/%s' % fn, destpth)
-                addfiles.append(os.path.join(relpth, fn))
-        if addfiles:
-            bb.process.run('git add %s' % ' '.join(addfiles), cwd=srctree)
-            useroptions = []
-            oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
-            bb.process.run('git %s commit -m "Committing local file symlinks\n\n%s"' % (' '.join(useroptions), oe.patch.GitApplyTree.ignore_commit_prefix), cwd=srctree)
 def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False):
    """Extract sources of a recipe"""
-    import oe.recipeutils
-    import oe.patch
    import oe.path
+    import bb.data
+    import bb.process
    pn = d.getVar('PN')
@@ -523,8 +509,10 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
        history = d.varhistory.variable('SRC_URI')
        for event in history:
            if not 'flag' in event:
-                if event['op'].startswith(('_append[', '_prepend[')):
+                if event['op'].startswith((':append[', ':prepend[')):
-                    extra_overrides.append(event['op'].split('[')[1].split(']')[0])
+                    override = event['op'].split('[')[1].split(']')[0]
+                    if not override.startswith('pn-'):
+                        extra_overrides.append(override)
        # We want to remove duplicate overrides. If a recipe had multiple
        # SRC_URI_override += values it would cause mulitple instances of
        # overrides. This doesn't play nicely with things like creating a
@@ -554,6 +542,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
    tempbasedir = d.getVar('WORKDIR')
    bb.utils.mkdirhier(tempbasedir)
    tempdir = tempfile.mkdtemp(prefix='devtooltmp-', dir=tempbasedir)
+    appendbackup = None
    try:
        tinfoil.logger.setLevel(logging.WARNING)
@@ -564,11 +553,13 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
            appendbackup = os.path.join(tempdir, os.path.basename(appendfile) + '.bak')
            shutil.copyfile(appendfile, appendbackup)
        else:
-            appendbackup = None
            bb.utils.mkdirhier(os.path.dirname(appendfile))
        logger.debug('writing append file %s' % appendfile)
        with open(appendfile, 'a') as f:
            f.write('###--- _extract_source\n')
+            f.write('deltask do_recipe_qa\n')
+            f.write('deltask do_recipe_qa_setscene\n')
+            f.write('ERROR_QA:remove = "patch-fuzz"\n')
            f.write('DEVTOOL_TEMPDIR = "%s"\n' % tempdir)
            f.write('DEVTOOL_DEVBRANCH = "%s"\n' % devbranch)
            if not is_kernel_yocto:
@@ -586,6 +577,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
        preservestampfile = os.path.join(sstate_manifests, 'preserve-stamps')
        with open(preservestampfile, 'w') as f:
            f.write(d.getVar('STAMP'))
+        tinfoil.modified_files()
        try:
            if is_kernel_yocto:
                # We need to generate the kernel config
@@ -633,7 +625,7 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
                srcsubdir = f.read()
        except FileNotFoundError as e:
            raise DevtoolError('Something went wrong with source extraction - the devtool-source class was not active or did not function correctly:\n%s' % str(e))
-        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir'))
+        srcsubdir_rel = os.path.relpath(srcsubdir, os.path.join(tempdir, 'workdir', os.path.relpath(d.getVar('UNPACKDIR'), d.getVar('WORKDIR'))))
        # Check if work-shared is empty, if yes
        # find source and copy to work-shared
@@ -648,39 +640,26 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
            if os.path.exists(workshareddir) and (not os.listdir(workshareddir) or kernelVersion != staging_kerVer):
                shutil.rmtree(workshareddir)
-                oe.path.copyhardlinktree(srcsubdir,workshareddir)
+                oe.path.copyhardlinktree(srcsubdir, workshareddir)
            elif not os.path.exists(workshareddir):
-                oe.path.copyhardlinktree(srcsubdir,workshareddir)
+                oe.path.copyhardlinktree(srcsubdir, workshareddir)
-        tempdir_localdir = os.path.join(tempdir, 'oe-local-files')
-        srctree_localdir = os.path.join(srctree, 'oe-local-files')
        if sync:
-            bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
+            try:
+                logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch))
-            # Move oe-local-files directory to srctree
+                bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree)
-            # As the oe-local-files is not part of the constructed git tree,
-            # remove them directly during the synchrounizating might surprise
+                # Use git fetch to update the source with the current recipe
-            # the users.  Instead, we move it to oe-local-files.bak and remind
+                # To be able to update the currently checked out branch with
-            # user in the log message.
+                # possibly new history (no fast-forward) git needs to be told
-            if os.path.exists(srctree_localdir + '.bak'):
+                # that's ok
-                shutil.rmtree(srctree_localdir, srctree_localdir + '.bak')
+                logger.info('Syncing source files including patches to git branch: %s' % devbranch)
+                bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree)
-            if os.path.exists(srctree_localdir):
+            except bb.process.ExecutionError as e:
-                logger.info('Backing up current local file directory %s' % srctree_localdir)
+                raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e))
-                shutil.move(srctree_localdir, srctree_localdir + '.bak')
-            if os.path.exists(tempdir_localdir):
-                logger.info('Syncing local source files to srctree...')
-                shutil.copytree(tempdir_localdir, srctree_localdir)
-        else:
-            # Move oe-local-files directory to srctree
-            if os.path.exists(tempdir_localdir):
-                logger.info('Adding local source files to srctree...')
-                shutil.move(tempdir_localdir, srcsubdir)
+        else:
            shutil.move(srcsubdir, srctree)
-            symlink_oelocal_files_srctree(d,srctree)
        if is_kernel_yocto:
            logger.info('Copying kernel config to srctree')
@@ -699,8 +678,6 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works
 def _add_md5(config, recipename, filename):
    """Record checksum of a file (or recursively for a directory) to the md5-file of the workspace"""
-    import bb.utils
    def addfile(fn):
        md5 = bb.utils.md5_file(fn)
        with open(os.path.join(config.workspace_path, '.devtool_md5'), 'a+') as f:
@@ -719,7 +696,6 @@ def _add_md5(config, recipename, filename):
 def _check_preserve(config, recipename):
    """Check if a file was manually changed and needs to be saved in 'attic'
       directory"""
-    import bb.utils
    origfile = os.path.join(config.workspace_path, '.devtool_md5')
    newfile = os.path.join(config.workspace_path, '.devtool_md5_new')
    preservepath = os.path.join(config.workspace_path, 'attic', recipename)
@@ -746,30 +722,40 @@ def _check_preserve(config, recipename):
                        os.remove(removefile)
                else:
                    tf.write(line)
-    os.rename(newfile, origfile)
+    bb.utils.rename(newfile, origfile)
 def get_staging_kver(srcdir):
    # Kernel version from work-shared
-    kerver = []
+    import itertools
-    staging_kerVer=""
+    try:
-    if os.path.exists(srcdir) and os.listdir(srcdir):
+        with open(os.path.join(srcdir, "Makefile")) as f:
-        with open(os.path.join(srcdir,"Makefile")) as f:
+            # Take VERSION, PATCHLEVEL, SUBLEVEL from lines 1, 2, 3
-            version = [next(f) for x in range(5)][1:4]
+            return ".".join(line.rstrip().split('= ')[1] for line in itertools.islice(f, 1, 4))
-            for word in version:
+    except FileNotFoundError:
-                kerver.append(word.split('= ')[1].split('\n')[0])
+        return ""
-            staging_kerVer = ".".join(kerver)
-    return staging_kerVer
 def get_staging_kbranch(srcdir):
+    import bb.process
    staging_kbranch = ""
    if os.path.exists(srcdir) and os.listdir(srcdir):
-        (branch, _) = bb.process.run('git branch | grep \* | cut -d \' \' -f2', cwd=srcdir)
+        (branch, _) = bb.process.run('git branch | grep \\* | cut -d \' \' -f2', cwd=srcdir)
        staging_kbranch = "".join(branch.split('\n')[0])
    return staging_kbranch
+def get_real_srctree(srctree, s, unpackdir):
+    # Check that recipe isn't using a shared workdir
+    s = os.path.abspath(s)
+    unpackdir = os.path.abspath(unpackdir)
+    if s.startswith(unpackdir) and s != unpackdir and os.path.dirname(s) != unpackdir:
+        # Handle if S is set to a subdirectory of the source
+        srcsubdir = os.path.relpath(s, unpackdir).split(os.sep, 1)[1]
+        srctree = os.path.join(srctree, srcsubdir)
+    return srctree
 def modify(args, config, basepath, workspace):
    """Entry point for the devtool 'modify' subcommand"""
-    import bb
+    import bb.data
+    import bb.process
    import oe.recipeutils
    import oe.patch
    import oe.path
@@ -811,8 +797,8 @@ def modify(args, config, basepath, workspace):
        _check_compatible_recipe(pn, rd)
-        initial_rev = None
+        initial_revs = {}
-        commits = []
+        commits = {}
        check_commits = False
        if bb.data.inherits_class('kernel-yocto', rd):
@@ -824,36 +810,22 @@ def modify(args, config, basepath, workspace):
            staging_kerVer = get_staging_kver(srcdir)
            staging_kbranch = get_staging_kbranch(srcdir)
            if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch):
-                oe.path.copyhardlinktree(srcdir,srctree)
+                oe.path.copyhardlinktree(srcdir, srctree)
-                workdir = rd.getVar('WORKDIR')
+                unpackdir = rd.getVar('UNPACKDIR')
                srcsubdir = rd.getVar('S')
-                localfilesdir = os.path.join(srctree,'oe-local-files')
-                # Move local source files into separate subdir
-                recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)]
-                local_files = oe.recipeutils.get_recipe_local_files(rd)
-                for key in local_files.copy():
+                # Add locally copied files to gitignore as we add back to the metadata directly
-                    if key.endswith('scc'):
+                local_files = oe.recipeutils.get_recipe_local_files(rd)
-                        sccfile = open(local_files[key], 'r')
-                        for l in sccfile:
-                            line = l.split()
-                            if line and line[0] in ('kconf', 'patch'):
-                                cfg = os.path.join(os.path.dirname(local_files[key]), line[-1])
-                                if not cfg in local_files.values():
-                                    local_files[line[-1]] = cfg
-                                    shutil.copy2(cfg, workdir)
-                        sccfile.close()
-                # Ignore local files with subdir={BP}
                srcabspath = os.path.abspath(srcsubdir)
-                local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and  (srcabspath == workdir or not  os.path.join(workdir, fname).startswith(srcabspath + os.sep))]
+                local_files = [fname for fname in local_files if
+                               os.path.exists(os.path.join(unpackdir, fname)) and
+                               srcabspath == unpackdir]
                if local_files:
-                    for fname in local_files:
+                    with open(os.path.join(srctree, '.gitignore'), 'a+') as f:
-                        _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname))
+                        f.write('# Ignore local files, by default. Remove following lines'
-                    with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f:
+                                'if you want to commit the directory to Git\n')
-                        f.write('# Ignore local files, by default. Remove this file ''if you want to commit the directory to Git\n*\n')
+                        for fname in local_files:
+                            f.write('%s\n' % fname)
-                symlink_oelocal_files_srctree(rd,srctree)
                task = 'do_configure'
                res = tinfoil.build_targets(pn, task, handle_events=True)
@@ -861,22 +833,33 @@ def modify(args, config, basepath, workspace):
                # Copy .config to workspace
                kconfpath = rd.getVar('B')
                logger.info('Copying kernel config to workspace')
-                shutil.copy2(os.path.join(kconfpath, '.config'),srctree)
+                shutil.copy2(os.path.join(kconfpath, '.config'), srctree)
                # Set this to true, we still need to get initial_rev
                # by parsing the git repo
                args.no_extract = True
        if not args.no_extract:
-            initial_rev, _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
+            initial_revs["."], _ = _extract_source(srctree, args.keep_temp, args.branch, False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
-            if not initial_rev:
+            if not initial_revs["."]:
                return 1
            logger.info('Source tree extracted to %s' % srctree)
            if os.path.exists(os.path.join(srctree, '.git')):
                # Get list of commits since this revision
-                (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_rev, cwd=srctree)
+                (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree)
-                commits = stdout.split()
+                commits["."] = stdout.split()
                check_commits = True
+                try:
+                    (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive  \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree)
+                except bb.process.ExecutionError:
+                    stdout = ""
+                for line in stdout.splitlines():
+                    (rev, submodule_path) = line.split()
+                    submodule = os.path.relpath(submodule_path, srctree)
+                    initial_revs[submodule] = rev
+                    (stdout, _) = bb.process.run('git rev-list --reverse devtool-base..HEAD', cwd=submodule_path)
+                    commits[submodule] = stdout.split()
        else:
            if os.path.exists(os.path.join(srctree, '.git')):
                # Check if it's a tree previously extracted by us. This is done
@@ -893,11 +876,11 @@ def modify(args, config, basepath, workspace):
                for line in stdout.splitlines():
                    if line.startswith('*'):
                        (stdout, _) = bb.process.run('git rev-parse devtool-base', cwd=srctree)
-                        initial_rev = stdout.rstrip()
+                        initial_revs["."] = stdout.rstrip()
-                if not initial_rev:
+                if "." not in initial_revs:
                    # Otherwise, just grab the head revision
                    (stdout, _) = bb.process.run('git rev-parse HEAD', cwd=srctree)
-                    initial_rev = stdout.rstrip()
+                    initial_revs["."] = stdout.rstrip()
        branch_patches = {}
        if check_commits:
@@ -914,67 +897,81 @@ def modify(args, config, basepath, workspace):
            seen_patches = []
            for branch in branches:
                branch_patches[branch] = []
-                (stdout, _) = bb.process.run('git log devtool-base..%s' % branch, cwd=srctree)
+                (stdout, _) = bb.process.run('git rev-list devtool-base..%s' % branch, cwd=srctree)
-                for line in stdout.splitlines():
+                for sha1 in stdout.splitlines():
-                    line = line.strip()
+                    notes = oe.patch.GitApplyTree.getNotes(srctree, sha1.strip())
-                    if line.startswith(oe.patch.GitApplyTree.patch_line_prefix):
+                    origpatch = notes.get(oe.patch.GitApplyTree.original_patch)
-                        origpatch = line[len(oe.patch.GitApplyTree.patch_line_prefix):].split(':', 1)[-1].strip()
+                    if origpatch and origpatch not in seen_patches:
-                        if not origpatch in seen_patches:
+                        seen_patches.append(origpatch)
-                            seen_patches.append(origpatch)
+                        branch_patches[branch].append(origpatch)
-                            branch_patches[branch].append(origpatch)
        # Need to grab this here in case the source is within a subdirectory
        srctreebase = srctree
+        srctree = get_real_srctree(srctree, rd.getVar('S'), rd.getVar('UNPACKDIR'))
-        # Check that recipe isn't using a shared workdir
-        s = os.path.abspath(rd.getVar('S'))
-        workdir = os.path.abspath(rd.getVar('WORKDIR'))
-        if s.startswith(workdir) and s != workdir and os.path.dirname(s) != workdir:
-            # Handle if S is set to a subdirectory of the source
-            srcsubdir = os.path.relpath(s, workdir).split(os.sep, 1)[1]
-            srctree = os.path.join(srctree, srcsubdir)
        bb.utils.mkdirhier(os.path.dirname(appendfile))
        with open(appendfile, 'w') as f:
-            f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n')
+            # if not present, add type=git-dependency to the secondary sources
+            # (non local files) so they can be extracted correctly when building a recipe after
+            #  doing a devtool modify on it
+            src_uri = rd.getVar('SRC_URI').split()
+            src_uri_append = []
+            src_uri_remove = []
+            # Assume first entry is main source extracted in ${S} so skip it
+            src_uri = src_uri[1::]
+            # Add "type=git-dependency" to all non local sources
+            for url in src_uri:
+                if not url.startswith('file://') and not 'type=' in url:
+                    src_uri_remove.append(url)
+                    src_uri_append.append('%s;type=git-dependency' % url)
+            if src_uri_remove:
+                f.write('SRC_URI:remove = "%s"\n' % ' '.join(src_uri_remove))
+                f.write('SRC_URI:append = " %s"\n\n' % ' '.join(src_uri_append))
+            f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n')
            # Local files can be modified/tracked in separate subdir under srctree
            # Mostly useful for packages with S != WORKDIR
-            f.write('FILESPATH_prepend := "%s:"\n' %
+            f.write('FILESPATH:prepend := "%s:"\n' %
                    os.path.join(srctreebase, 'oe-local-files'))
            f.write('# srctreebase: %s\n' % srctreebase)
            f.write('\ninherit externalsrc\n')
            f.write('# NOTE: We use pn- overrides here to avoid affecting multiple variants in the case where the recipe uses BBCLASSEXTEND\n')
-            f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
+            f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
            b_is_s = use_external_build(args.same_dir, args.no_same_dir, rd)
            if b_is_s:
-                f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
+                f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
            if bb.data.inherits_class('kernel', rd):
-                f.write('SRCTREECOVEREDTASKS = "do_validate_branches do_kernel_checkout '
+                f.write('\ndo_kernel_configme:prepend() {\n'
-                        'do_fetch do_unpack do_kernel_configme do_kernel_configcheck"\n')
+                        '    if [ -e ${S}/.config ]; then\n'
-                f.write('\ndo_patch[noexec] = "1"\n')
+                        '        mv ${S}/.config ${S}/.config.old\n'
-                f.write('\ndo_configure_append() {\n'
+                        '    fi\n'
-                        '    cp ${B}/.config ${S}/.config.baseline\n'
-                        '    ln -sfT ${B}/.config ${S}/.config.new\n'
                        '}\n')
-            if rd.getVarFlag('do_menuconfig','task'):
+            if rd.getVarFlag('do_menuconfig', 'task'):
-                f.write('\ndo_configure_append() {\n'
+                f.write('\ndo_configure:append() {\n'
-                '    if [ ! ${DEVTOOL_DISABLE_MENUCONFIG} ]; then\n'
+                '    if [ ${@oe.types.boolean(d.getVar("KCONFIG_CONFIG_ENABLE_MENUCONFIG"))} = True ]; then\n'
-                '        cp ${B}/.config ${S}/.config.baseline\n'
+                '        cp ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.baseline\n'
-                '        ln -sfT ${B}/.config ${S}/.config.new\n'
+                '        ln -sfT ${KCONFIG_CONFIG_ROOTDIR}/.config ${S}/.config.new\n'
                '    fi\n'
                '}\n')
-            if initial_rev:
+            if initial_revs:
-                f.write('\n# initial_rev: %s\n' % initial_rev)
+                for name, rev in initial_revs.items():
-                for commit in commits:
+                    f.write('\n# initial_rev %s: %s\n' % (name, rev))
-                    f.write('# commit: %s\n' % commit)
+                    if name in commits:
+                        for commit in commits[name]:
+                            f.write('# commit %s: %s\n' % (name, commit))
            if branch_patches:
                for branch in branch_patches:
                    if branch == args.branch:
                        continue
                    f.write('# patches_%s: %s\n' % (branch, ','.join(branch_patches[branch])))
+            if args.debug_build:
+                f.write('\nDEBUG_BUILD = "1"\n')
        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
@@ -1019,6 +1016,7 @@ def rename(args, config, basepath, workspace):
        origfnver = ''
    recipefilemd5 = None
+    newrecipefilemd5 = None
    tinfoil = setup_tinfoil(basepath=basepath, tracking=True)
    try:
        rd = parse_recipe(config, tinfoil, args.recipename, True)
@@ -1089,13 +1087,14 @@ def rename(args, config, basepath, workspace):
    # Rename bbappend
    logger.info('Renaming %s to %s' % (append, newappend))
-    os.rename(append, newappend)
+    bb.utils.rename(append, newappend)
    # Rename recipe file
    logger.info('Renaming %s to %s' % (recipefile, newfile))
-    os.rename(recipefile, newfile)
+    bb.utils.rename(recipefile, newfile)
    # Rename source tree if it's the default path
    appendmd5 = None
+    newappendmd5 = None
    if not args.no_srctree:
        srctree = workspace[args.recipename]['srctree']
        if os.path.abspath(srctree) == os.path.join(config.workspace_path, 'sources', args.recipename):
@@ -1184,7 +1183,7 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre
    """Get initial and update rev of a recipe. These are the start point of the
    whole patchset and start point for the patches to be re-generated/updated.
    """
-    import bb
+    import bb.process
    # Get current branch
    stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
@@ -1192,44 +1191,56 @@ def _get_patchset_revs(srctree, recipe_path, initial_rev=None, force_patch_refre
    branchname = stdout.rstrip()
    # Parse initial rev from recipe if not specified
-    commits = []
+    commits = {}
    patches = []
+    initial_revs = {}
    with open(recipe_path, 'r') as f:
        for line in f:
-            if line.startswith('# initial_rev:'):
+            pattern = r'^#\s.*\s(.*):\s([0-9a-fA-F]+)$'
-                if not initial_rev:
+            match = re.search(pattern, line)
-                    initial_rev = line.split(':')[-1].strip()
+            if match:
-            elif line.startswith('# commit:') and not force_patch_refresh:
+                name = match.group(1)
-                commits.append(line.split(':')[-1].strip())
+                rev = match.group(2)
-            elif line.startswith('# patches_%s:' % branchname):
+                if line.startswith('# initial_rev'):
-                patches = line.split(':')[-1].strip().split(',')
+                    if not (name == "." and initial_rev):
+                        initial_revs[name] = rev
-    update_rev = initial_rev
+                elif line.startswith('# commit') and not force_patch_refresh:
-    changed_revs = None
+                    if name not in commits:
-    if initial_rev:
+                        commits[name] = [rev]
+                    else:
+                        commits[name].append(rev)
+                elif line.startswith('# patches_%s:' % branchname):
+                    patches = line.split(':')[-1].strip().split(',')
+    update_revs = dict(initial_revs)
+    changed_revs = {}
+    for name, rev in initial_revs.items():
        # Find first actually changed revision
        stdout, _ = bb.process.run('git rev-list --reverse %s..HEAD' %
-                                   initial_rev, cwd=srctree)
+                                   rev, cwd=os.path.join(srctree, name))
        newcommits = stdout.split()
-        for i in range(min(len(commits), len(newcommits))):
+        if name in commits:
-            if newcommits[i] == commits[i]:
+            for i in range(min(len(commits[name]), len(newcommits))):
-                update_rev = commits[i]
+                if newcommits[i] == commits[name][i]:
+                    update_revs[name] = commits[name][i]
        try:
            stdout, _ = bb.process.run('git cherry devtool-patched',
-                                        cwd=srctree)
+                                        cwd=os.path.join(srctree, name))
        except bb.process.ExecutionError as err:
            stdout = None
        if stdout is not None and not force_patch_refresh:
-            changed_revs = []
            for line in stdout.splitlines():
                if line.startswith('+ '):
                    rev = line.split()[1]
                    if rev in newcommits:
-                        changed_revs.append(rev)
+                        if name not in changed_revs:
+                            changed_revs[name] = [rev]
+                        else:
+                            changed_revs[name].append(rev)
-    return initial_rev, update_rev, changed_revs, patches
+    return initial_revs, update_revs, changed_revs, patches
 def _remove_file_entries(srcuri, filelist):
    """Remove file:// entries from SRC_URI"""
@@ -1284,17 +1295,21 @@ def _remove_source_files(append, files, destpath, no_report_remove=False, dry_ru
                        raise
-def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
+def _export_patches(srctree, rd, start_revs, destdir, changed_revs=None):
    """Export patches from srctree to given location.
       Returns three-tuple of dicts:
         1. updated - patches that already exist in SRCURI
         2. added - new patches that don't exist in SRCURI
         3  removed - patches that exist in SRCURI but not in exported patches
-      In each dict the key is the 'basepath' of the URI and value is the
+       In each dict the key is the 'basepath' of the URI and value is:
-      absolute path to the existing file in recipe space (if any).
+         - for updated and added dicts, a dict with 2 optionnal keys:
+            - 'path': the absolute path to the existing file in recipe space (if any)
+            - 'patchdir': the directory in wich the patch should be applied (if any)
+         - for removed dict, the absolute path to the existing file in recipe space
    """
    import oe.recipeutils
    from oe.patch import GitApplyTree
+    import bb.process
    updated = OrderedDict()
    added = OrderedDict()
    seqpatch_re = re.compile('^([0-9]{4}-)?(.+)')
@@ -1305,59 +1320,67 @@ def _export_patches(srctree, rd, start_rev, destdir, changed_revs=None):
    # Generate patches from Git, exclude local files directory
    patch_pathspec = _git_exclude_path(srctree, 'oe-local-files')
-    GitApplyTree.extractPatches(srctree, start_rev, destdir, patch_pathspec)
+    GitApplyTree.extractPatches(srctree, start_revs, destdir, patch_pathspec)
+    for dirpath, dirnames, filenames in os.walk(destdir):
-    new_patches = sorted(os.listdir(destdir))
+        new_patches = filenames
-    for new_patch in new_patches:
+        reldirpath = os.path.relpath(dirpath, destdir)
-        # Strip numbering from patch names. If it's a git sequence named patch,
+        for new_patch in new_patches:
-        # the numbers might not match up since we are starting from a different
+            # Strip numbering from patch names. If it's a git sequence named patch,
-        # revision This does assume that people are using unique shortlog
+            # the numbers might not match up since we are starting from a different
-        # values, but they ought to be anyway...
+            # revision This does assume that people are using unique shortlog
-        new_basename = seqpatch_re.match(new_patch).group(2)
+            # values, but they ought to be anyway...
-        match_name = None
+            new_basename = seqpatch_re.match(new_patch).group(2)
-        for old_patch in existing_patches:
+            match_name = None
-            old_basename = seqpatch_re.match(old_patch).group(2)
+            old_patch = None
-            old_basename_splitext = os.path.splitext(old_basename)
+            for old_patch in existing_patches:
-            if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
+                old_basename = seqpatch_re.match(old_patch).group(2)
-                old_patch_noext = os.path.splitext(old_patch)[0]
+                old_basename_splitext = os.path.splitext(old_basename)
-                match_name = old_patch_noext
+                if old_basename.endswith(('.gz', '.bz2', '.Z')) and old_basename_splitext[0] == new_basename:
-                break
+                    old_patch_noext = os.path.splitext(old_patch)[0]
-            elif new_basename == old_basename:
+                    match_name = old_patch_noext
-                match_name = old_patch
+                    break
-                break
+                elif new_basename == old_basename:
-        if match_name:
+                    match_name = old_patch
-            # Rename patch files
+                    break
-            if new_patch != match_name:
+            if match_name:
-                os.rename(os.path.join(destdir, new_patch),
+                # Rename patch files
-                          os.path.join(destdir, match_name))
+                if new_patch != match_name:
-            # Need to pop it off the list now before checking changed_revs
+                    bb.utils.rename(os.path.join(destdir, new_patch),
-            oldpath = existing_patches.pop(old_patch)
+                              os.path.join(destdir, match_name))
-            if changed_revs is not None:
+                # Need to pop it off the list now before checking changed_revs
-                # Avoid updating patches that have not actually changed
+                oldpath = existing_patches.pop(old_patch)
-                with open(os.path.join(destdir, match_name), 'r') as f:
+                if changed_revs is not None and dirpath in changed_revs:
-                    firstlineitems = f.readline().split()
+                    # Avoid updating patches that have not actually changed
-                    # Looking for "From <hash>" line
+                    with open(os.path.join(dirpath, match_name), 'r') as f:
-                    if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
+                        firstlineitems = f.readline().split()
-                        if not firstlineitems[1] in changed_revs:
+                        # Looking for "From <hash>" line
-                            continue
+                        if len(firstlineitems) > 1 and len(firstlineitems[1]) == 40:
-            # Recompress if necessary
+                            if not firstlineitems[1] in changed_revs[dirpath]:
-            if oldpath.endswith(('.gz', '.Z')):
+                                continue
-                bb.process.run(['gzip', match_name], cwd=destdir)
+                # Recompress if necessary
-                if oldpath.endswith('.gz'):
+                if oldpath.endswith(('.gz', '.Z')):
-                    match_name += '.gz'
+                    bb.process.run(['gzip', match_name], cwd=destdir)
-                else:
+                    if oldpath.endswith('.gz'):
-                    match_name += '.Z'
+                        match_name += '.gz'
-            elif oldpath.endswith('.bz2'):
+                    else:
-                bb.process.run(['bzip2', match_name], cwd=destdir)
+                        match_name += '.Z'
-                match_name += '.bz2'
+                elif oldpath.endswith('.bz2'):
-            updated[match_name] = oldpath
+                    bb.process.run(['bzip2', match_name], cwd=destdir)
-        else:
+                    match_name += '.bz2'
-            added[new_patch] = None
+                updated[match_name] = {'path' : oldpath}
+                if reldirpath != ".":
+                    updated[match_name]['patchdir'] = reldirpath
+            else:
+                added[new_patch] = {}
+                if reldirpath != ".":
+                    added[new_patch]['patchdir'] = reldirpath
    return (updated, added, existing_patches)
 def _create_kconfig_diff(srctree, rd, outfile):
    """Create a kconfig fragment"""
+    import bb.process
    # Only update config fragment if both config files exist
    orig_config = os.path.join(srctree, '.config.baseline')
    new_config = os.path.join(srctree, '.config.new')
@@ -1389,38 +1412,59 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
         1. updated - files that already exist in SRCURI
         2. added - new files files that don't exist in SRCURI
         3  removed - files that exist in SRCURI but not in exported files
-      In each dict the key is the 'basepath' of the URI and value is the
+       In each dict the key is the 'basepath' of the URI and value is:
-      absolute path to the existing file in recipe space (if any).
+         - for updated and added dicts, a dict with 1 optionnal key:
+           - 'path': the absolute path to the existing file in recipe space (if any)
+         - for removed dict, the absolute path to the existing file in recipe space
    """
    import oe.recipeutils
+    import bb.data
+    import bb.process
    # Find out local files (SRC_URI files that exist in the "recipe space").
    # Local files that reside in srctree are not included in patch generation.
    # Instead they are directly copied over the original source files (in
    # recipe space).
    existing_files = oe.recipeutils.get_recipe_local_files(rd)
    new_set = None
    updated = OrderedDict()
    added = OrderedDict()
    removed = OrderedDict()
-    local_files_dir = os.path.join(srctreebase, 'oe-local-files')
-    git_files = _git_ls_tree(srctree)
+    # Get current branch and return early with empty lists
-    if 'oe-local-files' in git_files:
+    # if on one of the override branches
-        # If tracked by Git, take the files from srctree HEAD. First get
+    # (local files are provided only for the main branch and processing
-        # the tree object of the directory
+    # them against lists from recipe overrides will result in mismatches
-        tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool')
+    # and broken modifications to recipes).
-        tree = git_files['oe-local-files'][2]
+    stdout, _ = bb.process.run('git rev-parse --abbrev-ref HEAD',
-        bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree,
+                               cwd=srctree)
-                        env=dict(os.environ, GIT_WORK_TREE=destdir,
+    branchname = stdout.rstrip()
-                                 GIT_INDEX_FILE=tmp_index))
+    if branchname.startswith(override_branch_prefix):
-        new_set = list(_git_ls_tree(srctree, tree, True).keys())
+        return (updated, added, removed)
-    elif os.path.isdir(local_files_dir):
-        # If not tracked by Git, just copy from working copy
+    files = _git_modified(srctree)
-        new_set = _ls_tree(local_files_dir)
+        #if not files:
-        bb.process.run(['cp', '-ax',
+        #    files = _ls_tree(srctree)
-                        os.path.join(local_files_dir, '.'), destdir])
+    for f in files:
-    else:
+        fullfile = os.path.join(srctree, f)
-        new_set = []
+        if os.path.exists(os.path.join(fullfile, ".git")):
+            # submodules handled elsewhere
+            continue
+        if f not in existing_files:
+            added[f] = {}
+            if os.path.isdir(os.path.join(srctree, f)):
+                shutil.copytree(fullfile, os.path.join(destdir, f))
+            else:
+                shutil.copy2(fullfile, os.path.join(destdir, f))
+        elif not os.path.exists(fullfile):
+            removed[f] = existing_files[f]
+        elif f in existing_files:
+            updated[f] = {'path' : existing_files[f]}
+            if os.path.isdir(os.path.join(srctree, f)):
+                shutil.copytree(fullfile, os.path.join(destdir, f))
+            else:
+                shutil.copy2(fullfile, os.path.join(destdir, f))
    # Special handling for kernel config
    if bb.data.inherits_class('kernel-yocto', rd):
@@ -1428,17 +1472,14 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
        fragment_path = os.path.join(destdir, fragment_fn)
        if _create_kconfig_diff(srctree, rd, fragment_path):
            if os.path.exists(fragment_path):
-                if fragment_fn not in new_set:
+                if fragment_fn in removed:
-                    new_set.append(fragment_fn)
+                    del removed[fragment_fn]
-                # Copy fragment to local-files
+                if fragment_fn not in updated and fragment_fn not in added:
-                if os.path.isdir(local_files_dir):
+                    added[fragment_fn] = {}
-                    shutil.copy2(fragment_path, local_files_dir)
            else:
-                if fragment_fn in new_set:
+                if fragment_fn in updated:
-                    new_set.remove(fragment_fn)
+                    removed[fragment_fn] = updated[fragment_fn]
-                # Remove fragment from local-files
+                    del updated[fragment_fn]
-                if os.path.exists(os.path.join(local_files_dir, fragment_fn)):
-                    os.unlink(os.path.join(local_files_dir, fragment_fn))
    # Special handling for cml1, ccmake, etc bbclasses that generated
    # configuration fragment files that are consumed as source files
@@ -1446,42 +1487,13 @@ def _export_local_files(srctree, rd, destdir, srctreebase):
        if bb.data.inherits_class(frag_class, rd):
            srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name)
            if os.path.exists(srcpath):
-                if frag_name not in new_set:
+                if frag_name in removed:
-                    new_set.append(frag_name)
+                    del removed[frag_name]
+                if frag_name not in updated:
+                    added[frag_name] = {}
                # copy fragment into destdir
                shutil.copy2(srcpath, destdir)
-                # copy fragment into local files if exists
-                if os.path.isdir(local_files_dir):
-                    shutil.copy2(srcpath, local_files_dir)
-    if new_set is not None:
-        for fname in new_set:
-            if fname in existing_files:
-                origpath = existing_files.pop(fname)
-                workpath = os.path.join(local_files_dir, fname)
-                if not filecmp.cmp(origpath, workpath):
-                    updated[fname] = origpath
-            elif fname != '.gitignore':
-                added[fname] = None
-        workdir = rd.getVar('WORKDIR')
-        s = rd.getVar('S')
-        if not s.endswith(os.sep):
-            s += os.sep
-        if workdir != s:
-            # Handle files where subdir= was specified
-            for fname in list(existing_files.keys()):
-                # FIXME handle both subdir starting with BP and not?
-                fworkpath = os.path.join(workdir, fname)
-                if fworkpath.startswith(s):
-                    fpath = os.path.join(srctree, os.path.relpath(fworkpath, s))
-                    if os.path.exists(fpath):
-                        origpath = existing_files.pop(fname)
-                        if not filecmp.cmp(origpath, fpath):
-                            updated[fpath] = origpath
-        removed = existing_files
    return (updated, added, removed)
@@ -1499,7 +1511,7 @@ def _determine_files_dir(rd):
 def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, dry_run_outdir=None):
    """Implement the 'srcrev' mode of update-recipe"""
-    import bb
+    import bb.process
    import oe.recipeutils
    dry_run_suffix = ' (dry-run)' if dry_run_outdir else ''
@@ -1508,6 +1520,12 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
    recipedir = os.path.basename(recipefile)
    logger.info('Updating SRCREV in recipe %s%s' % (recipedir, dry_run_suffix))
+    # Get original SRCREV
+    old_srcrev = rd.getVar('SRCREV') or ''
+    if old_srcrev == "INVALID":
+            raise DevtoolError('Update mode srcrev is only valid for recipe fetched from an SCM repository')
+    old_srcrev = {'.': old_srcrev}
    # Get HEAD revision
    try:
        stdout, _ = bb.process.run('git rev-parse HEAD', cwd=srctree)
@@ -1531,16 +1549,16 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
        local_files_dir = tempfile.mkdtemp(dir=tempdir)
        srctreebase = workspace[recipename]['srctreebase']
        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
+        removedentries = {}
        if not no_remove:
            # Find list of existing patches in recipe file
            patches_dir = tempfile.mkdtemp(dir=tempdir)
-            old_srcrev = rd.getVar('SRCREV') or ''
            upd_p, new_p, del_p = _export_patches(srctree, rd, old_srcrev,
                                                  patches_dir)
            logger.debug('Patches: update %s, new %s, delete %s' % (dict(upd_p), dict(new_p), dict(del_p)))
            # Remove deleted local files and "overlapping" patches
-            remove_files = list(del_f.values()) + list(upd_p.values()) + list(del_p.values())
+            remove_files = list(del_f.values()) + [value["path"] for value in upd_p.values() if "path" in value] + [value["path"] for value in del_p.values() if "path" in value]
            if remove_files:
                removedentries = _remove_file_entries(srcuri, remove_files)[0]
                update_srcuri = True
@@ -1554,14 +1572,14 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                patchfields['SRC_URI'] = '\\\n    '.join(srcuri)
            if dry_run_outdir:
                logger.info('Creating bbappend (dry-run)')
-            else:
+            appendfile, destpath = oe.recipeutils.bbappend_recipe(
-                appendfile, destpath = oe.recipeutils.bbappend_recipe(
+                    rd, appendlayerdir, files, wildcardver=wildcard_version,
-                        rd, appendlayerdir, files, wildcardver=wildcard_version,
+                    extralines=patchfields, removevalues=removevalues,
-                        extralines=patchfields, removevalues=removevalues,
+                    redirect_output=dry_run_outdir)
-                        redirect_output=dry_run_outdir)
        else:
            files_dir = _determine_files_dir(rd)
-            for basepath, path in upd_f.items():
+            for basepath, param in upd_f.items():
+                path = param['path']
                logger.info('Updating file %s%s' % (basepath, dry_run_suffix))
                if os.path.isabs(basepath):
                    # Original file (probably with subdir pointing inside source tree)
@@ -1571,7 +1589,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
                    _move_file(os.path.join(local_files_dir, basepath), path,
                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                update_srcuri= True
-            for basepath, path in new_f.items():
+            for basepath, param in new_f.items():
+                path = param['path']
                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                _move_file(os.path.join(local_files_dir, basepath),
                           os.path.join(files_dir, basepath),
@@ -1594,7 +1613,6 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi
 def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wildcard_version, no_remove, no_report_remove, initial_rev, dry_run_outdir=None, force_patch_refresh=False):
    """Implement the 'patch' mode of update-recipe"""
-    import bb
    import oe.recipeutils
    recipefile = rd.getVar('FILE')
@@ -1603,9 +1621,22 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
    if not os.path.exists(append):
        raise DevtoolError('unable to find workspace bbappend for recipe %s' %
                           recipename)
+    srctreebase = workspace[recipename]['srctreebase']
+    relpatchdir = os.path.relpath(srctreebase, srctree)
+    if relpatchdir == '.':
+        patchdir_params = {}
+    else:
+        patchdir_params = {'patchdir': relpatchdir}
+    def srcuri_entry(basepath, patchdir_params):
+        if patchdir_params:
+            paramstr = ';' + ';'.join('%s=%s' % (k,v) for k,v in patchdir_params.items())
+        else:
+            paramstr = ''
+        return 'file://%s%s' % (basepath, paramstr)
-    initial_rev, update_rev, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
+    initial_revs, update_revs, changed_revs, filter_patches = _get_patchset_revs(srctree, append, initial_rev, force_patch_refresh)
-    if not initial_rev:
+    if not initial_revs:
        raise DevtoolError('Unable to find initial revision - please specify '
                           'it with --initial-rev')
@@ -1619,61 +1650,69 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
    tempdir = tempfile.mkdtemp(prefix='devtool')
    try:
        local_files_dir = tempfile.mkdtemp(dir=tempdir)
-        if filter_patches:
+        upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
-            upd_f = {}
-            new_f = {}
-            del_f = {}
-        else:
-            srctreebase = workspace[recipename]['srctreebase']
-            upd_f, new_f, del_f = _export_local_files(srctree, rd, local_files_dir, srctreebase)
-        remove_files = []
-        if not no_remove:
-            # Get all patches from source tree and check if any should be removed
-            all_patches_dir = tempfile.mkdtemp(dir=tempdir)
-            _, _, del_p = _export_patches(srctree, rd, initial_rev,
-                                          all_patches_dir)
-            # Remove deleted local files and  patches
-            remove_files = list(del_f.values()) + list(del_p.values())
        # Get updated patches from source tree
        patches_dir = tempfile.mkdtemp(dir=tempdir)
-        upd_p, new_p, _ = _export_patches(srctree, rd, update_rev,
+        upd_p, new_p, _ = _export_patches(srctree, rd, update_revs,
                                          patches_dir, changed_revs)
+        # Get all patches from source tree and check if any should be removed
+        all_patches_dir = tempfile.mkdtemp(dir=tempdir)
+        _, _, del_p = _export_patches(srctree, rd, initial_revs,
+                                      all_patches_dir)
        logger.debug('Pre-filtering: update: %s, new: %s' % (dict(upd_p), dict(new_p)))
        if filter_patches:
            new_p = OrderedDict()
            upd_p = OrderedDict((k,v) for k,v in upd_p.items() if k in filter_patches)
-            remove_files = [f for f in remove_files if f in filter_patches]
+            del_p = OrderedDict((k,v) for k,v in del_p.items() if k in filter_patches)
+        remove_files = []
+        if not no_remove:
+            # Remove deleted local files and  patches
+            remove_files = list(del_f.values()) + list(del_p.values())
        updatefiles = False
        updaterecipe = False
        destpath = None
        srcuri = (rd.getVar('SRC_URI', False) or '').split()
        if appendlayerdir:
            files = OrderedDict((os.path.join(local_files_dir, key), val) for
                         key, val in list(upd_f.items()) + list(new_f.items()))
            files.update(OrderedDict((os.path.join(patches_dir, key), val) for
                              key, val in list(upd_p.items()) + list(new_p.items())))
+            params = []
+            for file, param in files.items():
+                patchdir_param = dict(patchdir_params)
+                patchdir = param.get('patchdir', ".")
+                if patchdir != "." :
+                    if patchdir_param:
+                       patchdir_param['patchdir'] += patchdir
+                    else:
+                        patchdir_param['patchdir'] = patchdir
+                params.append(patchdir_param)
            if files or remove_files:
                removevalues = None
                if remove_files:
                    removedentries, remaining = _remove_file_entries(
                                                    srcuri, remove_files)
                    if removedentries or remaining:
-                        remaining = ['file://' + os.path.basename(item) for
+                        remaining = [srcuri_entry(os.path.basename(item), patchdir_params) for
                                     item in remaining]
                        removevalues = {'SRC_URI': removedentries + remaining}
                appendfile, destpath = oe.recipeutils.bbappend_recipe(
                                rd, appendlayerdir, files,
                                wildcardver=wildcard_version,
                                removevalues=removevalues,
-                                redirect_output=dry_run_outdir)
+                                redirect_output=dry_run_outdir,
+                                params=params)
            else:
                logger.info('No patches or local source files needed updating')
        else:
            # Update existing files
            files_dir = _determine_files_dir(rd)
-            for basepath, path in upd_f.items():
+            for basepath, param in upd_f.items():
+                path = param['path']
                logger.info('Updating file %s' % basepath)
                if os.path.isabs(basepath):
                    # Original file (probably with subdir pointing inside source tree)
@@ -1684,14 +1723,23 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                    _move_file(os.path.join(local_files_dir, basepath), path,
                               dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                updatefiles = True
-            for basepath, path in upd_p.items():
+            for basepath, param in upd_p.items():
-                patchfn = os.path.join(patches_dir, basepath)
+                path = param['path']
+                patchdir = param.get('patchdir', ".")
+                patchdir_param = {}
+                if patchdir != "." :
+                    patchdir_param = dict(patchdir_params)
+                    if patchdir_param:
+                       patchdir_param['patchdir'] += patchdir
+                    else:
+                        patchdir_param['patchdir'] = patchdir
+                patchfn = os.path.join(patches_dir, patchdir, basepath)
                if os.path.dirname(path) + '/' == dl_dir:
                    # This is a a downloaded patch file - we now need to
                    # replace the entry in SRC_URI with our local version
                    logger.info('Replacing remote patch %s with updated local version' % basepath)
                    path = os.path.join(files_dir, basepath)
-                    _replace_srcuri_entry(srcuri, basepath, 'file://%s' % basepath)
+                    _replace_srcuri_entry(srcuri, basepath, srcuri_entry(basepath, patchdir_param))
                    updaterecipe = True
                else:
                    logger.info('Updating patch %s%s' % (basepath, dry_run_suffix))
@@ -1699,21 +1747,29 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
                           dry_run_outdir=dry_run_outdir, base_outdir=recipedir)
                updatefiles = True
            # Add any new files
-            for basepath, path in new_f.items():
+            for basepath, param in new_f.items():
                logger.info('Adding new file %s%s' % (basepath, dry_run_suffix))
                _move_file(os.path.join(local_files_dir, basepath),
                           os.path.join(files_dir, basepath),
                           dry_run_outdir=dry_run_outdir,
                           base_outdir=recipedir)
-                srcuri.append('file://%s' % basepath)
+                srcuri.append(srcuri_entry(basepath, patchdir_params))
                updaterecipe = True
-            for basepath, path in new_p.items():
+            for basepath, param in new_p.items():
+                patchdir = param.get('patchdir', ".")
                logger.info('Adding new patch %s%s' % (basepath, dry_run_suffix))
-                _move_file(os.path.join(patches_dir, basepath),
+                _move_file(os.path.join(patches_dir, patchdir, basepath),
                           os.path.join(files_dir, basepath),
                           dry_run_outdir=dry_run_outdir,
                           base_outdir=recipedir)
-                srcuri.append('file://%s' % basepath)
+                params = dict(patchdir_params)
+                if patchdir != "." :
+                    if params:
+                       params['patchdir'] += patchdir
+                    else:
+                        params['patchdir'] = patchdir
+                srcuri.append(srcuri_entry(basepath, params))
                updaterecipe = True
            # Update recipe, if needed
            if _remove_file_entries(srcuri, remove_files)[0]:
@@ -1736,6 +1792,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil
 def _guess_recipe_update_mode(srctree, rdata):
    """Guess the recipe update mode to use"""
+    import bb.process
    src_uri = (rdata.getVar('SRC_URI') or '').split()
    git_uris = [uri for uri in src_uri if uri.startswith('git://')]
    if not git_uris:
@@ -1757,6 +1814,8 @@ def _guess_recipe_update_mode(srctree, rdata):
    return 'patch'
 def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_version, no_remove, initial_rev, no_report_remove=False, dry_run_outdir=None, no_overrides=False, force_patch_refresh=False):
+    import bb.data
+    import bb.process
    srctree = workspace[recipename]['srctree']
    if mode == 'auto':
        mode = _guess_recipe_update_mode(srctree, rd)
@@ -1770,6 +1829,8 @@ def _update_recipe(recipename, workspace, rd, mode, appendlayerdir, wildcard_ver
        for line in stdout.splitlines():
            branchname = line[2:]
            if line.startswith('* '):
+                if 'HEAD' in line:
+                    raise DevtoolError('Detached HEAD - please check out a branch, e.g., "devtool"')
                startbranch = branchname
            if branchname.startswith(override_branch_prefix):
                override_branches.append(branchname)
@@ -1877,6 +1938,7 @@ def status(args, config, basepath, workspace):
 def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
    """Reset one or more recipes"""
+    import bb.process
    import oe.path
    def clean_preferred_provider(pn, layerconf_path):
@@ -1889,7 +1951,7 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
            lines = f.readlines()
            with open(new_layerconf_file, 'a') as nf:
                for line in lines:
-                    pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + pn + r'"$'
+                    pprovider_exp = r'^PREFERRED_PROVIDER_.*? = "' + re.escape(pn) + r'"$'
                    if not re.match(pprovider_exp, line):
                        nf.write(line)
                    else:
@@ -1959,9 +2021,19 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
                        shutil.rmtree(srctreebase)
                    else:
                        # We don't want to risk wiping out any work in progress
-                        logger.info('Leaving source tree %s as-is; if you no '
+                        if srctreebase.startswith(os.path.join(config.workspace_path, 'sources')):
-                                    'longer need it then please delete it manually'
+                            from datetime import datetime
-                                    % srctreebase)
+                            preservesrc = os.path.join(config.workspace_path, 'attic', 'sources', "{}.{}".format(pn, datetime.now().strftime("%Y%m%d%H%M%S")))
+                            logger.info('Preserving source tree in %s\nIf you no '
+                                        'longer need it then please delete it manually.\n'
+                                        'It is also possible to reuse it via devtool source tree argument.'
+                                        % preservesrc)
+                            bb.utils.mkdirhier(os.path.dirname(preservesrc))
+                            shutil.move(srctreebase, preservesrc)
+                        else:
+                            logger.info('Leaving source tree %s as-is; if you no '
+                                        'longer need it then please delete it manually'
+                                        % srctreebase)
            else:
                # This is unlikely, but if it's empty we can just remove it
                os.rmdir(srctreebase)
@@ -1970,8 +2042,6 @@ def _reset(recipes, no_clean, remove_work, config, basepath, workspace):
 def reset(args, config, basepath, workspace):
    """Entry point for the devtool 'reset' subcommand"""
-    import bb
-    import shutil
    recipes = ""
@@ -2221,6 +2291,7 @@ def register_commands(subparsers, context):
    group.add_argument('--no-same-dir', help='Force build in a separate build directory', action="store_true")
    parser_add.add_argument('--fetch', '-f', help='Fetch the specified URI and extract it to create the source tree (deprecated - pass as positional argument instead)', metavar='URI')
    parser_add.add_argument('--npm-dev', help='For npm, also fetch devDependencies', action="store_true")
+    parser_add.add_argument('--no-pypi', help='Do not inherit pypi class', action="store_true")
    parser_add.add_argument('--version', '-V', help='Version to use within recipe (PV)')
    parser_add.add_argument('--no-git', '-g', help='If fetching source, do not set up source tree as a git repository', action="store_true")
    group = parser_add.add_mutually_exclusive_group()
@@ -2249,6 +2320,7 @@ def register_commands(subparsers, context):
    parser_modify.add_argument('--branch', '-b', default="devtool", help='Name for development branch to checkout (when not using -n/--no-extract) (default "%(default)s")')
    parser_modify.add_argument('--no-overrides', '-O', action="store_true", help='Do not create branches for other override configurations')
    parser_modify.add_argument('--keep-temp', help='Keep temporary directory (for debugging)', action="store_true")
+    parser_modify.add_argument('--debug-build', action="store_true", help='Add DEBUG_BUILD = "1" to the modified recipe')
    parser_modify.set_defaults(func=modify, fixed_setup=context.fixed_setup)
    parser_extract = subparsers.add_parser('extract', help='Extract the source for an existing recipe',
diff --git a/scripts/lib/devtool/upgrade.py b/scripts/lib/devtool/upgrade.py
index 5a057e95f5..d9aca6e2db 100644
--- a/scripts/lib/devtool/upgrade.py
+++ b/scripts/lib/devtool/upgrade.py
@@ -32,9 +32,11 @@ def _run(cmd, cwd=''):
 def _get_srctree(tmpdir):
    srctree = tmpdir
-    dirs = scriptutils.filter_src_subdirs(tmpdir)
+    dirs = os.listdir(tmpdir)
    if len(dirs) == 1:
        srctree = os.path.join(tmpdir, dirs[0])
+    else:
+        raise DevtoolError("Cannot determine where the source tree is after unpacking in {}: {}".format(tmpdir,dirs))
    return srctree
 def _copy_source_code(orig, dest):
@@ -71,23 +73,24 @@ def _rename_recipe_dirs(oldpv, newpv, path):
                if oldfile.find(oldpv) != -1:
                    newfile = oldfile.replace(oldpv, newpv)
                    if oldfile != newfile:
-                        os.rename(os.path.join(path, oldfile), os.path.join(path, newfile))
+                        bb.utils.rename(os.path.join(path, oldfile),
+                              os.path.join(path, newfile))
-def _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path):
+def _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path):
    oldrecipe = os.path.basename(oldrecipe)
    if oldrecipe.endswith('_%s.bb' % oldpv):
-        newrecipe = '%s_%s.bb' % (bpn, newpv)
+        newrecipe = '%s_%s.bb' % (pn, newpv)
        if oldrecipe != newrecipe:
            shutil.move(os.path.join(path, oldrecipe), os.path.join(path, newrecipe))
    else:
        newrecipe = oldrecipe
    return os.path.join(path, newrecipe)
-def _rename_recipe_files(oldrecipe, bpn, oldpv, newpv, path):
+def _rename_recipe_files(oldrecipe, pn, oldpv, newpv, path):
    _rename_recipe_dirs(oldpv, newpv, path)
-    return _rename_recipe_file(oldrecipe, bpn, oldpv, newpv, path)
+    return _rename_recipe_file(oldrecipe, pn, oldpv, newpv, path)
-def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d):
+def _write_append(rc, srctreebase, srctree, same_dir, no_same_dir, revs, copied, workspace, d):
    """Writes an append file"""
    if not os.path.exists(rc):
        raise DevtoolError("bbappend not created because %s does not exist" % rc)
@@ -102,36 +105,38 @@ def _write_append(rc, srctree, same_dir, no_same_dir, rev, copied, workspace, d)
    pn = d.getVar('PN')
    af = os.path.join(appendpath, '%s.bbappend' % brf)
    with open(af, 'w') as f:
-        f.write('FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n\n')
+        f.write('FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:"\n\n')
+        # Local files can be modified/tracked in separate subdir under srctree
+        # Mostly useful for packages with S != WORKDIR
+        f.write('FILESPATH:prepend := "%s:"\n' %
+                os.path.join(srctreebase, 'oe-local-files'))
+        f.write('# srctreebase: %s\n' % srctreebase)
        f.write('inherit externalsrc\n')
        f.write(('# NOTE: We use pn- overrides here to avoid affecting'
                 'multiple variants in the case where the recipe uses BBCLASSEXTEND\n'))
-        f.write('EXTERNALSRC_pn-%s = "%s"\n' % (pn, srctree))
+        f.write('EXTERNALSRC:pn-%s = "%s"\n' % (pn, srctree))
        b_is_s = use_external_build(same_dir, no_same_dir, d)
        if b_is_s:
-            f.write('EXTERNALSRC_BUILD_pn-%s = "%s"\n' % (pn, srctree))
+            f.write('EXTERNALSRC_BUILD:pn-%s = "%s"\n' % (pn, srctree))
        f.write('\n')
-        if rev:
+        if revs:
-            f.write('# initial_rev: %s\n' % rev)
+            for name, rev in revs.items():
+                f.write('# initial_rev %s: %s\n' % (name, rev))
        if copied:
            f.write('# original_path: %s\n' % os.path.dirname(d.getVar('FILE')))
            f.write('# original_files: %s\n' % ' '.join(copied))
    return af
-def _cleanup_on_error(rf, srctree):
+def _cleanup_on_error(rd, srctree):
-    rfp = os.path.split(rf)[0] # recipe folder
+    if os.path.exists(rd):
-    rfpp = os.path.split(rfp)[0] # recipes folder
+        shutil.rmtree(rd)
-    if os.path.exists(rfp):
-        shutil.rmtree(rfp)
-    if not len(os.listdir(rfpp)):
-        os.rmdir(rfpp)
    srctree = os.path.abspath(srctree)
    if os.path.exists(srctree):
        shutil.rmtree(srctree)
-def _upgrade_error(e, rf, srctree, keep_failure=False, extramsg=None):
+def _upgrade_error(e, rd, srctree, keep_failure=False, extramsg=None):
-    if rf and not keep_failure:
+    if not keep_failure:
-        _cleanup_on_error(rf, srctree)
+        _cleanup_on_error(rd, srctree)
    logger.error(e)
    if extramsg:
        logger.error(extramsg)
@@ -164,6 +169,7 @@ def _get_uri(rd):
 def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, keep_temp, tinfoil, rd):
    """Extract sources of a recipe with a new version"""
+    import oe.patch
    def __run(cmd):
        """Simple wrapper which calls _run with srctree as cwd"""
@@ -178,12 +184,16 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
    uri, rev = _get_uri(crd)
    if srcrev:
        rev = srcrev
+    paths = [srctree]
    if uri.startswith('git://') or uri.startswith('gitsm://'):
        __run('git fetch')
        __run('git checkout %s' % rev)
-        __run('git tag -f devtool-base-new')
+        __run('git tag -f --no-sign devtool-base-new')
-        md5 = None
+        __run('git submodule update --recursive')
-        sha256 = None
+        __run('git submodule foreach \'git tag -f --no-sign devtool-base-new\'')
+        (stdout, _) = __run('git submodule --quiet foreach \'echo $sm_path\'')
+        paths += [os.path.join(srctree, p) for p in stdout.splitlines()]
+        checksums = {}
        _, _, _, _, _, params = bb.fetch2.decodeurl(uri)
        srcsubdir_rel = params.get('destsuffix', 'git')
        if not srcbranch:
@@ -191,14 +201,15 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
            get_branch = [x.strip() for x in check_branch.splitlines()]
            # Remove HEAD reference point and drop remote prefix
            get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
-            if 'master' in get_branch:
+            if len(get_branch) == 1:
-                # If it is master, we do not need to append 'branch=master' as this is default.
+                # If srcrev is on only ONE branch, then use that branch
-                # Even with the case where get_branch has multiple objects, if 'master' is one
-                # of them, we should default take from 'master'
-                srcbranch = ''
-            elif len(get_branch) == 1:
-                # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
                srcbranch = get_branch[0]
+            elif 'main' in get_branch:
+                # If srcrev is on multiple branches, then choose 'main' if it is one of them
+                srcbranch = 'main'
+            elif 'master' in get_branch:
+                # Otherwise choose 'master' if it is one of the branches
+                srcbranch = 'master'
            else:
                # If get_branch contains more than one objects, then display error and exit.
                mbrch = '\n  ' + '\n  '.join(get_branch)
@@ -215,9 +226,6 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
        if ftmpdir and keep_temp:
            logger.info('Fetch temp directory is %s' % ftmpdir)
-        md5 = checksums['md5sum']
-        sha256 = checksums['sha256sum']
        tmpsrctree = _get_srctree(tmpdir)
        srctree = os.path.abspath(srctree)
        srcsubdir_rel = os.path.relpath(tmpsrctree, tmpdir)
@@ -249,32 +257,52 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
        useroptions = []
        oe.patch.GitApplyTree.gitCommandUserOptions(useroptions, d=rd)
        __run('git %s commit -q -m "Commit of upstream changes at version %s" --allow-empty' % (' '.join(useroptions), newpv))
-        __run('git tag -f devtool-base-%s' % newpv)
+        __run('git tag -f --no-sign devtool-base-%s' % newpv)
-    (stdout, _) = __run('git rev-parse HEAD')
+    revs = {}
-    rev = stdout.rstrip()
+    for path in paths:
+        (stdout, _) = _run('git rev-parse HEAD', cwd=path)
+        revs[os.path.relpath(path, srctree)] = stdout.rstrip()
    if no_patch:
        patches = oe.recipeutils.get_recipe_patches(crd)
        if patches:
            logger.warning('By user choice, the following patches will NOT be applied to the new source tree:\n  %s' % '\n  '.join([os.path.basename(patch) for patch in patches]))
    else:
-        __run('git checkout devtool-patched -b %s' % branch)
+        for path in paths:
-        skiptag = False
+            _run('git checkout devtool-patched -b %s' % branch, cwd=path)
-        try:
+            (stdout, _) = _run('git branch --list devtool-override-*', cwd=path)
-            __run('git rebase %s' % rev)
+            branches_to_rebase = [branch] + stdout.split()
-        except bb.process.ExecutionError as e:
+            target_branch = revs[os.path.relpath(path, srctree)]
-            skiptag = True
-            if 'conflict' in e.stdout:
+            # There is a bug (or feature?) in git rebase where if a commit with
-                logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
+            # a note is fully rebased away by being part of an old commit, the
-            else:
+            # note is still attached to the old commit. Avoid this by making
-                logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+            # sure all old devtool related commits have a note attached to them
-        if not skiptag:
+            # (this assumes git config notes.rewriteMode is set to ignore).
-            if uri.startswith('git://') or uri.startswith('gitsm://'):
+            (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
-                suffix = 'new'
+            for rev in stdout.splitlines():
-            else:
+                if not oe.patch.GitApplyTree.getNotes(path, rev):
-                suffix = newpv
+                    oe.patch.GitApplyTree.addNote(path, rev, "dummy")
-            __run('git tag -f devtool-patched-%s' % suffix)
+            for b in branches_to_rebase:
+                logger.info("Rebasing {} onto {}".format(b, target_branch))
+                _run('git checkout %s' % b, cwd=path)
+                try:
+                    _run('git rebase %s' % target_branch, cwd=path)
+                except bb.process.ExecutionError as e:
+                    if 'conflict' in e.stdout:
+                        logger.warning('Command \'%s\' failed:\n%s\n\nYou will need to resolve conflicts in order to complete the upgrade.' % (e.command, e.stdout.rstrip()))
+                        _run('git rebase --abort', cwd=path)
+                    else:
+                        logger.warning('Command \'%s\' failed:\n%s' % (e.command, e.stdout))
+            # Remove any dummy notes added above.
+            (stdout, _) = __run('git rev-list devtool-base..%s' % target_branch)
+            for rev in stdout.splitlines():
+                oe.patch.GitApplyTree.removeNote(path, rev, "dummy")
+            _run('git checkout %s' % branch, cwd=path)
    if tmpsrctree:
        if keep_temp:
@@ -284,7 +312,7 @@ def _extract_new_source(newpv, srctree, no_patch, srcrev, srcbranch, branch, kee
            if tmpdir != tmpsrctree:
                shutil.rmtree(tmpdir)
-    return (rev, md5, sha256, srcbranch, srcsubdir_rel)
+    return (revs, checksums, srcbranch, srcsubdir_rel)
 def _add_license_diff_to_recipe(path, diff):
    notice_text = """# FIXME: the LIC_FILES_CHKSUM values have been updated by 'devtool upgrade'.
@@ -305,22 +333,22 @@ def _add_license_diff_to_recipe(path, diff):
        f.write("\n#\n\n".encode())
        f.write(orig_content)
-def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
+def _create_new_recipe(newpv, checksums, srcrev, srcbranch, srcsubdir_old, srcsubdir_new, workspace, tinfoil, rd, license_diff, new_licenses, srctree, keep_failure):
    """Creates the new recipe under workspace"""
-    bpn = rd.getVar('BPN')
+    pn = rd.getVar('PN')
-    path = os.path.join(workspace, 'recipes', bpn)
+    path = os.path.join(workspace, 'recipes', pn)
    bb.utils.mkdirhier(path)
    copied, _ = oe.recipeutils.copy_recipe_files(rd, path, all_variants=True)
    if not copied:
-        raise DevtoolError('Internal error - no files were copied for recipe %s' % bpn)
+        raise DevtoolError('Internal error - no files were copied for recipe %s' % pn)
    logger.debug('Copied %s to %s' % (copied, path))
    oldpv = rd.getVar('PV')
    if not newpv:
        newpv = oldpv
    origpath = rd.getVar('FILE')
-    fullpath = _rename_recipe_files(origpath, bpn, oldpv, newpv, path)
+    fullpath = _rename_recipe_files(origpath, pn, oldpv, newpv, path)
    logger.debug('Upgraded %s => %s' % (origpath, fullpath))
    newvalues = {}
@@ -336,7 +364,10 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
        replacing = True
        new_src_uri = []
        for entry in src_uri:
-            scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
+            try:
+                scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(entry)
+            except bb.fetch2.MalformedUrl as e:
+                raise DevtoolError("Could not decode SRC_URI: {}".format(e))
            if replacing and scheme in ['git', 'gitsm']:
                branch = params.get('branch', 'master')
                if rd.expand(branch) != srcbranch:
@@ -374,30 +405,39 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
                addnames.append(params['name'])
    # Find what's been set in the original recipe
    oldnames = []
+    oldsums = []
    noname = False
    for varflag in rd.getVarFlags('SRC_URI'):
-        if varflag.endswith(('.md5sum', '.sha256sum')):
+        for checksum in checksums:
-            name = varflag.rsplit('.', 1)[0]
+            if varflag.endswith('.' + checksum):
-            if name not in oldnames:
+                name = varflag.rsplit('.', 1)[0]
-                oldnames.append(name)
+                if name not in oldnames:
-        elif varflag in ['md5sum', 'sha256sum']:
+                    oldnames.append(name)
-            noname = True
+                oldsums.append(checksum)
+            elif varflag == checksum:
+                noname = True
+                oldsums.append(checksum)
    # Even if SRC_URI has named entries it doesn't have to actually use the name
    if noname and addnames and addnames[0] not in oldnames:
        addnames = []
    # Drop any old names (the name actually might include ${PV})
    for name in oldnames:
        if name not in newnames:
-            newvalues['SRC_URI[%s.md5sum]' % name] = None
+            for checksum in oldsums:
-            newvalues['SRC_URI[%s.sha256sum]' % name] = None
+                newvalues['SRC_URI[%s.%s]' % (name, checksum)] = None
-    if sha256:
+    nameprefix = '%s.' % addnames[0] if addnames else ''
-        if addnames:
-            nameprefix = '%s.' % addnames[0]
+    # md5sum is deprecated, remove any traces of it. If it was the only old
-        else:
+    # checksum, then replace it with the default checksums.
-            nameprefix = ''
+    if 'md5sum' in oldsums:
        newvalues['SRC_URI[%smd5sum]' % nameprefix] = None
-        newvalues['SRC_URI[%ssha256sum]' % nameprefix] = sha256
+        oldsums.remove('md5sum')
+        if not oldsums:
+            oldsums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
+    for checksum in oldsums:
+        newvalues['SRC_URI[%s%s]' % (nameprefix, checksum)] = checksums[checksum]
    if srcsubdir_new != srcsubdir_old:
        s_subdir_old = os.path.relpath(os.path.abspath(rd.getVar('S')), rd.getVar('WORKDIR'))
@@ -422,10 +462,11 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
        newvalues["LIC_FILES_CHKSUM"] = newlicchksum
        _add_license_diff_to_recipe(fullpath, license_diff)
+    tinfoil.modified_files()
    try:
        rd = tinfoil.parse_recipe_file(fullpath, False)
    except bb.tinfoil.TinfoilCommandFailed as e:
-        _upgrade_error(e, fullpath, srctree, keep_failure, 'Parsing of upgraded recipe failed')
+        _upgrade_error(e, os.path.dirname(fullpath), srctree, keep_failure, 'Parsing of upgraded recipe failed')
    oe.recipeutils.patch_recipe(rd, fullpath, newvalues)
    return fullpath, copied
@@ -434,7 +475,7 @@ def _create_new_recipe(newpv, md5, sha256, srcrev, srcbranch, srcsubdir_old, src
 def _check_git_config():
    def getconfig(name):
        try:
-            value = bb.process.run('git config --global %s' % name)[0].strip()
+            value = bb.process.run('git config %s' % name)[0].strip()
        except bb.process.ExecutionError as e:
            if e.exitcode == 1:
                value = None
@@ -494,6 +535,15 @@ def _generate_license_diff(old_licenses, new_licenses):
            diff = diff + line
    return diff
+def _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil):
+    tasks = []
+    for task in (rd.getVar('RECIPE_UPGRADE_EXTRA_TASKS') or '').split():
+        logger.info('Running extra recipe upgrade task: %s' % task)
+        res = tinfoil.build_targets(pn, task, handle_events=True)
+        if not res:
+            raise DevtoolError('Running extra recipe upgrade task %s for %s failed' % (task, pn))
 def upgrade(args, config, basepath, workspace):
    """Entry point for the devtool 'upgrade' subcommand"""
@@ -521,6 +571,8 @@ def upgrade(args, config, basepath, workspace):
        else:
            srctree = standard.get_default_srctree(config, pn)
+        srctree_s = standard.get_real_srctree(srctree, rd.getVar('S'), rd.getVar('UNPACKDIR'))
        # try to automatically discover latest version and revision if not provided on command line
        if not args.version and not args.srcrev:
            version_info = oe.recipeutils.get_recipe_upstream_version(rd)
@@ -550,30 +602,34 @@ def upgrade(args, config, basepath, workspace):
        try:
            logger.info('Extracting current version source...')
            rev1, srcsubdir1 = standard._extract_source(srctree, False, 'devtool-orig', False, config, basepath, workspace, args.fixed_setup, rd, tinfoil, no_overrides=args.no_overrides)
-            old_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or ""))
+            old_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
            logger.info('Extracting upgraded version source...')
-            rev2, md5, sha256, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
+            rev2, checksums, srcbranch, srcsubdir2 = _extract_new_source(args.version, srctree, args.no_patch,
                                                    args.srcrev, args.srcbranch, args.branch, args.keep_temp,
                                                    tinfoil, rd)
-            new_licenses = _extract_licenses(srctree, (rd.getVar('LIC_FILES_CHKSUM') or ""))
+            new_licenses = _extract_licenses(srctree_s, (rd.getVar('LIC_FILES_CHKSUM') or ""))
            license_diff = _generate_license_diff(old_licenses, new_licenses)
-            rf, copied = _create_new_recipe(args.version, md5, sha256, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
+            rf, copied = _create_new_recipe(args.version, checksums, args.srcrev, srcbranch, srcsubdir1, srcsubdir2, config.workspace_path, tinfoil, rd, license_diff, new_licenses, srctree, args.keep_failure)
-        except bb.process.CmdError as e:
+        except (bb.process.CmdError, DevtoolError) as e:
-            _upgrade_error(e, rf, srctree, args.keep_failure)
+            recipedir = os.path.join(config.workspace_path, 'recipes', rd.getVar('PN'))
-        except DevtoolError as e:
+            _upgrade_error(e, recipedir, srctree, args.keep_failure)
-            _upgrade_error(e, rf, srctree, args.keep_failure)
        standard._add_md5(config, pn, os.path.dirname(rf))
-        af = _write_append(rf, srctree, args.same_dir, args.no_same_dir, rev2,
+        af = _write_append(rf, srctree, srctree_s, args.same_dir, args.no_same_dir, rev2,
                        copied, config.workspace_path, rd)
        standard._add_md5(config, pn, af)
+        _run_recipe_upgrade_extra_tasks(pn, rd, tinfoil)
        update_unlockedsigs(basepath, workspace, args.fixed_setup, [pn])
        logger.info('Upgraded source extracted to %s' % srctree)
        logger.info('New recipe is %s' % rf)
        if license_diff:
            logger.info('License checksums have been updated in the new recipe; please refer to it for the difference between the old and the new license texts.')
+        preferred_version = rd.getVar('PREFERRED_VERSION_%s' % rd.getVar('PN'))
+        if preferred_version:
+            logger.warning('Version is pinned to %s via PREFERRED_VERSION; it may need adjustment to match the new version before any further steps are taken' % preferred_version)
    finally:
        tinfoil.shutdown()
    return 0
@@ -599,18 +655,28 @@ def latest_version(args, config, basepath, workspace):
    return 0
 def check_upgrade_status(args, config, basepath, workspace):
+    def _print_status(recipe):
+        print("{:25} {:15} {:15} {} {} {}".format(   recipe['pn'],
+                                                               recipe['cur_ver'],
+                                                               recipe['status'] if recipe['status'] != 'UPDATE' else (recipe['next_ver'] if not recipe['next_ver'].endswith("new-commits-available") else "new commits"),
+                                                               recipe['maintainer'],
+                                                               recipe['revision'] if recipe['revision'] != 'N/A' else "",
+                                                               "cannot be updated due to: %s" %(recipe['no_upgrade_reason']) if recipe['no_upgrade_reason'] else ""))
    if not args.recipe:
        logger.info("Checking the upstream status for all recipes may take a few minutes")
    results = oe.recipeutils.get_recipe_upgrade_status(args.recipe)
-    for result in results:
+    for recipegroup in results:
-        # pn, update_status, current, latest, maintainer, latest_commit, no_update_reason
+        upgrades = [r for r in recipegroup if r['status'] != 'MATCH']
-        if args.all or result[1] != 'MATCH':
+        currents = [r for r in recipegroup if r['status'] == 'MATCH']
-            logger.info("{:25} {:15} {:15} {} {} {}".format(   result[0],
+        if len(upgrades) > 1:
-                                                               result[2],
+            print("These recipes need to be upgraded together {")
-                                                               result[1] if result[1] != 'UPDATE' else (result[3] if not result[3].endswith("new-commits-available") else "new commits"),
+        for r in sorted(upgrades, key=lambda r:r['pn']):
-                                                               result[4],
+            _print_status(r)
-                                                               result[5] if result[5] != 'N/A' else "",
+        if len(upgrades) > 1:
-                                                               "cannot be updated due to: %s" %(result[6]) if result[6] else ""))
+            print("}")
+        for r in currents:
+            if args.all:
+                _print_status(r)
 def register_commands(subparsers, context):
    """Register devtool subcommands from this plugin"""
diff --git a/scripts/lib/devtool/utilcmds.py b/scripts/lib/devtool/utilcmds.py
index 964817766b..bf39f71b11 100644
--- a/scripts/lib/devtool/utilcmds.py
+++ b/scripts/lib/devtool/utilcmds.py
@@ -64,7 +64,7 @@ def configure_help(args, config, basepath, workspace):
        b = rd.getVar('B')
        s = rd.getVar('S')
        configurescript = os.path.join(s, 'configure')
-        confdisabled = 'noexec' in rd.getVarFlags('do_configure') or 'do_configure' not in (rd.getVar('__BBTASKS', False) or [])
+        confdisabled = 'noexec' in rd.getVarFlags('do_configure') or 'do_configure' not in (bb.build.listtasks(rd))
        configureopts = oe.utils.squashspaces(rd.getVar('CONFIGUREOPTS') or '')
        extra_oeconf = oe.utils.squashspaces(rd.getVar('EXTRA_OECONF') or '')
        extra_oecmake = oe.utils.squashspaces(rd.getVar('EXTRA_OECMAKE') or '')
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index e9d52bb67b..041d79f162 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -18,6 +18,7 @@ import shutil
 import scriptutils
 import errno
 from collections import defaultdict
+import difflib
 logger = logging.getLogger('recipetool')
@@ -49,7 +50,7 @@ def find_target_file(targetpath, d, pkglist=None):
                      '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
                      '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
                      '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
-                      '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
+                      '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',}
    for pthspec, message in invalidtargets.items():
        if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
@@ -72,15 +73,15 @@ def find_target_file(targetpath, d, pkglist=None):
                # This does assume that PN comes before other values, but that's a fairly safe assumption
                for line in f:
                    if line.startswith('PN:'):
-                        pn = line.split(':', 1)[1].strip()
+                        pn = line.split(': ', 1)[1].strip()
-                    elif line.startswith('FILES_INFO:'):
+                    elif line.startswith('FILES_INFO'):
-                        val = line.split(':', 1)[1].strip()
+                        val = line.split(': ', 1)[1].strip()
                        dictval = json.loads(val)
                        for fullpth in dictval.keys():
                            if fnmatch.fnmatchcase(fullpth, targetpath):
                                recipes[targetpath].append(pn)
-                    elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
+                    elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'):
-                        scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape')
+                        scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape')
                        if 'update-alternatives --install %s ' % targetpath in scriptval:
                            recipes[targetpath].append('?%s' % pn)
                        elif targetpath_re.search(scriptval):
@@ -100,7 +101,7 @@ def determine_file_source(targetpath, rd):
    import oe.recipeutils
    # See if it's in do_install for the recipe
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
    src_uri = rd.getVar('SRC_URI')
    srcfile = ''
    modpatches = []
@@ -112,9 +113,9 @@ def determine_file_source(targetpath, rd):
        if not srcpath.startswith('/'):
            # Handle non-absolute path
            srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
-        if srcpath.startswith(workdir):
+        if srcpath.startswith(unpackdir):
            # OK, now we have the source file name, look for it in SRC_URI
-            workdirfile = os.path.relpath(srcpath, workdir)
+            workdirfile = os.path.relpath(srcpath, unpackdir)
            # FIXME this is where we ought to have some code in the fetcher, because this is naive
            for item in src_uri.split():
                localpath = bb.fetch2.localpath(item, rd)
@@ -299,7 +300,10 @@ def appendfile(args):
                if st.st_mode & stat.S_IXUSR:
                    perms = '0755'
            install = {args.newfile: (args.targetpath, perms)}
-        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        if sourcepath:
+            sourcepath = os.path.basename(sourcepath)
+        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        tinfoil.modified_files()
        return 0
    else:
        if alternative_pns:
@@ -313,7 +317,7 @@ def appendsrc(args, files, rd, extralines=None):
    import oe.recipeutils
    srcdir = rd.getVar('S')
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
    import bb.fetch
    simplified = {}
@@ -327,35 +331,57 @@ def appendsrc(args, files, rd, extralines=None):
    copyfiles = {}
    extralines = extralines or []
+    params = []
    for newfile, srcfile in files.items():
        src_destdir = os.path.dirname(srcfile)
        if not args.use_workdir:
            if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
-                srcdir = os.path.join(workdir, 'git')
+                srcdir = os.path.join(unpackdir, rd.getVar('BB_GIT_DEFAULT_DESTSUFFIX'))
                if not bb.data.inherits_class('kernel-yocto', rd):
-                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
+                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${UNPACKDIR}/${BB_GIT_DEFAULT_DESTSUFFIX}')
-            src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
+            src_destdir = os.path.join(os.path.relpath(srcdir, unpackdir), src_destdir)
        src_destdir = os.path.normpath(src_destdir)
-        source_uri = 'file://{0}'.format(os.path.basename(srcfile))
        if src_destdir and src_destdir != '.':
-            source_uri += ';subdir={0}'.format(src_destdir)
+            params.append({'subdir': src_destdir})
-        simple = bb.fetch.URI(source_uri)
-        simple.params = {}
-        simple_str = str(simple)
-        if simple_str in simplified:
-            existing = simplified[simple_str]
-            if source_uri != existing:
-                logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
-            else:
-                logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri))
        else:
-            extralines.append('SRC_URI += {0}'.format(source_uri))
+            params.append({})
-        copyfiles[newfile] = srcfile
+        copyfiles[newfile] = {'newname' : os.path.basename(srcfile)}
-    oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
+    dry_run_output = None
+    dry_run_outdir = None
+    if args.dry_run:
+        import tempfile
+        dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+        dry_run_outdir = dry_run_output.name
+    appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params,
+                                                   redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe)
+    if not appendfile:
+        return
+    if args.dry_run:
+        output = ''
+        appendfilename = os.path.basename(appendfile)
+        newappendfile = appendfile
+        if appendfile and os.path.exists(appendfile):
+            with open(appendfile, 'r') as f:
+                oldlines = f.readlines()
+        else:
+            appendfile = '/dev/null'
+            oldlines = []
+        with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f:
+            newlines = f.readlines()
+        diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile)
+        difflines = list(diff)
+        if difflines:
+            output += ''.join(difflines)
+        if output:
+            logger.info('Diff of changed files:\n%s' % output)
+        else:
+            logger.info('No changed files')
+    tinfoil.modified_files()
 def appendsrcfiles(parser, args):
    recipedata = _parse_recipe(args.recipe, tinfoil)
@@ -435,6 +461,8 @@ def register_commands(subparsers):
                                   help='Create/update a bbappend to add or replace source files',
                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
    parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
    parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
    parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
@@ -442,6 +470,8 @@ def register_commands(subparsers):
                                   parents=[common_src],
                                   help='Create/update a bbappend to add or replace a source file',
                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
    parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
    parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
    parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 566c75369a..edb6467103 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -18,6 +18,7 @@ from urllib.parse import urlparse, urldefrag, urlsplit
 import hashlib
 import bb.fetch2
 logger = logging.getLogger('recipetool')
+from oe.license_finder import find_licenses
 tinfoil = None
 plugins = None
@@ -115,8 +116,8 @@ class RecipeHandler(object):
                for line in f:
                    if line.startswith('PN:'):
                        pn = line.split(':', 1)[-1].strip()
-                    elif line.startswith('FILES_INFO:'):
+                    elif line.startswith('FILES_INFO:%s:' % pkg):
-                        val = line.split(':', 1)[1].strip()
+                        val = line.split(': ', 1)[1].strip()
                        dictval = json.loads(val)
                        for fullpth in sorted(dictval):
                            if fullpth.startswith(includedir) and fullpth.endswith('.h'):
@@ -366,7 +367,7 @@ def supports_srcrev(uri):
 def reformat_git_uri(uri):
    '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
    checkuri = uri.split(';', 1)[0]
-    if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
+    if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri):
        # Appends scheme if the scheme is missing
        if not '://' in uri:
            uri = 'git://' + uri
@@ -423,6 +424,36 @@ def create_recipe(args):
    storeTagName = ''
    pv_srcpv = False
+    handled = []
+    classes = []
+    # Find all plugins that want to register handlers
+    logger.debug('Loading recipe handlers')
+    raw_handlers = []
+    for plugin in plugins:
+        if hasattr(plugin, 'register_recipe_handlers'):
+            plugin.register_recipe_handlers(raw_handlers)
+    # Sort handlers by priority
+    handlers = []
+    for i, handler in enumerate(raw_handlers):
+        if isinstance(handler, tuple):
+            handlers.append((handler[0], handler[1], i))
+        else:
+            handlers.append((handler, 0, i))
+    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
+    for handler, priority, _ in handlers:
+        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+        setattr(handler, '_devtool', args.devtool)
+    handlers = [item[0] for item in handlers]
+    fetchuri = None
+    for handler in handlers:
+        if hasattr(handler, 'process_url'):
+            ret = handler.process_url(args, classes, handled, extravalues)
+            if 'url' in handled and ret:
+                fetchuri = ret
+                break
    if os.path.isfile(source):
        source = 'file://%s' % os.path.abspath(source)
@@ -431,11 +462,12 @@ def create_recipe(args):
        if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source):
            logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).')
        # Fetch a URL
-        fetchuri = reformat_git_uri(urldefrag(source)[0])
+        if not fetchuri:
+            fetchuri = reformat_git_uri(urldefrag(source)[0])
        if args.binary:
            # Assume the archive contains the directory structure verbatim
            # so we need to extract to a subdirectory
-            fetchuri += ';subdir=${BP}'
+            fetchuri += ';subdir=${BPN}'
        srcuri = fetchuri
        rev_re = re.compile(';rev=([^;]+)')
        res = rev_re.search(srcuri)
@@ -478,6 +510,9 @@ def create_recipe(args):
            storeTagName = params['tag']
            params['nobranch'] = '1'
            del params['tag']
+        # Assume 'master' branch if not set
+        if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params:
+            params['branch'] = 'master'
        fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
        tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -494,7 +529,7 @@ def create_recipe(args):
        if ftmpdir and args.keep_temp:
            logger.info('Fetch temp directory is %s' % ftmpdir)
-        dirlist = scriptutils.filter_src_subdirs(srctree)
+        dirlist = os.listdir(srctree)
        logger.debug('Directory listing (excluding filtered out):\n  %s' % '\n  '.join(dirlist))
        if len(dirlist) == 1:
            singleitem = os.path.join(srctree, dirlist[0])
@@ -527,10 +562,9 @@ def create_recipe(args):
            # Remove HEAD reference point and drop remote prefix
            get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
            if 'master' in get_branch:
-                # If it is master, we do not need to append 'branch=master' as this is default.
                # Even with the case where get_branch has multiple objects, if 'master' is one
                # of them, we should default take from 'master'
-                srcbranch = ''
+                srcbranch = 'master'
            elif len(get_branch) == 1:
                # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
                srcbranch = get_branch[0]
@@ -543,8 +577,8 @@ def create_recipe(args):
        # Since we might have a value in srcbranch, we need to
        # recontruct the srcuri to include 'branch' in params.
        scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri)
-        if srcbranch:
+        if scheme in ['git', 'gitsm']:
-            params['branch'] = srcbranch
+            params['branch'] = srcbranch or 'master'
        if storeTagName and scheme in ['git', 'gitsm']:
            # Check srcrev using tag and check validity of the tag
@@ -603,8 +637,7 @@ def create_recipe(args):
                    splitline = line.split()
                    if len(splitline) > 1:
                        if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
-                            srcuri = reformat_git_uri(splitline[1])
+                            srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
-                            srcsubdir = 'git'
                            break
    if args.src_subdir:
@@ -636,8 +669,6 @@ def create_recipe(args):
    # We'll come back and replace this later in handle_license_vars()
    lines_before.append('##LICENSE_PLACEHOLDER##')
-    handled = []
-    classes = []
    # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
    pn = None
@@ -675,8 +706,10 @@ def create_recipe(args):
    if not srcuri:
        lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
    lines_before.append('SRC_URI = "%s"' % srcuri)
+    shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
    for key, value in sorted(checksums.items()):
-        lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
+        if key in shown_checksums:
+            lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
    if srcuri and supports_srcrev(srcuri):
        lines_before.append('')
        lines_before.append('# Modify these as desired')
@@ -688,7 +721,7 @@ def create_recipe(args):
            srcpvprefix = 'svnr'
        else:
            srcpvprefix = scheme
-        lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix))
+        lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix))
        pv_srcpv = True
        if not args.autorev and srcrev == '${AUTOREV}':
            if os.path.exists(os.path.join(srctree, '.git')):
@@ -702,7 +735,7 @@ def create_recipe(args):
    if srcsubdir and not args.binary:
        # (for binary packages we explicitly specify subdir= when fetching to
        # match the default value of S, so we don't need to set it in that case)
-        lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
+        lines_before.append('S = "${UNPACKDIR}/%s"' % srcsubdir)
        lines_before.append('')
    if pkgarch:
@@ -710,31 +743,12 @@ def create_recipe(args):
        lines_after.append('')
    if args.binary:
-        lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
+        lines_after.append('INSANE_SKIP:${PN} += "already-stripped"')
        lines_after.append('')
    if args.npm_dev:
        extravalues['NPM_INSTALL_DEV'] = 1
-    # Find all plugins that want to register handlers
-    logger.debug('Loading recipe handlers')
-    raw_handlers = []
-    for plugin in plugins:
-        if hasattr(plugin, 'register_recipe_handlers'):
-            plugin.register_recipe_handlers(raw_handlers)
-    # Sort handlers by priority
-    handlers = []
-    for i, handler in enumerate(raw_handlers):
-        if isinstance(handler, tuple):
-            handlers.append((handler[0], handler[1], i))
-        else:
-            handlers.append((handler, 0, i))
-    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
-    for handler, priority, _ in handlers:
-        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
-        setattr(handler, '_devtool', args.devtool)
-    handlers = [item[0] for item in handlers]
    # Apply the handlers
    if args.binary:
        classes.append('bin_package')
@@ -743,6 +757,10 @@ def create_recipe(args):
    for handler in handlers:
        handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
+    # native and nativesdk classes are special and must be inherited last
+    # If present, put them at the end of the classes list
+    classes.sort(key=lambda c: c in ("native", "nativesdk"))
    extrafiles = extravalues.pop('extrafiles', {})
    extra_pn = extravalues.pop('PN', None)
    extra_pv = extravalues.pop('PV', None)
@@ -821,7 +839,7 @@ def create_recipe(args):
                line = line.replace(realpv, '${PV}')
            if pn:
                line = line.replace(pn, '${BPN}')
-            if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
+            if line == 'S = "${UNPACKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
                skipblank = True
                continue
        elif line.startswith('SRC_URI = '):
@@ -867,8 +885,10 @@ def create_recipe(args):
        outlines.append('')
    outlines.extend(lines_after)
+    outlines = [ line.rstrip('\n') +"\n" for line in outlines]
    if extravalues:
-        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True)
    if args.extract_to:
        scriptutils.git_convert_standalone_clone(srctree)
@@ -884,7 +904,7 @@ def create_recipe(args):
        log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
    if outfile == '-':
-        sys.stdout.write('\n'.join(outlines) + '\n')
+        sys.stdout.write(''.join(outlines) + '\n')
    else:
        with open(outfile, 'w') as f:
            lastline = None
@@ -892,9 +912,10 @@ def create_recipe(args):
                if not lastline and not line:
                    # Skip extra blank lines
                    continue
-                f.write('%s\n' % line)
+                f.write('%s' % line)
                lastline = line
        log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
+        tinfoil.modified_files()
    if tempsrc:
        if args.keep_temp:
@@ -917,23 +938,42 @@ def split_value(value):
    else:
        return value
+def fixup_license(value):
+    # Ensure licenses with OR starts and ends with brackets
+    if '|' in value:
+        return '(' + value + ')'
+    return value
+def tidy_licenses(value):
+    """Flat, split and sort licenses"""
+    from oe.license import flattened_licenses
+    def _choose(a, b):
+        str_a, str_b  = sorted((" & ".join(a), " & ".join(b)), key=str.casefold)
+        return ["(%s | %s)" % (str_a, str_b)]
+    if not isinstance(value, str):
+        value = " & ".join(value)
+    return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold)
 def handle_license_vars(srctree, lines_before, handled, extravalues, d):
    lichandled = [x for x in handled if x[0] == 'license']
    if lichandled:
        # Someone else has already handled the license vars, just return their value
        return lichandled[0][1]
-    licvalues = guess_license(srctree, d)
+    licvalues = find_licenses(srctree, d)
    licenses = []
    lic_files_chksum = []
    lic_unknown = []
    lines = []
    if licvalues:
        for licvalue in licvalues:
-            if not licvalue[0] in licenses:
+            license = licvalue[0]
-                licenses.append(licvalue[0])
+            lics = tidy_licenses(fixup_license(license))
+            lics = [lic for lic in lics if lic not in licenses]
+            if len(lics):
+                licenses.extend(lics)
            lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
-            if licvalue[0] == 'Unknown':
+            if license == 'Unknown':
                lic_unknown.append(licvalue[1])
        if lic_unknown:
            lines.append('#')
@@ -942,9 +982,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
            for licfile in lic_unknown:
                lines.append('#   %s' % licfile)
-    extra_license = split_value(extravalues.pop('LICENSE', []))
+    extra_license = tidy_licenses(extravalues.pop('LICENSE', ''))
-    if '&' in extra_license:
-        extra_license.remove('&')
    if extra_license:
        if licenses == ['Unknown']:
            licenses = extra_license
@@ -985,7 +1023,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
        lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
        lines.append('# to determine which situation is applicable.')
-    lines.append('LICENSE = "%s"' % ' & '.join(licenses))
+    lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold)))
    lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
    lines.append('')
@@ -1002,166 +1040,15 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
    handled.append(('license', licvalues))
    return licvalues
-def get_license_md5sums(d, static_only=False):
-    import bb.utils
-    md5sums = {}
-    if not static_only:
-        # Gather md5sums of license files in common license dir
-        commonlicdir = d.getVar('COMMON_LICENSE_DIR')
-        for fn in os.listdir(commonlicdir):
-            md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
-            md5sums[md5value] = fn
-    # The following were extracted from common values in various recipes
-    # (double checking the license against the license file itself, not just
-    # the LICENSE value in the recipe)
-    md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2'
-    md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2'
-    md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2'
-    md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2'
-    md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2'
-    md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2'
-    md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2'
-    md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2'
-    md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2'
-    md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2'
-    md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2'
-    md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2'
-    md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2'
-    md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2'
-    md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file
-    md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1'
-    md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1'
-    md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1'
-    md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1'
-    md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1'
-    md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1'
-    md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1'
-    md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1'
-    md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2'
-    md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2'
-    md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
-    md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
-    md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
-    md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2'
-    md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2'
-    md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
-    md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
-    md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
-    md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
-    md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
-    md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
-    md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1'
-    return md5sums
-def crunch_license(licfile):
-    '''
-    Remove non-material text from a license file and then check
-    its md5sum against a known list. This works well for licenses
-    which contain a copyright statement, but is also a useful way
-    to handle people's insistence upon reformatting the license text
-    slightly (with no material difference to the text of the
-    license).
-    '''
-    import oe.utils
-    # Note: these are carefully constructed!
-    license_title_re = re.compile(r'^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
-    license_statement_re = re.compile(r'^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
-    copyright_re = re.compile('^(#+)? *Copyright .*$')
-    crunched_md5sums = {}
-    # The following two were gleaned from the "forever" npm package
-    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
-    crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
-    # https://github.com/vasi/pixz/blob/master/LICENSE
-    crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
-    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
-    crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
-    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
-    crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
-    # https://github.com/datto/dattobd/blob/master/COPYING
-    # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
-    crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
-    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
-    # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
-    crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
-    # https://github.com/gkos/nrf24/blob/master/COPYING
-    crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
-    # https://github.com/josch09/resetusb/blob/master/COPYING
-    crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
-    crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
-    # unixODBC-2.3.4 COPYING
-    crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
-    crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10
-    crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/edl-v10
-    crunched_md5sums['0a9c78c0a398d1bbce4a166757d60387'] = 'EDL-1.0'
-    lictext = []
-    with open(licfile, 'r', errors='surrogateescape') as f:
-        for line in f:
-            # Drop opening statements
-            if copyright_re.match(line):
-                continue
-            elif license_title_re.match(line):
-                continue
-            elif license_statement_re.match(line):
-                continue
-            # Squash spaces, and replace smart quotes, double quotes
-            # and backticks with single quotes
-            line = oe.utils.squashspaces(line.strip())
-            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
-            if line:
-                lictext.append(line)
-    m = hashlib.md5()
-    try:
-        m.update(' '.join(lictext).encode('utf-8'))
-        md5val = m.hexdigest()
-    except UnicodeEncodeError:
-        md5val = None
-        lictext = ''
-    license = crunched_md5sums.get(md5val, None)
-    return license, md5val, lictext
-def guess_license(srctree, d):
-    import bb
-    md5sums = get_license_md5sums(d)
-    licenses = []
-    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
-    licfiles = []
-    for root, dirs, files in os.walk(srctree):
-        for fn in files:
-            for spec in licspecs:
-                if fnmatch.fnmatch(fn, spec):
-                    fullpath = os.path.join(root, fn)
-                    if not fullpath in licfiles:
-                        licfiles.append(fullpath)
-    for licfile in licfiles:
-        md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, None)
-        if not license:
-            license, crunched_md5, lictext = crunch_license(licfile)
-            if not license:
-                license = 'Unknown'
-        licenses.append((license, os.path.relpath(licfile, srctree), md5value))
-    # FIXME should we grab at least one source file with a license header and add that too?
-    return licenses
 def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
    """
-    Given a list of (license, path, md5sum) as returned by guess_license(),
+    Given a list of (license, path, md5sum) as returned by match_licenses(),
    a dict of package name to path mappings, write out a set of
    package-specific LICENSE values.
    """
    pkglicenses = {pn: []}
    for license, licpath, _ in licvalues:
+        license = fixup_license(license)
        for pkgname, pkgpath in packages.items():
            if licpath.startswith(pkgpath + '/'):
                if pkgname in pkglicenses:
@@ -1174,13 +1061,24 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
            pkglicenses[pn].append(license)
    outlicenses = {}
    for pkgname in packages:
-        license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+        # Assume AND operator between license files
-        if license == 'Unknown' and pkgname in fallback_licenses:
+        license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+        if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses:
            license = fallback_licenses[pkgname]
-        outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
+        licenses = tidy_licenses(license)
-        outlicenses[pkgname] = license.split()
+        license = ' & '.join(licenses)
+        outlines.append('LICENSE:%s = "%s"' % (pkgname, license))
+        outlicenses[pkgname] = licenses
    return outlicenses
+def generate_common_licenses_chksums(common_licenses, d):
+    lic_files_chksums = []
+    for license in tidy_licenses(common_licenses):
+        licfile = '${COMMON_LICENSE_DIR}/' + license
+        md5value = bb.utils.md5_file(d.expand(licfile))
+        lic_files_chksums.append('file://%s;md5=%s' % (licfile, md5value))
+    return lic_files_chksums
 def read_pkgconfig_provides(d):
    pkgdatadir = d.getVar('PKGDATA_DIR')
    pkgmap = {}
@@ -1311,7 +1209,7 @@ def register_commands(subparsers):
    parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)')
    parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
    parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies')
+    parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class')
    parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
    parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
    parser_create.set_defaults(func=create_recipe)
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 35a97c9345..ec9d510e23 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -5,9 +5,9 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
+import os
 import re
 import logging
-import glob
 from recipetool.create import RecipeHandler, validate_pv
 logger = logging.getLogger('recipetool')
@@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler):
        deps = []
        unmappedpkgs = []
-        proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE)
+        proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE)
-        pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
+        pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
-        pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
+        pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
-        findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
+        findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
-        findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
+        findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
-        checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
+        checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
-        include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
+        dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
        def find_cmake_package(pkg):
            RecipeHandler.load_devel_filemap(tinfoil.config_data)
@@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler):
                'makeinfo': 'texinfo',
                }
-        pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+        pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
-        lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+        lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
-        libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+        libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
-        progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
+        dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-        ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+        ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
-        am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+        am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
-        define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
+        define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
-        version_re = re.compile('([0-9.]+)')
+        version_re = re.compile(r'([0-9.]+)')
        defines = {}
        def subst_defines(value):
@@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                deps.append('zlib')
            elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'):
                deps.append('openssl')
-            elif keyword == 'AX_LIB_CURL':
+            elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'):
                deps.append('curl')
            elif keyword == 'AX_LIB_BEECRYPT':
                deps.append('beecrypt')
@@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                    'AX_CHECK_OPENSSL',
                    'AX_LIB_CRYPTO',
                    'AX_LIB_CURL',
+                    'LIBCURL_CHECK_CONFIG',
                    'AX_LIB_BEECRYPT',
                    'AX_LIB_EXPAT',
                    'AX_LIB_GCRYPT',
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index adfa377956..a807dafae5 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -8,9 +8,9 @@
 import ast
 import codecs
 import collections
-import distutils.command.build_py
+import setuptools.command.build_py
 import email
-import imp
+import importlib
 import glob
 import itertools
 import logging
@@ -18,7 +18,11 @@ import os
 import re
 import sys
 import subprocess
+import json
+import urllib.request
 from recipetool.create import RecipeHandler
+from urllib.parse import urldefrag
+from recipetool.create import determine_from_url
 logger = logging.getLogger('recipetool')
@@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler):
    assume_provided = ['builtins', 'os.path']
    # Assumes that the host python3 builtin_module_names is sane for target too
    assume_provided = assume_provided + list(sys.builtin_module_names)
+    excluded_fields = []
+    classifier_license_map = {
+        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
+        'License :: OSI Approved :: Apache Software License': 'Apache',
+        'License :: OSI Approved :: Apple Public Source License': 'APSL',
+        'License :: OSI Approved :: Artistic License': 'Artistic',
+        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
+        'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
+        'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
+        'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
+        'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
+        'License :: OSI Approved :: Common Public License': 'CPL',
+        'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
+        'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
+        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
+        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
+        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
+        'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
+        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
+        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
+        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
+        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
+        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
+        'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
+        'License :: OSI Approved :: IBM Public License': 'IPL',
+        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
+        'License :: OSI Approved :: Intel Open Source License': 'Intel',
+        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
+        'License :: OSI Approved :: MIT License': 'MIT',
+        'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
+        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
+        'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
+        'License :: OSI Approved :: Motosoto License': 'Motosoto',
+        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
+        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
+        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
+        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
+        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
+        'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
+        'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
+        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
+        'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
+        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
+        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
+        'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
+        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
+        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
+        'License :: OSI Approved :: Sun Public License': 'SPL',
+        'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
+        'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
+        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
+        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
+        'License :: OSI Approved :: W3C License': 'W3C',
+        'License :: OSI Approved :: X.Net License': 'Xnet',
+        'License :: OSI Approved :: Zope Public License': 'ZPL',
+        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
+        'License :: Other/Proprietary License': 'Proprietary',
+        'License :: Public Domain': 'PD',
+    }
+    def __init__(self):
+        pass
+    def process_url(self, args, classes, handled, extravalues):
+        """
+        Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/...
+        which corresponds to the archive location, and add pypi class
+        """
+        if 'url' in handled:
+            return None
+        fetch_uri = None
+        source = args.source
+        required_version = args.version if args.version else None
+        match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0])
+        if match:
+            package = match.group(1)
+            version = match.group(2) if match.group(2) else required_version
+            json_url = f"https://pypi.org/pypi/%s/json" % package
+            response = urllib.request.urlopen(json_url)
+            if response.status == 200:
+                data = json.loads(response.read())
+                if not version:
+                    # grab latest version
+                    version = data["info"]["version"]
+                pypi_package = data["info"]["name"]
+                for release in reversed(data["releases"][version]):
+                    if release["packagetype"] == "sdist":
+                        fetch_uri = release["url"]
+                        break
+            else:
+                logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url)
+                return None
+        else:
+            match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source)
+            if match:
+                fetch_uri = source
+                pypi_package = match.group(1)
+                _, version = determine_from_url(fetch_uri)
+        if match and not args.no_pypi:
+            if required_version and version != required_version:
+                raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version))
+            # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass)
+            # but at this point we cannot know because because user can specify the output name of the recipe on the command line
+            extravalues["PYPI_PACKAGE"] = pypi_package
+            # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe
+            pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri)
+            if pypi_package_ext:
+                pypi_package_ext = pypi_package_ext.group(1)
+                if pypi_package_ext != "tar.gz":
+                    extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext
+            # Pypi class will handle S and SRC_URI variables, so remove them
+            # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to:
+            # extravalues['SRC_URI(?:\[.*?\])?'] = None
+            extravalues['S'] = None
+            extravalues['SRC_URI'] = None
+            classes.append('pypi')
+        handled.append('url')
+        return fetch_uri
+    def handle_classifier_license(self, classifiers, existing_licenses=""):
+        licenses = []
+        for classifier in classifiers:
+            if classifier in self.classifier_license_map:
+                license = self.classifier_license_map[classifier]
+                if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+                    license = 'Apache-2.0'
+                elif license == 'GPL':
+                    if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+                        license = 'GPL-2.0'
+                    elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+                        license = 'GPL-3.0'
+                elif license == 'LGPL':
+                    if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+                        license = 'LGPL-2.1'
+                    elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+                        license = 'LGPL-2.0'
+                    elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+                        license = 'LGPL-3.0'
+                licenses.append(license)
+        if licenses:
+            return ' & '.join(licenses)
+        return None
+    def map_info_to_bbvar(self, info, extravalues):
+        # Map PKG-INFO & setup.py fields to bitbake variables
+        for field, values in info.items():
+            if field in self.excluded_fields:
+                continue
+            if field not in self.bbvar_map:
+                continue
+            if isinstance(values, str):
+                value = values
+            else:
+                value = ' '.join(str(v) for v in values if v)
+            bbvar = self.bbvar_map[field]
+            if bbvar == "PN":
+                # by convention python recipes start with "python3-"
+                if not value.startswith('python'):
+                    value = 'python3-' + value
+            if bbvar not in extravalues and value:
+                extravalues[bbvar] = value
+    def apply_info_replacements(self, info):
+        if not self.replacements:
+            return
+        for variable, search, replace in self.replacements:
+            if variable not in info:
+                continue
+            def replace_value(search, replace, value):
+                if replace is None:
+                    if re.search(search, value):
+                        return None
+                else:
+                    new_value = re.sub(search, replace, value)
+                    if value != new_value:
+                        return new_value
+                return value
+            value = info[variable]
+            if isinstance(value, str):
+                new_value = replace_value(search, replace, value)
+                if new_value is None:
+                    del info[variable]
+                elif new_value != value:
+                    info[variable] = new_value
+            elif hasattr(value, 'items'):
+                for dkey, dvalue in list(value.items()):
+                    new_list = []
+                    for pos, a_value in enumerate(dvalue):
+                        new_value = replace_value(search, replace, a_value)
+                        if new_value is not None and new_value != value:
+                            new_list.append(new_value)
+                    if value != new_list:
+                        value[dkey] = new_list
+            else:
+                new_list = []
+                for pos, a_value in enumerate(value):
+                    new_value = replace_value(search, replace, a_value)
+                    if new_value is not None and new_value != value:
+                        new_list.append(new_value)
+                if value != new_list:
+                    info[variable] = new_list
+    def scan_python_dependencies(self, paths):
+        deps = set()
+        try:
+            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            for line in dep_output.splitlines():
+                line = line.rstrip()
+                dep, filename = line.split('\t', 1)
+                if filename.endswith('/setup.py'):
+                    continue
+                deps.add(dep)
+        try:
+            provides_output = self.run_command(['pythondeps', '-p'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            provides_lines = (l.rstrip() for l in provides_output.splitlines())
+            provides = set(l for l in provides_lines if l and l != 'setup')
+            deps -= provides
+        return deps
+    def parse_pkgdata_for_python_packages(self):
+        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+        ldata = tinfoil.config_data.createCopy()
+        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
+        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+        python_dirs = [python_sitedir + os.sep,
+                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                       os.path.dirname(python_sitedir) + os.sep]
+        packages = {}
+        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+            files_info = None
+            with open(pkgdatafile, 'r') as f:
+                for line in f.readlines():
+                    field, value = line.split(': ', 1)
+                    if field.startswith('FILES_INFO'):
+                        files_info = ast.literal_eval(value)
+                        break
+                else:
+                    continue
+            for fn in files_info:
+                for suffix in importlib.machinery.all_suffixes():
+                    if fn.endswith(suffix):
+                        break
+                else:
+                    continue
+                if fn.startswith(dynload_dir + os.sep):
+                    if '/.debug/' in fn:
+                        continue
+                    base = os.path.basename(fn)
+                    provided = base.split('.', 1)[0]
+                    packages[provided] = os.path.basename(pkgdatafile)
+                    continue
+                for python_dir in python_dirs:
+                    if fn.startswith(python_dir):
+                        relpath = fn[len(python_dir):]
+                        relstart, _, relremaining = relpath.partition(os.sep)
+                        if relstart.endswith('.egg'):
+                            relpath = relremaining
+                        base, _ = os.path.splitext(relpath)
+                        if '/.debug/' in base:
+                            continue
+                        if os.path.basename(base) == '__init__':
+                            base = os.path.dirname(base)
+                        base = base.replace(os.sep + os.sep, os.sep)
+                        provided = base.replace(os.sep, '.')
+                        packages[provided] = os.path.basename(pkgdatafile)
+        return packages
+    @classmethod
+    def run_command(cls, cmd, **popenargs):
+        if 'stderr' not in popenargs:
+            popenargs['stderr'] = subprocess.STDOUT
+        try:
+            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+        except OSError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+            raise
+        except subprocess.CalledProcessError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+            raise
+class PythonSetupPyRecipeHandler(PythonRecipeHandler):
    bbvar_map = {
        'Name': 'PN',
        'Version': 'PV',
@@ -45,9 +376,9 @@ class PythonRecipeHandler(RecipeHandler):
        'Summary': 'SUMMARY',
        'Description': 'DESCRIPTION',
        'License': 'LICENSE',
-        'Requires': 'RDEPENDS_${PN}',
+        'Requires': 'RDEPENDS:${PN}',
-        'Provides': 'RPROVIDES_${PN}',
+        'Provides': 'RPROVIDES:${PN}',
-        'Obsoletes': 'RREPLACES_${PN}',
+        'Obsoletes': 'RREPLACES:${PN}',
    }
    # PN/PV are already set by recipetool core & desc can be extremely long
    excluded_fields = [
@@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler):
        'Supported-Platform',
    ]
    setuparg_multi_line_values = ['Description']
    replacements = [
        ('License', r' +$', ''),
        ('License', r'^ +', ''),
@@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler):
        ('Install-requires', r'\[[^\]]+\]$', ''),
    ]
-    classifier_license_map = {
-        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
-        'License :: OSI Approved :: Apache Software License': 'Apache',
-        'License :: OSI Approved :: Apple Public Source License': 'APSL',
-        'License :: OSI Approved :: Artistic License': 'Artistic',
-        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
-        'License :: OSI Approved :: BSD License': 'BSD',
-        'License :: OSI Approved :: Common Public License': 'CPL',
-        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
-        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
-        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
-        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
-        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
-        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
-        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
-        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
-        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
-        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
-        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
-        'License :: OSI Approved :: IBM Public License': 'IPL',
-        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
-        'License :: OSI Approved :: Intel Open Source License': 'Intel',
-        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
-        'License :: OSI Approved :: MIT License': 'MIT',
-        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
-        'License :: OSI Approved :: Motosoto License': 'Motosoto',
-        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
-        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
-        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
-        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
-        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
-        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
-        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
-        'License :: OSI Approved :: Python Software Foundation License': 'PSF',
-        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
-        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
-        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
-        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '--  Sun Industry Standards Source License (SISSL)',
-        'License :: OSI Approved :: Sun Public License': 'SPL',
-        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
-        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
-        'License :: OSI Approved :: W3C License': 'W3C',
-        'License :: OSI Approved :: X.Net License': 'Xnet',
-        'License :: OSI Approved :: Zope Public License': 'ZPL',
-        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
-    }
    def __init__(self):
        pass
+    def parse_setup_py(self, setupscript='./setup.py'):
+        with codecs.open(setupscript) as f:
+            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+        def _map(key):
+            key = key.replace('_', '-')
+            key = key[0].upper() + key[1:]
+            if key in self.setup_parse_map:
+                key = self.setup_parse_map[key]
+            return key
+        # Naive mapping of setup() arguments to PKG-INFO field names
+        for d in [info, non_literals]:
+            for key, value in list(d.items()):
+                if key is None:
+                    continue
+                new_key = _map(key)
+                if new_key != key:
+                    del d[key]
+                    d[new_key] = value
+        return info, 'setuptools' in imported_modules, non_literals, extensions
+    def get_setup_args_info(self, setupscript='./setup.py'):
+        cmd = ['python3', setupscript]
+        info = {}
+        keys = set(self.bbvar_map.keys())
+        keys |= set(self.setuparg_list_fields)
+        keys |= set(self.setuparg_multi_line_values)
+        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
+        for index, keys in grouped_keys:
+            if index == (True, False):
+                # Splitlines output for each arg as a list value
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
+            elif index == (False, True):
+                # Entire output for each arg
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = arg_info
+            else:
+                info.update(self.get_setup_byline(list(keys), setupscript))
+        return info
+    def get_setup_byline(self, fields, setupscript='./setup.py'):
+        info = {}
+        cmd = ['python3', setupscript]
+        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
+        try:
+            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            if len(fields) != len(info_lines):
+                logger.error('Mismatch between setup.py output lines and number of fields')
+                sys.exit(1)
+            for lineno, line in enumerate(info_lines):
+                line = line.rstrip()
+                info[fields[lineno]] = line
+        return info
+    def get_pkginfo(self, pkginfo_fn):
+        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+        msginfo = {}
+        for field in msg.keys():
+            values = msg.get_all(field)
+            if len(values) == 1:
+                msginfo[field] = values[0]
+            else:
+                msginfo[field] = values
+        return msginfo
+    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
+        if 'Package-dir' in setup_info:
+            package_dir = setup_info['Package-dir']
+        else:
+            package_dir = {}
+        dist = setuptools.Distribution()
+        class PackageDir(setuptools.command.build_py.build_py):
+            def __init__(self, package_dir):
+                self.package_dir = package_dir
+                self.dist = dist
+                super().__init__(self.dist)
+        pd = PackageDir(package_dir)
+        to_scan = []
+        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
+            if 'Py-modules' in setup_info:
+                for module in setup_info['Py-modules']:
+                    try:
+                        package, module = module.rsplit('.', 1)
+                    except ValueError:
+                        package, module = '.', module
+                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
+                    to_scan.append(module_path)
+            if 'Packages' in setup_info:
+                for package in setup_info['Packages']:
+                    to_scan.append(pd.get_package_dir(package))
+            if 'Scripts' in setup_info:
+                to_scan.extend(setup_info['Scripts'])
+        else:
+            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+        if not to_scan:
+            to_scan = ['.']
+        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
+        provided_packages = self.parse_pkgdata_for_python_packages()
+        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
+        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
+        for dep in scanned_deps:
+            mapped = provided_packages.get(dep)
+            if mapped:
+                logger.debug('Mapped %s to %s' % (dep, mapped))
+                mapped_deps.add(mapped)
+            else:
+                logger.debug('Could not map %s' % dep)
+                unmapped_deps.add(dep)
+        return mapped_deps, unmapped_deps
    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        if 'buildsystem' in handled:
            return False
+        logger.debug("Trying setup.py parser")
        # Check for non-zero size setup.py files
        setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
        for fn in setupfiles:
            if os.path.getsize(fn):
                break
        else:
+            logger.debug("No setup.py found")
            return False
        # setup.py is always parsed to get at certain required information, such as
@@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler):
                            continue
                        if line.startswith('['):
+                            # PACKAGECONFIG must not contain expressions or whitespace
+                            line = line.replace(" ", "")
+                            line = line.replace(':', "")
+                            line = line.replace('.', "-dot-")
+                            line = line.replace('"', "")
+                            line = line.replace('<', "-smaller-")
+                            line = line.replace('>', "-bigger-")
+                            line = line.replace('_', "-")
+                            line = line.replace('(', "")
+                            line = line.replace(')', "")
+                            line = line.replace('!', "-not-")
+                            line = line.replace('=', "-equals-")
                            current_feature = line[1:-1]
                        elif current_feature:
                            extras_req[current_feature].append(line)
@@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler):
        if license_str:
            for i, line in enumerate(lines_before):
-                if line.startswith('LICENSE = '):
+                if line.startswith('##LICENSE_PLACEHOLDER##'):
                    lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
                    break
        if 'Classifier' in info:
-            existing_licenses = info.get('License', '')
+            license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
-            licenses = []
+            if license:
-            for classifier in info['Classifier']:
+                info['License'] = license
-                if classifier in self.classifier_license_map:
-                    license = self.classifier_license_map[classifier]
-                    if license == 'Apache' and 'Apache-2.0' in existing_licenses:
-                        license = 'Apache-2.0'
-                    elif license == 'GPL':
-                        if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
-                            license = 'GPL-2.0'
-                        elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
-                            license = 'GPL-3.0'
-                    elif license == 'LGPL':
-                        if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
-                            license = 'LGPL-2.1'
-                        elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
-                            license = 'LGPL-2.0'
-                        elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
-                            license = 'LGPL-3.0'
-                    licenses.append(license)
-            if licenses:
-                info['License'] = ' & '.join(licenses)
-        # Map PKG-INFO & setup.py fields to bitbake variables
+        self.map_info_to_bbvar(info, extravalues)
-        for field, values in info.items():
-            if field in self.excluded_fields:
-                continue
-            if field not in self.bbvar_map:
-                continue
-            if isinstance(values, str):
-                value = values
-            else:
-                value = ' '.join(str(v) for v in values if v)
-            bbvar = self.bbvar_map[field]
-            if bbvar not in extravalues and value:
-                extravalues[bbvar] = value
        mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
@@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler):
                lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
                lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
                lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
+                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
                lines_after.append('#')
                lines_after.append('# Uncomment this line to enable all the optional features.')
                lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
@@ -301,7 +701,7 @@ class PythonRecipeHandler(RecipeHandler):
                inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
                lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
                lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
-                lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
+                lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
        if mapped_deps:
            name = info.get('Name')
@@ -313,7 +713,7 @@ class PythonRecipeHandler(RecipeHandler):
                lines_after.append('')
            lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
            lines_after.append('# python sources, and might not be 100% accurate.')
-            lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
+            lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
        unmapped_deps -= set(extensions)
        unmapped_deps -= set(self.assume_provided)
@@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler):
        handled.append('buildsystem')
-    def get_pkginfo(self, pkginfo_fn):
+class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
-        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+    """Base class to support PEP517 and PEP518
-        msginfo = {}
-        for field in msg.keys():
+    PEP517 https://peps.python.org/pep-0517/#source-trees
-            values = msg.get_all(field)
+    PEP518 https://peps.python.org/pep-0518/#build-system-table
-            if len(values) == 1:
+    """
-                msginfo[field] = values[0]
+    # bitbake currently supports the 4 following backends
-            else:
+    build_backend_map = {
-                msginfo[field] = values
+        "setuptools.build_meta": "python_setuptools_build_meta",
-        return msginfo
+        "poetry.core.masonry.api": "python_poetry_core",
+        "flit_core.buildapi": "python_flit_core",
+        "hatchling.build": "python_hatchling",
+        "maturin": "python_maturin",
+        "mesonpy": "python_mesonpy",
+    }
-    def parse_setup_py(self, setupscript='./setup.py'):
+    # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml
-        with codecs.open(setupscript) as f:
+    # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
-            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+    # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/
+    # keys from "project" and "tool.poetry" sections are almost the same except for the  HOMEPAGE which is "homepage" for tool.poetry
+    # and "Homepage" for "project" section. So keep both
+    bbvar_map = {
+        "name": "PN",
+        "version": "PV",
+        "Homepage": "HOMEPAGE",
+        "homepage": "HOMEPAGE",
+        "description": "SUMMARY",
+        "license": "LICENSE",
+        "dependencies": "RDEPENDS:${PN}",
+        "requires": "DEPENDS",
+    }
-        def _map(key):
+    replacements = [
-            key = key.replace('_', '-')
+        ("license", r" +$", ""),
-            key = key[0].upper() + key[1:]
+        ("license", r"^ +", ""),
-            if key in self.setup_parse_map:
+        ("license", r" ", "-"),
-                key = self.setup_parse_map[key]
+        ("license", r"^GNU-", ""),
-            return key
+        ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
+        ("license", r"^UNKNOWN$", ""),
+        # Remove currently unhandled version numbers from these variables
+        ("requires", r"\[[^\]]+\]$", ""),
+        ("requires", r"^([^><= ]+).*", r"\1"),
+        ("dependencies", r"\[[^\]]+\]$", ""),
+        ("dependencies", r"^([^><= ]+).*", r"\1"),
+    ]
-        # Naive mapping of setup() arguments to PKG-INFO field names
+    excluded_native_pkgdeps = [
-        for d in [info, non_literals]:
+        # already provided by python_setuptools_build_meta.bbclass
-            for key, value in list(d.items()):
+        "python3-setuptools-native",
-                if key is None:
+        "python3-wheel-native",
-                    continue
+        # already provided by python_poetry_core.bbclass
-                new_key = _map(key)
+        "python3-poetry-core-native",
-                if new_key != key:
+        # already provided by python_flit_core.bbclass
-                    del d[key]
+        "python3-flit-core-native",
-                    d[new_key] = value
+        # already provided by python_mesonpy
+        "python3-meson-python-native",
+    ]
-        return info, 'setuptools' in imported_modules, non_literals, extensions
+    # add here a list of known and often used packages and the corresponding bitbake package
+    known_deps_map = {
+        "setuptools": "python3-setuptools",
+        "wheel": "python3-wheel",
+        "poetry-core": "python3-poetry-core",
+        "flit_core": "python3-flit-core",
+        "setuptools-scm": "python3-setuptools-scm",
+        "hatchling": "python3-hatchling",
+        "hatch-vcs": "python3-hatch-vcs",
+        "meson-python" : "python3-meson-python",
+    }
-    def get_setup_args_info(self, setupscript='./setup.py'):
+    def __init__(self):
-        cmd = ['python3', setupscript]
+        pass
-        info = {}
-        keys = set(self.bbvar_map.keys())
-        keys |= set(self.setuparg_list_fields)
-        keys |= set(self.setuparg_multi_line_values)
-        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
-        for index, keys in grouped_keys:
-            if index == (True, False):
-                # Splitlines output for each arg as a list value
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
-            elif index == (False, True):
-                # Entire output for each arg
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = arg_info
-            else:
-                info.update(self.get_setup_byline(list(keys), setupscript))
-        return info
-    def get_setup_byline(self, fields, setupscript='./setup.py'):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        info = {}
+        metadata = {}
-        cmd = ['python3', setupscript]
+        if 'buildsystem' in handled:
-        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
+            return False
-        try:
-            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            if len(fields) != len(info_lines):
-                logger.error('Mismatch between setup.py output lines and number of fields')
-                sys.exit(1)
-            for lineno, line in enumerate(info_lines):
-                line = line.rstrip()
-                info[fields[lineno]] = line
-        return info
-    def apply_info_replacements(self, info):
-        for variable, search, replace in self.replacements:
-            if variable not in info:
-                continue
-            def replace_value(search, replace, value):
-                if replace is None:
-                    if re.search(search, value):
-                        return None
-                else:
-                    new_value = re.sub(search, replace, value)
-                    if value != new_value:
-                        return new_value
-                return value
-            value = info[variable]
-            if isinstance(value, str):
-                new_value = replace_value(search, replace, value)
-                if new_value is None:
-                    del info[variable]
-                elif new_value != value:
-                    info[variable] = new_value
-            elif hasattr(value, 'items'):
-                for dkey, dvalue in list(value.items()):
-                    new_list = []
-                    for pos, a_value in enumerate(dvalue):
-                        new_value = replace_value(search, replace, a_value)
-                        if new_value is not None and new_value != value:
-                            new_list.append(new_value)
-                    if value != new_list:
-                        value[dkey] = new_list
-            else:
-                new_list = []
-                for pos, a_value in enumerate(value):
-                    new_value = replace_value(search, replace, a_value)
-                    if new_value is not None and new_value != value:
-                        new_list.append(new_value)
-                if value != new_list:
-                    info[variable] = new_list
-    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
-        if 'Package-dir' in setup_info:
-            package_dir = setup_info['Package-dir']
-        else:
-            package_dir = {}
-        class PackageDir(distutils.command.build_py.build_py):
-            def __init__(self, package_dir):
-                self.package_dir = package_dir
-        pd = PackageDir(package_dir)
-        to_scan = []
-        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
-            if 'Py-modules' in setup_info:
-                for module in setup_info['Py-modules']:
-                    try:
-                        package, module = module.rsplit('.', 1)
-                    except ValueError:
-                        package, module = '.', module
-                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
-                    to_scan.append(module_path)
-            if 'Packages' in setup_info:
+        logger.debug("Trying pyproject.toml parser")
-                for package in setup_info['Packages']:
-                    to_scan.append(pd.get_package_dir(package))
-            if 'Scripts' in setup_info:
+        # Check for non-zero size setup.py files
-                to_scan.extend(setup_info['Scripts'])
+        setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
+        for fn in setupfiles:
+            if os.path.getsize(fn):
+                break
        else:
-            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+            logger.debug("No pyproject.toml found")
+            return False
-        if not to_scan:
-            to_scan = ['.']
-        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
-        provided_packages = self.parse_pkgdata_for_python_packages()
+        setupscript = os.path.join(srctree, "pyproject.toml")
-        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
-        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
-        for dep in scanned_deps:
-            mapped = provided_packages.get(dep)
-            if mapped:
-                logger.debug('Mapped %s to %s' % (dep, mapped))
-                mapped_deps.add(mapped)
-            else:
-                logger.debug('Could not map %s' % dep)
-                unmapped_deps.add(dep)
-        return mapped_deps, unmapped_deps
-    def scan_python_dependencies(self, paths):
-        deps = set()
        try:
-            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+            try:
-        except (OSError, subprocess.CalledProcessError):
+                import tomllib
-            pass
+            except ImportError:
-        else:
+                try:
-            for line in dep_output.splitlines():
+                    import tomli as tomllib
-                line = line.rstrip()
+                except ImportError:
-                dep, filename = line.split('\t', 1)
+                    logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.")
-                if filename.endswith('/setup.py'):
+                    return False
-                    continue
-                deps.add(dep)
+            try:
+                with open(setupscript, "rb") as f:
+                    config = tomllib.load(f)
+            except Exception:
+                logger.exception("Failed to parse pyproject.toml")
+                return False
+            build_backend = config["build-system"]["build-backend"]
+            if build_backend in self.build_backend_map:
+                classes.append(self.build_backend_map[build_backend])
+            else:
+                logger.error(
+                    "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py"
+                    % build_backend
+                )
+                return False
-        try:
+            licfile = ""
-            provides_output = self.run_command(['pythondeps', '-p'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            provides_lines = (l.rstrip() for l in provides_output.splitlines())
-            provides = set(l for l in provides_lines if l and l != 'setup')
-            deps -= provides
-        return deps
+            if build_backend == "poetry.core.masonry.api":
+                if "tool" in config and "poetry" in config["tool"]:
+                    metadata = config["tool"]["poetry"]
+            else:
+                if "project" in config:
+                    metadata = config["project"]
+            if metadata:
+                for field, values in metadata.items():
+                    if field == "license":
+                        # For setuptools.build_meta and flit, licence is a table
+                        # but for poetry licence is a string
+                        # for hatchling, both table (jsonschema) and string (iniconfig) have been used
+                        if build_backend == "poetry.core.masonry.api":
+                            value = values
+                        else:
+                            value = values.get("text", "")
+                            if not value:
+                                licfile = values.get("file", "")
+                                continue
+                    elif field == "dependencies" and build_backend == "poetry.core.masonry.api":
+                        # For poetry backend, "dependencies" section looks like:
+                        # [tool.poetry.dependencies]
+                        # requests = "^2.13.0"
+                        # requests = { version = "^2.13.0", source = "private" }
+                        # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details
+                        # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list
+                        value = []
+                        for k in values.keys():
+                            value.append(k)
+                    elif isinstance(values, dict):
+                        for k, v in values.items():
+                            info[k] = v
+                        continue
+                    else:
+                        value = values
-    def parse_pkgdata_for_python_packages(self):
+                    info[field] = value
-        suffixes = [t[0] for t in imp.get_suffixes()]
-        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
-        ldata = tinfoil.config_data.createCopy()
+            # Grab the license value before applying replacements
-        bb.parse.handle('classes/python3-dir.bbclass', ldata, True)
+            license_str = info.get("license", "").strip()
-        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
-        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+            if license_str:
-        python_dirs = [python_sitedir + os.sep,
+                for i, line in enumerate(lines_before):
-                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                    if line.startswith("##LICENSE_PLACEHOLDER##"):
-                       os.path.dirname(python_sitedir) + os.sep]
+                        lines_before.insert(
-        packages = {}
+                            i, "# NOTE: License in pyproject.toml is: %s" % license_str
-        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+                        )
-            files_info = None
-            with open(pkgdatafile, 'r') as f:
-                for line in f.readlines():
-                    field, value = line.split(': ', 1)
-                    if field == 'FILES_INFO':
-                        files_info = ast.literal_eval(value)
                        break
-                else:
-                    continue
-            for fn in files_info:
+            info["requires"] = config["build-system"]["requires"]
-                for suffix in suffixes:
-                    if fn.endswith(suffix):
+            self.apply_info_replacements(info)
-                        break
-                else:
+            if "classifiers" in info:
-                    continue
+                license = self.handle_classifier_license(
+                    info["classifiers"], info.get("license", "")
+                )
+                if license:
+                    if licfile:
+                        lines = []
+                        md5value = bb.utils.md5_file(os.path.join(srctree, licfile))
+                        lines.append('LICENSE = "%s"' % license)
+                        lines.append(
+                            'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
+                            % (licfile, md5value)
+                        )
+                        lines.append("")
+                        # Replace the placeholder so we get the values in the right place in the recipe file
+                        try:
+                            pos = lines_before.index("##LICENSE_PLACEHOLDER##")
+                        except ValueError:
+                            pos = -1
+                        if pos == -1:
+                            lines_before.extend(lines)
+                        else:
+                            lines_before[pos : pos + 1] = lines
-                if fn.startswith(dynload_dir + os.sep):
+                        handled.append(("license", [license, licfile, md5value]))
-                    if '/.debug/' in fn:
+                    else:
-                        continue
+                        info["license"] = license
-                    base = os.path.basename(fn)
-                    provided = base.split('.', 1)[0]
-                    packages[provided] = os.path.basename(pkgdatafile)
-                    continue
-                for python_dir in python_dirs:
+            provided_packages = self.parse_pkgdata_for_python_packages()
-                    if fn.startswith(python_dir):
+            provided_packages.update(self.known_deps_map)
-                        relpath = fn[len(python_dir):]
+            native_mapped_deps, native_unmapped_deps = set(), set()
-                        relstart, _, relremaining = relpath.partition(os.sep)
+            mapped_deps, unmapped_deps = set(), set()
-                        if relstart.endswith('.egg'):
-                            relpath = relremaining
-                        base, _ = os.path.splitext(relpath)
-                        if '/.debug/' in base:
+            if "requires" in info:
-                            continue
+                for require in info["requires"]:
-                        if os.path.basename(base) == '__init__':
+                    mapped = provided_packages.get(require)
-                            base = os.path.dirname(base)
-                        base = base.replace(os.sep + os.sep, os.sep)
-                        provided = base.replace(os.sep, '.')
-                        packages[provided] = os.path.basename(pkgdatafile)
-        return packages
-    @classmethod
+                    if mapped:
-    def run_command(cls, cmd, **popenargs):
+                        logger.debug("Mapped %s to %s" % (require, mapped))
-        if 'stderr' not in popenargs:
+                        native_mapped_deps.add(mapped)
-            popenargs['stderr'] = subprocess.STDOUT
+                    else:
-        try:
+                        logger.debug("Could not map %s" % require)
-            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+                        native_unmapped_deps.add(require)
-        except OSError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+                info.pop("requires")
-            raise
-        except subprocess.CalledProcessError as exc:
+                if native_mapped_deps != set():
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+                    native_mapped_deps = {
-            raise
+                        item + "-native" for item in native_mapped_deps
+                    }
+                    native_mapped_deps -= set(self.excluded_native_pkgdeps)
+                    if native_mapped_deps != set():
+                        info["requires"] = " ".join(sorted(native_mapped_deps))
+                if native_unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(native_unmapped_deps)
+                    )
+            if "dependencies" in info:
+                for dependency in info["dependencies"]:
+                    mapped = provided_packages.get(dependency)
+                    if mapped:
+                        logger.debug("Mapped %s to %s" % (dependency, mapped))
+                        mapped_deps.add(mapped)
+                    else:
+                        logger.debug("Could not map %s" % dependency)
+                        unmapped_deps.add(dependency)
+                info.pop("dependencies")
+                if mapped_deps != set():
+                    if mapped_deps != set():
+                        info["dependencies"] = " ".join(sorted(mapped_deps))
+                if unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# runtime dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(unmapped_deps)
+                    )
+            self.map_info_to_bbvar(info, extravalues)
+            handled.append("buildsystem")
+        except Exception:
+            logger.exception("Failed to correctly handle pyproject.toml, falling back to another method")
+            return False
 def gather_setup_info(fileobj):
@@ -710,5 +1118,7 @@ def has_non_literals(value):
 def register_recipe_handlers(handlers):
-    # We need to make sure this is ahead of the makefile fallback handler
+    # We need to make sure these are ahead of the makefile fallback handler
-    handlers.append((PythonRecipeHandler(), 70))
+    # and the pyproject.toml handler ahead of the setup.py handler
+    handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
+    handlers.append((PythonSetupPyRecipeHandler(), 70))
diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 0000000000..5cc53931f0
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,777 @@
+# Recipe creation tool - go support plugin
+#
+# The code is based on golang internals. See the afftected
+# methods for further reference and information.
+#
+# Copyright (C) 2023 Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+from collections import namedtuple
+from enum import Enum
+from html.parser import HTMLParser
+from recipetool.create import RecipeHandler, handle_license_vars
+from recipetool.create import find_licenses, tidy_licenses, fixup_license
+from recipetool.create import determine_from_url
+from urllib.error import URLError, HTTPError
+import bb.utils
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import shutil
+import tempfile
+import urllib.parse
+import urllib.request
+GoImport = namedtuple('GoImport', 'root vcs url suffix')
+logger = logging.getLogger('recipetool')
+CodeRepo = namedtuple(
+    'CodeRepo', 'path codeRoot codeDir pathMajor pathPrefix pseudoMajor')
+tinfoil = None
+# Regular expression to parse pseudo semantic version
+# see https://go.dev/ref/mod#pseudo-versions
+re_pseudo_semver = re.compile(
+    r"^v[0-9]+\.(0\.0-|\d+\.\d+-([^+]*\.)?0\.)(?P<utc>\d{14})-(?P<commithash>[A-Za-z0-9]+)(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$")
+# Regular expression to parse semantic version
+re_semver = re.compile(
+    r"^v(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$")
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+class GoRecipeHandler(RecipeHandler):
+    """Class to handle the go recipe creation"""
+    @staticmethod
+    def __ensure_go():
+        """Check if the 'go' command is available in the recipes"""
+        recipe = "go-native"
+        if not tinfoil.recipes_parsed:
+            tinfoil.parse_recipes()
+        try:
+            rd = tinfoil.parse_recipe(recipe)
+        except bb.providers.NoProvider:
+            bb.error(
+                "Nothing provides '%s' which is required for the build" % (recipe))
+            bb.note(
+                "You will likely need to add a layer that provides '%s'" % (recipe))
+            return None
+        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+        gopath = os.path.join(bindir, 'go')
+        if not os.path.exists(gopath):
+            tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
+            if not os.path.exists(gopath):
+                logger.error(
+                    '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
+                return None
+        return bindir
+    def __resolve_repository_static(self, modulepath):
+        """Resolve the repository in a static manner
+            The method is based on the go implementation of
+            `repoRootFromVCSPaths` in
+            https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
+        """
+        url = urllib.parse.urlparse("https://" + modulepath)
+        req = urllib.request.Request(url.geturl())
+        try:
+            resp = urllib.request.urlopen(req)
+            # Some modulepath are just redirects to github (or some other vcs
+            # hoster). Therefore, we check if this modulepath redirects to
+            # somewhere else
+            if resp.geturl() != url.geturl():
+                bb.debug(1, "%s is redirectred to %s" %
+                         (url.geturl(), resp.geturl()))
+                url = urllib.parse.urlparse(resp.geturl())
+                modulepath = url.netloc + url.path
+        except URLError as url_err:
+            # This is probably because the module path
+            # contains the subdir and major path. Thus,
+            # we ignore this error for now
+            logger.debug(
+                1, "Failed to fetch page from [%s]: %s" % (url, str(url_err)))
+        host, _, _ = modulepath.partition('/')
+        class vcs(Enum):
+            pathprefix = "pathprefix"
+            regexp = "regexp"
+            type = "type"
+            repo = "repo"
+            check = "check"
+            schemelessRepo = "schemelessRepo"
+        # GitHub
+        vcsGitHub = {}
+        vcsGitHub[vcs.pathprefix] = "github.com"
+        vcsGitHub[vcs.regexp] = re.compile(
+            r'^(?P<root>github\.com/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsGitHub[vcs.type] = "git"
+        vcsGitHub[vcs.repo] = "https://\\g<root>"
+        # Bitbucket
+        vcsBitbucket = {}
+        vcsBitbucket[vcs.pathprefix] = "bitbucket.org"
+        vcsBitbucket[vcs.regexp] = re.compile(
+            r'^(?P<root>bitbucket\.org/(?P<bitname>[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+))(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsBitbucket[vcs.type] = "git"
+        vcsBitbucket[vcs.repo] = "https://\\g<root>"
+        # IBM DevOps Services (JazzHub)
+        vcsIBMDevOps = {}
+        vcsIBMDevOps[vcs.pathprefix] = "hub.jazz.net/git"
+        vcsIBMDevOps[vcs.regexp] = re.compile(
+            r'^(?P<root>hub\.jazz\.net/git/[a-z0-9]+/[A-Za-z0-9_.\-]+)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsIBMDevOps[vcs.type] = "git"
+        vcsIBMDevOps[vcs.repo] = "https://\\g<root>"
+        # Git at Apache
+        vcsApacheGit = {}
+        vcsApacheGit[vcs.pathprefix] = "git.apache.org"
+        vcsApacheGit[vcs.regexp] = re.compile(
+            r'^(?P<root>git\.apache\.org/[a-z0-9_.\-]+\.git)(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsApacheGit[vcs.type] = "git"
+        vcsApacheGit[vcs.repo] = "https://\\g<root>"
+        # Git at OpenStack
+        vcsOpenStackGit = {}
+        vcsOpenStackGit[vcs.pathprefix] = "git.openstack.org"
+        vcsOpenStackGit[vcs.regexp] = re.compile(
+            r'^(?P<root>git\.openstack\.org/[A-Za-z0-9_.\-]+/[A-Za-z0-9_.\-]+)(\.git)?(/(?P<suffix>[A-Za-z0-9_.\-]+))*$')
+        vcsOpenStackGit[vcs.type] = "git"
+        vcsOpenStackGit[vcs.repo] = "https://\\g<root>"
+        # chiselapp.com for fossil
+        vcsChiselapp = {}
+        vcsChiselapp[vcs.pathprefix] = "chiselapp.com"
+        vcsChiselapp[vcs.regexp] = re.compile(
+            r'^(?P<root>chiselapp\.com/user/[A-Za-z0-9]+/repository/[A-Za-z0-9_.\-]+)$')
+        vcsChiselapp[vcs.type] = "fossil"
+        vcsChiselapp[vcs.repo] = "https://\\g<root>"
+        # General syntax for any server.
+        # Must be last.
+        vcsGeneralServer = {}
+        vcsGeneralServer[vcs.regexp] = re.compile(
+            "(?P<root>(?P<repo>([a-z0-9.\\-]+\\.)+[a-z0-9.\\-]+(:[0-9]+)?(/~?[A-Za-z0-9_.\\-]+)+?)\\.(?P<vcs>bzr|fossil|git|hg|svn))(/~?(?P<suffix>[A-Za-z0-9_.\\-]+))*$")
+        vcsGeneralServer[vcs.schemelessRepo] = True
+        vcsPaths = [vcsGitHub, vcsBitbucket, vcsIBMDevOps,
+                    vcsApacheGit, vcsOpenStackGit, vcsChiselapp,
+                    vcsGeneralServer]
+        if modulepath.startswith("example.net") or modulepath == "rsc.io":
+            logger.warning("Suspicious module path %s" % modulepath)
+            return None
+        if modulepath.startswith("http:") or modulepath.startswith("https:"):
+            logger.warning("Import path should not start with %s %s" %
+                           ("http", "https"))
+            return None
+        rootpath = None
+        vcstype = None
+        repourl = None
+        suffix = None
+        for srv in vcsPaths:
+            m = srv[vcs.regexp].match(modulepath)
+            if vcs.pathprefix in srv:
+                if host == srv[vcs.pathprefix]:
+                    rootpath = m.group('root')
+                    vcstype = srv[vcs.type]
+                    repourl = m.expand(srv[vcs.repo])
+                    suffix = m.group('suffix')
+                    break
+            elif m and srv[vcs.schemelessRepo]:
+                rootpath = m.group('root')
+                vcstype = m[vcs.type]
+                repourl = m[vcs.repo]
+                suffix = m.group('suffix')
+                break
+        return GoImport(rootpath, vcstype, repourl, suffix)
+    def __resolve_repository_dynamic(self, modulepath):
+        """Resolve the repository root in a dynamic manner.
+            The method is based on the go implementation of
+            `repoRootForImportDynamic` in
+            https://github.com/golang/go/blob/master/src/cmd/go/internal/vcs/vcs.go
+        """
+        url = urllib.parse.urlparse("https://" + modulepath)
+        class GoImportHTMLParser(HTMLParser):
+            def __init__(self):
+                super().__init__()
+                self.__srv = {}
+            def handle_starttag(self, tag, attrs):
+                if tag == 'meta' and list(
+                        filter(lambda a: (a[0] == 'name' and a[1] == 'go-import'), attrs)):
+                    content = list(
+                        filter(lambda a: (a[0] == 'content'), attrs))
+                    if content:
+                        srv = content[0][1].split()
+                        self.__srv[srv[0]] = srv
+            def go_import(self, modulepath):
+                if modulepath in self.__srv:
+                    srv = self.__srv[modulepath]
+                    return GoImport(srv[0], srv[1], srv[2], None)
+                return None
+        url = url.geturl() + "?go-get=1"
+        req = urllib.request.Request(url)
+        try:
+            body = urllib.request.urlopen(req).read()
+        except HTTPError as http_err:
+            logger.warning(
+                "Unclean status when fetching page from [%s]: %s", url, str(http_err))
+            body = http_err.fp.read()
+        except URLError as url_err:
+            logger.warning(
+                "Failed to fetch page from [%s]: %s", url, str(url_err))
+            return None
+        parser = GoImportHTMLParser()
+        parser.feed(body.decode('utf-8'))
+        parser.close()
+        return parser.go_import(modulepath)
+    def __resolve_from_golang_proxy(self, modulepath, version):
+        """
+        Resolves repository data from golang proxy
+        """
+        url = urllib.parse.urlparse("https://proxy.golang.org/"
+                                    + modulepath
+                                    + "/@v/"
+                                    + version
+                                    + ".info")
+        # Transform url to lower case, golang proxy doesn't like mixed case
+        req = urllib.request.Request(url.geturl().lower())
+        try:
+            resp = urllib.request.urlopen(req)
+        except URLError as url_err:
+            logger.warning(
+                "Failed to fetch page from [%s]: %s", url, str(url_err))
+            return None
+        golang_proxy_res = resp.read().decode('utf-8')
+        modinfo = json.loads(golang_proxy_res)
+        if modinfo and 'Origin' in modinfo:
+            origin = modinfo['Origin']
+            _root_url = urllib.parse.urlparse(origin['URL'])
+            # We normalize the repo URL since we don't want the scheme in it
+            _subdir = origin['Subdir'] if 'Subdir' in origin else None
+            _root, _, _ = self.__split_path_version(modulepath)
+            if _subdir:
+                _root = _root[:-len(_subdir)].strip('/')
+            _commit = origin['Hash']
+            _vcs = origin['VCS']
+            return (GoImport(_root, _vcs, _root_url.geturl(), None), _commit)
+        return None
+    def __resolve_repository(self, modulepath):
+        """
+        Resolves src uri from go module-path
+        """
+        repodata = self.__resolve_repository_static(modulepath)
+        if not repodata or not repodata.url:
+            repodata = self.__resolve_repository_dynamic(modulepath)
+            if not repodata or not repodata.url:
+                logger.error(
+                    "Could not resolve repository for module path '%s'" % modulepath)
+                # There is no way to recover from this
+                sys.exit(14)
+        if repodata:
+            logger.debug(1, "Resolved download path for import '%s' => %s" % (
+                modulepath, repodata.url))
+        return repodata
+    def __split_path_version(self, path):
+        i = len(path)
+        dot = False
+        for j in range(i, 0, -1):
+            if path[j - 1] < '0' or path[j - 1] > '9':
+                break
+            if path[j - 1] == '.':
+                dot = True
+                break
+            i = j - 1
+        if i <= 1 or i == len(
+                path) or path[i - 1] != 'v' or path[i - 2] != '/':
+            return path, "", True
+        prefix, pathMajor = path[:i - 2], path[i - 2:]
+        if dot or len(
+                pathMajor) <= 2 or pathMajor[2] == '0' or pathMajor == "/v1":
+            return path, "", False
+        return prefix, pathMajor, True
+    def __get_path_major(self, pathMajor):
+        if not pathMajor:
+            return ""
+        if pathMajor[0] != '/' and pathMajor[0] != '.':
+            logger.error(
+                "pathMajor suffix %s passed to PathMajorPrefix lacks separator", pathMajor)
+        if pathMajor.startswith(".v") and pathMajor.endswith("-unstable"):
+            pathMajor = pathMajor[:len("-unstable") - 2]
+        return pathMajor[1:]
+    def __build_coderepo(self, repo, path):
+        codedir = ""
+        pathprefix, pathMajor, _ = self.__split_path_version(path)
+        if repo.root == path:
+            pathprefix = path
+        elif path.startswith(repo.root):
+            codedir = pathprefix[len(repo.root):].strip('/')
+        pseudoMajor = self.__get_path_major(pathMajor)
+        logger.debug("root='%s', codedir='%s', prefix='%s', pathMajor='%s', pseudoMajor='%s'",
+                     repo.root, codedir, pathprefix, pathMajor, pseudoMajor)
+        return CodeRepo(path, repo.root, codedir,
+                        pathMajor, pathprefix, pseudoMajor)
+    def __resolve_version(self, repo, path, version):
+        hash = None
+        coderoot = self.__build_coderepo(repo, path)
+        def vcs_fetch_all():
+            tmpdir = tempfile.mkdtemp()
+            clone_cmd = "%s clone --bare %s %s" % ('git', repo.url, tmpdir)
+            bb.process.run(clone_cmd)
+            log_cmd = "git log --all --pretty='%H %d' --decorate=short"
+            output, _ = bb.process.run(
+                log_cmd, shell=True, stderr=subprocess.PIPE, cwd=tmpdir)
+            bb.utils.prunedir(tmpdir)
+            return output.strip().split('\n')
+        def vcs_fetch_remote(tag):
+            # add * to grab ^{}
+            refs = {}
+            ls_remote_cmd = "git ls-remote -q --tags {} {}*".format(
+                repo.url, tag)
+            output, _ = bb.process.run(ls_remote_cmd)
+            output = output.strip().split('\n')
+            for line in output:
+                f = line.split(maxsplit=1)
+                if len(f) != 2:
+                    continue
+                for prefix in ["HEAD", "refs/heads/", "refs/tags/"]:
+                    if f[1].startswith(prefix):
+                        refs[f[1][len(prefix):]] = f[0]
+            for key, hash in refs.items():
+                if key.endswith(r"^{}"):
+                    refs[key.strip(r"^{}")] = hash
+            return refs[tag]
+        m_pseudo_semver = re_pseudo_semver.match(version)
+        if m_pseudo_semver:
+            remote_refs = vcs_fetch_all()
+            short_commit = m_pseudo_semver.group('commithash')
+            for l in remote_refs:
+                r = l.split(maxsplit=1)
+                sha1 = r[0] if len(r) else None
+                if not sha1:
+                    logger.error(
+                        "Ups: could not resolve abbref commit for %s" % short_commit)
+                elif sha1.startswith(short_commit):
+                    hash = sha1
+                    break
+        else:
+            m_semver = re_semver.match(version)
+            if m_semver:
+                def get_sha1_remote(re):
+                    rsha1 = None
+                    for line in remote_refs:
+                        # Split lines of the following format:
+                        # 22e90d9b964610628c10f673ca5f85b8c2a2ca9a  (tag: sometag)
+                        lineparts = line.split(maxsplit=1)
+                        sha1 = lineparts[0] if len(lineparts) else None
+                        refstring = lineparts[1] if len(
+                            lineparts) == 2 else None
+                        if refstring:
+                            # Normalize tag string and split in case of multiple
+                            # regs e.g. (tag: speech/v1.10.0, tag: orchestration/v1.5.0 ...)
+                            refs = refstring.strip('(), ').split(',')
+                            for ref in refs:
+                                if re.match(ref.strip()):
+                                    rsha1 = sha1
+                    return rsha1
+                semver = "v" + m_semver.group('major') + "."\
+                             + m_semver.group('minor') + "."\
+                             + m_semver.group('patch') \
+                             + (("-" + m_semver.group('prerelease'))
+                                if m_semver.group('prerelease') else "")
+                tag = os.path.join(
+                    coderoot.codeDir, semver) if coderoot.codeDir else semver
+                # probe tag using 'ls-remote', which is faster than fetching
+                # complete history
+                hash = vcs_fetch_remote(tag)
+                if not hash:
+                    # backup: fetch complete history
+                    remote_refs = vcs_fetch_all()
+                    hash = get_sha1_remote(
+                        re.compile(fr"(tag:|HEAD ->) ({tag})"))
+                logger.debug(
+                    "Resolving commit for tag '%s' -> '%s'", tag, hash)
+        return hash
+    def __generate_srcuri_inline_fcn(self, path, version, replaces=None):
+        """Generate SRC_URI functions for go imports"""
+        logger.info("Resolving repository for module %s", path)
+        # First try to resolve repo and commit from golang proxy
+        # Most info is already there and we don't have to go through the
+        # repository or even perform the version resolve magic
+        golang_proxy_info = self.__resolve_from_golang_proxy(path, version)
+        if golang_proxy_info:
+            repo = golang_proxy_info[0]
+            commit = golang_proxy_info[1]
+        else:
+            # Fallback
+            # Resolve repository by 'hand'
+            repo = self.__resolve_repository(path)
+            commit = self.__resolve_version(repo, path, version)
+        url = urllib.parse.urlparse(repo.url)
+        repo_url = url.netloc + url.path
+        coderoot = self.__build_coderepo(repo, path)
+        inline_fcn = "${@go_src_uri("
+        inline_fcn += f"'{repo_url}','{version}'"
+        if repo_url != path:
+            inline_fcn += f",path='{path}'"
+        if coderoot.codeDir:
+            inline_fcn += f",subdir='{coderoot.codeDir}'"
+        if repo.vcs != 'git':
+            inline_fcn += f",vcs='{repo.vcs}'"
+        if replaces:
+            inline_fcn += f",replaces='{replaces}'"
+        if coderoot.pathMajor:
+            inline_fcn += f",pathmajor='{coderoot.pathMajor}'"
+        inline_fcn += ")}"
+        return inline_fcn, commit
+    def __go_handle_dependencies(self, go_mod, srctree, localfilesdir, extravalues, d):
+        import re
+        src_uris = []
+        src_revs = []
+        def generate_src_rev(path, version, commithash):
+            src_rev = f"# {path}@{version} => {commithash}\n"
+            # Ups...maybe someone manipulated the source repository and the
+            # version or commit could not be resolved. This is a sign of
+            # a) the supply chain was manipulated (bad)
+            # b) the implementation for the version resolving didn't work
+            #    anymore (less bad)
+            if not commithash:
+                src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+                src_rev += f"#!!!   Could not resolve version  !!!\n"
+                src_rev += f"#!!! Possible supply chain attack !!!\n"
+                src_rev += f"#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"
+            src_rev += f"SRCREV_{path.replace('/', '.')} = \"{commithash}\""
+            return src_rev
+        # we first go over replacement list, because we are essentialy
+        # interested only in the replaced path
+        if go_mod['Replace']:
+            for replacement in go_mod['Replace']:
+                oldpath = replacement['Old']['Path']
+                path = replacement['New']['Path']
+                version = ''
+                if 'Version' in replacement['New']:
+                    version = replacement['New']['Version']
+                if os.path.exists(os.path.join(srctree, path)):
+                    # the module refers to the local path, remove it from requirement list
+                    # because it's a local module
+                    go_mod['Require'][:] = [v for v in go_mod['Require'] if v.get('Path') != oldpath]
+                else:
+                    # Replace the path and the version, so we don't iterate replacement list anymore
+                    for require in go_mod['Require']:
+                        if require['Path'] == oldpath:
+                            require.update({'Path': path, 'Version': version})
+                            break
+        for require in go_mod['Require']:
+            path = require['Path']
+            version = require['Version']
+            inline_fcn, commithash = self.__generate_srcuri_inline_fcn(
+                path, version)
+            src_uris.append(inline_fcn)
+            src_revs.append(generate_src_rev(path, version, commithash))
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
+        go_mods_basename = "%s-modules.inc" % pn
+        go_mods_filename = os.path.join(localfilesdir, go_mods_basename)
+        with open(go_mods_filename, "w") as f:
+            # We introduce this indirection to make the tests a little easier
+            f.write("SRC_URI += \"${GO_DEPENDENCIES_SRC_URI}\"\n")
+            f.write("GO_DEPENDENCIES_SRC_URI = \"\\\n")
+            for uri in src_uris:
+                f.write("    " + uri + " \\\n")
+            f.write("\"\n\n")
+            for rev in src_revs:
+                f.write(rev + "\n")
+        extravalues['extrafiles'][go_mods_basename] = go_mods_filename
+    def __go_run_cmd(self, cmd, cwd, d):
+        return bb.process.run(cmd, env=dict(os.environ, PATH=d.getVar('PATH')),
+                              shell=True, cwd=cwd)
+    def __go_native_version(self, d):
+        stdout, _ = self.__go_run_cmd("go version", None, d)
+        m = re.match(r".*\sgo((\d+).(\d+).(\d+))\s([\w\/]*)", stdout)
+        major = int(m.group(2))
+        minor = int(m.group(3))
+        patch = int(m.group(4))
+        return major, minor, patch
+    def __go_mod_patch(self, srctree, localfilesdir, extravalues, d):
+        patchfilename = "go.mod.patch"
+        go_native_version_major, go_native_version_minor, _ = self.__go_native_version(
+            d)
+        self.__go_run_cmd("go mod tidy -go=%d.%d" %
+                          (go_native_version_major, go_native_version_minor), srctree, d)
+        stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
+        # Create patch in order to upgrade go version
+        self.__go_run_cmd("git diff go.mod > %s" % (patchfilename), srctree, d)
+        # Restore original state
+        self.__go_run_cmd("git checkout HEAD go.mod go.sum", srctree, d)
+        go_mod = json.loads(stdout)
+        tmpfile = os.path.join(localfilesdir, patchfilename)
+        shutil.move(os.path.join(srctree, patchfilename), tmpfile)
+        extravalues['extrafiles'][patchfilename] = tmpfile
+        return go_mod, patchfilename
+    def __go_mod_vendor(self, go_mod, srctree, localfilesdir, extravalues, d):
+        # Perform vendoring to retrieve the correct modules.txt
+        tmp_vendor_dir = tempfile.mkdtemp()
+        # -v causes to go to print modules.txt to stderr
+        _, stderr = self.__go_run_cmd(
+            "go mod vendor -v -o %s" % (tmp_vendor_dir), srctree, d)
+        modules_txt_basename = "modules.txt"
+        modules_txt_filename = os.path.join(localfilesdir, modules_txt_basename)
+        with open(modules_txt_filename, "w") as f:
+            f.write(stderr)
+        extravalues['extrafiles'][modules_txt_basename] = modules_txt_filename
+        licenses = []
+        lic_files_chksum = []
+        licvalues = find_licenses(tmp_vendor_dir, d)
+        shutil.rmtree(tmp_vendor_dir)
+        if licvalues:
+            for licvalue in licvalues:
+                license = licvalue[0]
+                lics = tidy_licenses(fixup_license(license))
+                lics = [lic for lic in lics if lic not in licenses]
+                if len(lics):
+                    licenses.extend(lics)
+                lic_files_chksum.append(
+                    'file://src/${GO_IMPORT}/vendor/%s;md5=%s' % (licvalue[1], licvalue[2]))
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
+        licenses_basename = "%s-licenses.inc" % pn
+        licenses_filename = os.path.join(localfilesdir, licenses_basename)
+        with open(licenses_filename, "w") as f:
+            f.write("GO_MOD_LICENSES = \"%s\"\n\n" %
+                    ' & '.join(sorted(licenses, key=str.casefold)))
+            # We introduce this indirection to make the tests a little easier
+            f.write("LIC_FILES_CHKSUM  += \"${VENDORED_LIC_FILES_CHKSUM}\"\n")
+            f.write("VENDORED_LIC_FILES_CHKSUM = \"\\\n")
+            for lic in lic_files_chksum:
+                f.write("    " + lic + " \\\n")
+            f.write("\"\n")
+        extravalues['extrafiles'][licenses_basename] = licenses_filename
+    def process(self, srctree, classes, lines_before,
+                lines_after, handled, extravalues):
+        if 'buildsystem' in handled:
+            return False
+        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
+        if not files:
+            return False
+        d = bb.data.createCopy(tinfoil.config_data)
+        go_bindir = self.__ensure_go()
+        if not go_bindir:
+            sys.exit(14)
+        d.prependVar('PATH', '%s:' % go_bindir)
+        handled.append('buildsystem')
+        classes.append("go-vendor")
+        stdout, _ = self.__go_run_cmd("go mod edit -json", srctree, d)
+        go_mod = json.loads(stdout)
+        go_import = go_mod['Module']['Path']
+        go_version_match = re.match("([0-9]+).([0-9]+)", go_mod['Go'])
+        go_version_major = int(go_version_match.group(1))
+        go_version_minor = int(go_version_match.group(2))
+        src_uris = []
+        localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+        extravalues.setdefault('extrafiles', {})
+        # Use an explicit name determined from the module name because it
+        # might differ from the actual URL for replaced modules
+        # strip version part from module URL /vXX
+        baseurl = re.sub(r'/v(\d+)$', '', go_mod['Module']['Path'])
+        pn, _ = determine_from_url(baseurl)
+        # go.mod files with version < 1.17 may not include all indirect
+        # dependencies. Thus, we have to upgrade the go version.
+        if go_version_major == 1 and go_version_minor < 17:
+            logger.warning(
+                "go.mod files generated by Go < 1.17 might have incomplete indirect dependencies.")
+            go_mod, patchfilename = self.__go_mod_patch(srctree, localfilesdir,
+                                                        extravalues, d)
+            src_uris.append(
+                "file://%s;patchdir=src/${GO_IMPORT}" % (patchfilename))
+        # Check whether the module is vendored. If so, we have nothing to do.
+        # Otherwise we gather all dependencies and add them to the recipe
+        if not os.path.exists(os.path.join(srctree, "vendor")):
+            # Write additional $BPN-modules.inc file
+            self.__go_mod_vendor(go_mod, srctree, localfilesdir, extravalues, d)
+            lines_before.append("LICENSE += \" & ${GO_MOD_LICENSES}\"")
+            lines_before.append("require %s-licenses.inc" % (pn))
+            self.__rewrite_src_uri(lines_before, ["file://modules.txt"])
+            self.__go_handle_dependencies(go_mod, srctree, localfilesdir, extravalues, d)
+            lines_before.append("require %s-modules.inc" % (pn))
+        # Do generic license handling
+        handle_license_vars(srctree, lines_before, handled, extravalues, d)
+        self.__rewrite_lic_uri(lines_before)
+        lines_before.append("GO_IMPORT = \"{}\"".format(baseurl))
+        lines_before.append("SRCREV_FORMAT = \"${BPN}\"")
+    def __update_lines_before(self, updated, newlines, lines_before):
+        if updated:
+            del lines_before[:]
+            for line in newlines:
+                # Hack to avoid newlines that edit_metadata inserts
+                if line.endswith('\n'):
+                    line = line[:-1]
+                lines_before.append(line)
+        return updated
+    def __rewrite_lic_uri(self, lines_before):
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'LIC_FILES_CHKSUM':
+                new_licenses = []
+                licenses = origvalue.split('\\')
+                for license in licenses:
+                    if not license:
+                        logger.warning("No license file was detected for the main module!")
+                        # the license list of the main recipe must be empty
+                        # this can happen for example in case of CLOSED license
+                        # Fall through to complete recipe generation
+                        continue
+                    license = license.strip()
+                    uri, chksum = license.split(';', 1)
+                    url = urllib.parse.urlparse(uri)
+                    new_uri = os.path.join(
+                        url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
+                    new_licenses.append(new_uri)
+                return new_licenses, None, -1, True
+            return origvalue, None, 0, True
+        updated, newlines = bb.utils.edit_metadata(
+            lines_before, ['LIC_FILES_CHKSUM'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+    def __rewrite_src_uri(self, lines_before, additional_uris = []):
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'SRC_URI':
+                src_uri = ["git://${GO_IMPORT};destsuffix=git/src/${GO_IMPORT};nobranch=1;name=${BPN};protocol=https"]
+                src_uri.extend(additional_uris)
+                return src_uri, None, -1, True
+            return origvalue, None, 0, True
+        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+def register_recipe_handlers(handlers):
+    handlers.append((GoRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
index 85b5c48e53..cc00106961 100644
--- a/scripts/lib/recipetool/create_kmod.py
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -113,7 +113,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
                kdirpath, _ = check_target(compile_lines, install=False)
            if manual_install or not install_lines:
-                lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
+                lines_after.append('EXTRA_OEMAKE:append:task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
            elif install_target and install_target != 'modules_install':
                lines_after.append('MODULES_INSTALL_TARGET = "install"')
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 2bcae91dfa..8c4cdd5234 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -6,16 +6,20 @@
 """Recipe creation tool - npm module support plugin"""
 import json
+import logging
 import os
 import re
 import sys
 import tempfile
 import bb
 from bb.fetch2.npm import NpmEnvironment
+from bb.fetch2.npm import npm_package
 from bb.fetch2.npmsw import foreach_dependencies
+from oe.license_finder import match_licenses, find_license_files
 from recipetool.create import RecipeHandler
-from recipetool.create import guess_license
+from recipetool.create import generate_common_licenses_chksums
 from recipetool.create import split_pkg_licenses
+logger = logging.getLogger('recipetool')
 TINFOIL = None
@@ -28,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler):
    """Class to handle the npm recipe creation"""
    @staticmethod
-    def _npm_name(name):
-        """Generate a Yocto friendly npm name"""
-        name = re.sub("/", "-", name)
-        name = name.lower()
-        name = re.sub(r"[^\-a-z0-9]", "", name)
-        name = name.strip("-")
-        return name
-    @staticmethod
    def _get_registry(lines):
        """Get the registry value from the 'npm://registry' url"""
        registry = None
@@ -117,31 +112,71 @@ class NpmRecipeHandler(RecipeHandler):
        """Return the extra license files and the list of packages"""
        licfiles = []
        packages = {}
+        # Licenses from package.json will point to COMMON_LICENSE_DIR so we need
+        # to associate them explicitely to packages for split_pkg_licenses()
+        fallback_licenses = dict()
+        def _find_package_licenses(destdir):
+            """Either find license files, or use package.json metadata"""
+            def _get_licenses_from_package_json(package_json):
+                with open(os.path.join(srctree, package_json), "r") as f:
+                    data = json.load(f)
+                    if "license" in data:
+                        licenses = data["license"].split(" ")
+                        licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"]
+                        return [], licenses
+                    else:
+                        return [package_json], None
+            basedir = os.path.join(srctree, destdir)
+            licfiles = find_license_files(basedir)
+            if len(licfiles) > 0:
+                return licfiles, None
+            else:
+                # A license wasn't found in the package directory, so we'll use the package.json metadata
+                pkg_json = os.path.join(basedir, "package.json")
+                return _get_licenses_from_package_json(pkg_json)
+        def _get_package_licenses(destdir, package):
+            (package_licfiles, package_licenses) = _find_package_licenses(destdir)
+            if package_licfiles:
+                licfiles.extend(package_licfiles)
+            else:
+                fallback_licenses[package] = package_licenses
-        def _licfiles_append(licfile):
+        # Handle the dependencies
-            """Append 'licfile' to the license files list"""
+        def _handle_dependency(name, params, destdir):
-            licfilepath = os.path.join(srctree, licfile)
+            deptree = destdir.split('node_modules/')
-            licmd5 = bb.utils.md5_file(licfilepath)
+            suffix = "-".join([npm_package(dep) for dep in deptree])
-            licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
+            packages["${PN}" + suffix] = destdir
+            _get_package_licenses(destdir, "${PN}" + suffix)
+        with open(shrinkwrap_file, "r") as f:
+            shrinkwrap = json.load(f)
+        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
        # Handle the parent package
-        _licfiles_append("package.json")
        packages["${PN}"] = ""
+        _get_package_licenses(srctree, "${PN}")
-        # Handle the dependencies
+        return licfiles, packages, fallback_licenses
-        def _handle_dependency(name, params, deptree):
+    
-            suffix = "-".join([self._npm_name(dep) for dep in deptree])
+    # Handle the peer dependencies   
-            destdirs = [os.path.join("node_modules", dep) for dep in deptree]
+    def _handle_peer_dependency(self, shrinkwrap_file):
-            destdir = os.path.join(*destdirs)
+        """Check if package has peer dependencies and show warning if it is the case"""
-            _licfiles_append(os.path.join(destdir, "package.json"))
-            packages["${PN}-" + suffix] = destdir
        with open(shrinkwrap_file, "r") as f:
            shrinkwrap = json.load(f)
+        
+        packages = shrinkwrap.get("packages", {})
+        peer_deps = packages.get("", {}).get("peerDependencies", {})
+        
+        for peer_dep in peer_deps:
+            peer_dep_yocto_name = npm_package(peer_dep)
+            bb.warn(peer_dep + " is a peer dependencie of the actual package. " + 
+            "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
+            % peer_dep_yocto_name)
-        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
-        return licfiles, packages
    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        """Handle the npm recipe creation"""
@@ -160,7 +195,7 @@ class NpmRecipeHandler(RecipeHandler):
        if "name" not in data or "version" not in data:
            return False
-        extravalues["PN"] = self._npm_name(data["name"])
+        extravalues["PN"] = npm_package(data["name"])
        extravalues["PV"] = data["version"]
        if "description" in data:
@@ -229,7 +264,7 @@ class NpmRecipeHandler(RecipeHandler):
            value = origvalue.replace("version=" + data["version"], "version=${PV}")
            value = value.replace("version=latest", "version=${PV}")
            values = [line.strip() for line in value.strip('\n').splitlines()]
-            if "dependencies" in shrinkwrap:
+            if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
                values.append(url_recipe)
            return values, None, 4, False
@@ -245,13 +280,19 @@ class NpmRecipeHandler(RecipeHandler):
        fetcher.unpack(srctree)
        bb.note("Handling licences ...")
-        (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
+        (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev)
-        extravalues["LIC_FILES_CHKSUM"] = licfiles
+        licvalues = match_licenses(licfiles, srctree, d)
-        split_pkg_licenses(guess_license(srctree, d), packages, lines_after, [])
+        split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses)
+        fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist]
+        extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d)
+        extravalues["LICENSE"] = fallback_licenses_flat
        classes.append("npm")
        handled.append("buildsystem")
+        # Check if package has peer dependencies and inform the user
+        self._handle_peer_dependency(shrinkwrap_file)
        return True
 def register_recipe_handlers(handlers):
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index f8e2ee75fb..b5ad335cae 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -49,6 +49,7 @@ def setvar(args):
        for patch in patches:
            for line in patch:
                sys.stdout.write(line)
+    tinfoil.modified_files()
    return 0
diff --git a/scripts/lib/resulttool/junit.py b/scripts/lib/resulttool/junit.py
new file mode 100644
index 0000000000..c7a53dc550
--- /dev/null
+++ b/scripts/lib/resulttool/junit.py
@@ -0,0 +1,77 @@
+# resulttool - report test results in JUnit XML format
+#
+# Copyright (c) 2024, Siemens AG.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+import os
+import re
+import xml.etree.ElementTree as ET
+import resulttool.resultutils as resultutils
+def junit(args, logger):
+    testresults = resultutils.load_resultsdata(args.json_file, configmap=resultutils.store_map)
+    total_time = 0
+    skipped = 0
+    failures = 0
+    errors = 0
+    for tests in testresults.values():
+        results = tests[next(reversed(tests))].get("result", {})
+    for result_id, result in results.items():
+        # filter out ptestresult.rawlogs and ptestresult.sections
+        if re.search(r'\.test_', result_id):
+            total_time += result.get("duration", 0)
+            if result['status'] == "FAILED":
+                failures += 1
+            elif result['status'] == "ERROR":
+                errors += 1
+            elif result['status'] == "SKIPPED":
+                skipped += 1
+    testsuites_node = ET.Element("testsuites")
+    testsuites_node.set("time", "%s" % total_time)
+    testsuite_node = ET.SubElement(testsuites_node, "testsuite")
+    testsuite_node.set("name", "Testimage")
+    testsuite_node.set("time", "%s" % total_time)
+    testsuite_node.set("tests", "%s" % len(results))
+    testsuite_node.set("failures", "%s" % failures)
+    testsuite_node.set("errors", "%s" % errors)
+    testsuite_node.set("skipped", "%s" % skipped)
+    for result_id, result in results.items():
+        if re.search(r'\.test_', result_id):
+            testcase_node = ET.SubElement(testsuite_node, "testcase", {
+                "name": result_id,
+                "classname": "Testimage",
+                "time": str(result['duration'])
+            })
+            if result['status'] == "SKIPPED":
+                ET.SubElement(testcase_node, "skipped", message=result['log'])
+            elif result['status'] == "FAILED":
+                ET.SubElement(testcase_node, "failure", message=result['log'])
+            elif result['status'] == "ERROR":
+                ET.SubElement(testcase_node, "error", message=result['log'])
+    tree = ET.ElementTree(testsuites_node)
+    if args.junit_xml_path is None:
+        args.junit_xml_path = os.environ['BUILDDIR'] + '/tmp/log/oeqa/junit.xml'
+    tree.write(args.junit_xml_path, encoding='UTF-8', xml_declaration=True)
+    logger.info('Saved JUnit XML report as %s' % args.junit_xml_path)
+def register_commands(subparsers):
+    """Register subcommands from this plugin"""
+    parser_build = subparsers.add_parser('junit', help='create test report in JUnit XML format',
+                                         description='generate unit test report in JUnit XML format based on the latest test results in the testresults.json.',
+                                         group='analysis')
+    parser_build.set_defaults(func=junit)
+    parser_build.add_argument('json_file',
+                              help='json file should point to the testresults.json')
+    parser_build.add_argument('-j', '--junit_xml_path',
+                              help='junit xml path allows setting the path of the generated test report. The default location is <build_dir>/tmp/log/oeqa/junit.xml')
diff --git a/scripts/lib/resulttool/log.py b/scripts/lib/resulttool/log.py
index eb3927ec82..15148ca288 100644
--- a/scripts/lib/resulttool/log.py
+++ b/scripts/lib/resulttool/log.py
@@ -28,12 +28,10 @@ def show_reproducible(result, reproducible, logger):
 def log(args, logger):
    results = resultutils.load_resultsdata(args.source)
-    ptest_count = sum(1 for _, _, _, r in resultutils.test_run_results(results) if 'ptestresult.sections' in r)
-    if ptest_count > 1 and not args.prepend_run:
-        print("%i ptest sections found. '--prepend-run' is required" % ptest_count)
-        return 1
    for _, run_name, _, r in resultutils.test_run_results(results):
+        if args.list_ptest:
+            print('\n'.join(sorted(r['ptestresult.sections'].keys())))
        if args.dump_ptest:
            for sectname in ['ptestresult.sections', 'ltpposixresult.sections', 'ltpresult.sections']:
             if sectname in r:
@@ -48,6 +46,9 @@ def log(args, logger):
                    os.makedirs(dest_dir, exist_ok=True)
                    dest = os.path.join(dest_dir, '%s.log' % name)
+                    if os.path.exists(dest):
+                        print("Overlapping ptest logs found, skipping %s. The '--prepend-run' option would avoid this" % name)
+                        continue
                    print(dest)
                    with open(dest, 'w') as f:
                        f.write(logdata)
@@ -86,6 +87,8 @@ def register_commands(subparsers):
    parser.set_defaults(func=log)
    parser.add_argument('source',
            help='the results file/directory/URL to import')
+    parser.add_argument('--list-ptest', action='store_true',
+            help='list the ptest test names')
    parser.add_argument('--ptest', action='append', default=[],
            help='show logs for a ptest')
    parser.add_argument('--dump-ptest', metavar='DIR',
diff --git a/scripts/lib/resulttool/manualexecution.py b/scripts/lib/resulttool/manualexecution.py
index ecb27c5933..ae0861ac6b 100755
--- a/scripts/lib/resulttool/manualexecution.py
+++ b/scripts/lib/resulttool/manualexecution.py
@@ -22,7 +22,7 @@ def load_json_file(f):
 def write_json_file(f, json_data):
    os.makedirs(os.path.dirname(f), exist_ok=True)
    with open(f, 'w') as filedata:
-        filedata.write(json.dumps(json_data, sort_keys=True, indent=4))
+        filedata.write(json.dumps(json_data, sort_keys=True, indent=1))
 class ManualTestRunner(object):
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index 9f952951b3..33b3119c54 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -7,17 +7,213 @@
 #
 import resulttool.resultutils as resultutils
-import json
 from oeqa.utils.git import GitRepo
 import oeqa.utils.gitarchive as gitarchive
-def compare_result(logger, base_name, target_name, base_result, target_result):
+METADATA_MATCH_TABLE = {
+    "oeselftest": "OESELFTEST_METADATA"
+}
+OESELFTEST_METADATA_GUESS_TABLE={
+    "trigger-build-posttrigger": {
+        "run_all_tests": False,
+        "run_tests":["buildoptions.SourceMirroring.test_yocto_source_mirror"],
+        "skips": None,
+        "machine": None,
+        "select_tags":None,
+        "exclude_tags": None
+    },
+    "reproducible": {
+        "run_all_tests": False,
+        "run_tests":["reproducible"],
+        "skips": None,
+        "machine": None,
+        "select_tags":None,
+        "exclude_tags": None
+    },
+    "arch-qemu-quick": {
+        "run_all_tests": True,
+        "run_tests":None,
+        "skips": None,
+        "machine": None,
+        "select_tags":["machine"],
+        "exclude_tags": None
+    },
+    "arch-qemu-full-x86-or-x86_64": {
+        "run_all_tests": True,
+        "run_tests":None,
+        "skips": None,
+        "machine": None,
+        "select_tags":["machine", "toolchain-system"],
+        "exclude_tags": None
+    },
+    "arch-qemu-full-others": {
+        "run_all_tests": True,
+        "run_tests":None,
+        "skips": None,
+        "machine": None,
+        "select_tags":["machine", "toolchain-user"],
+        "exclude_tags": None
+    },
+    "selftest": {
+        "run_all_tests": True,
+        "run_tests":None,
+        "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"],
+        "machine": None,
+        "select_tags":None,
+        "exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
+    },
+    "bringup": {
+        "run_all_tests": True,
+        "run_tests":None,
+        "skips": ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"],
+        "machine": None,
+        "select_tags":None,
+        "exclude_tags": ["machine", "toolchain-system", "toolchain-user"]
+    }
+}
+STATUS_STRINGS = {
+    "None": "No matching test result"
+}
+REGRESSIONS_DISPLAY_LIMIT=50
+MISSING_TESTS_BANNER =   "-------------------------- Missing tests --------------------------"
+ADDITIONAL_DATA_BANNER = "--------------------- Matches and improvements --------------------"
+def test_has_at_least_one_matching_tag(test, tag_list):
+    return "oetags" in test and any(oetag in tag_list for oetag in test["oetags"])
+def all_tests_have_at_least_one_matching_tag(results, tag_list):
+    return all(test_has_at_least_one_matching_tag(test_result, tag_list) or test_name.startswith("ptestresult") for (test_name, test_result) in results.items())
+def any_test_have_any_matching_tag(results, tag_list):
+    return any(test_has_at_least_one_matching_tag(test, tag_list) for test in results.values())
+def have_skipped_test(result, test_prefix):
+    return all( result[test]['status'] == "SKIPPED" for test in result if test.startswith(test_prefix))
+def have_all_tests_skipped(result, test_prefixes_list):
+    return all(have_skipped_test(result, test_prefix) for test_prefix in test_prefixes_list)
+def guess_oeselftest_metadata(results):
+    """
+    When an oeselftest test result is lacking OESELFTEST_METADATA, we can try to guess it based on results content.
+    Check results for specific values (absence/presence of oetags, number and name of executed tests...),
+    and if it matches one of known configuration from autobuilder configuration, apply guessed OSELFTEST_METADATA
+    to it to allow proper test filtering.
+    This guessing process is tightly coupled to config.json in autobuilder. It should trigger less and less,
+    as new tests will have OESELFTEST_METADATA properly appended at test reporting time
+    """
+    if len(results) == 1 and "buildoptions.SourceMirroring.test_yocto_source_mirror" in results:
+        return OESELFTEST_METADATA_GUESS_TABLE['trigger-build-posttrigger']
+    elif all(result.startswith("reproducible") for result in results):
+        return OESELFTEST_METADATA_GUESS_TABLE['reproducible']
+    elif all_tests_have_at_least_one_matching_tag(results, ["machine"]):
+        return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-quick']
+    elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-system"]):
+        return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-x86-or-x86_64']
+    elif all_tests_have_at_least_one_matching_tag(results, ["machine", "toolchain-user"]):
+        return OESELFTEST_METADATA_GUESS_TABLE['arch-qemu-full-others']
+    elif not any_test_have_any_matching_tag(results, ["machine", "toolchain-user", "toolchain-system"]):
+        if have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror", "reproducible"]):
+            return OESELFTEST_METADATA_GUESS_TABLE['selftest']
+        elif have_all_tests_skipped(results, ["distrodata.Distrodata.test_checkpkg", "buildoptions.SourceMirroring.test_yocto_source_mirror"]):
+            return OESELFTEST_METADATA_GUESS_TABLE['bringup']
+    return None
+def metadata_matches(base_configuration, target_configuration):
+    """
+    For passed base and target, check test type. If test type matches one of
+    properties described in METADATA_MATCH_TABLE, compare metadata if it is
+    present in base. Return true if metadata matches, or if base lacks some
+    data (either TEST_TYPE or the corresponding metadata)
+    """
+    test_type = base_configuration.get('TEST_TYPE')
+    if test_type not in METADATA_MATCH_TABLE:
+        return True
+    metadata_key = METADATA_MATCH_TABLE.get(test_type)
+    if target_configuration.get(metadata_key) != base_configuration.get(metadata_key):
+        return False
+    return True
+def machine_matches(base_configuration, target_configuration):
+    return base_configuration.get('MACHINE') == target_configuration.get('MACHINE')
+def can_be_compared(logger, base, target):
+    """
+    Some tests are not relevant to be compared, for example some oeselftest
+    run with different tests sets or parameters. Return true if tests can be
+    compared
+    """
+    ret = True
+    base_configuration = base['configuration']
+    target_configuration = target['configuration']
+    # Older test results lack proper OESELFTEST_METADATA: if not present, try to guess it based on tests results.
+    if base_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in base_configuration:
+        guess = guess_oeselftest_metadata(base['result'])
+        if guess is None:
+            logger.error(f"ERROR: did not manage to guess oeselftest metadata for {base_configuration['STARTTIME']}")
+        else:
+            logger.debug(f"Enriching {base_configuration['STARTTIME']} with {guess}")
+            base_configuration['OESELFTEST_METADATA'] = guess
+    if target_configuration.get('TEST_TYPE') == 'oeselftest' and 'OESELFTEST_METADATA' not in target_configuration:
+        guess = guess_oeselftest_metadata(target['result'])
+        if guess is None:
+            logger.error(f"ERROR: did not manage to guess oeselftest metadata for {target_configuration['STARTTIME']}")
+        else:
+            logger.debug(f"Enriching {target_configuration['STARTTIME']} with {guess}")
+            target_configuration['OESELFTEST_METADATA'] = guess
+    # Test runs with LTP results in should only be compared with other runs with LTP tests in them
+    if base_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in base['result']):
+        ret = target_configuration.get('TEST_TYPE') == 'runtime' and any(result.startswith("ltpresult") for result in target['result'])
+    return ret and metadata_matches(base_configuration, target_configuration) \
+        and machine_matches(base_configuration, target_configuration)
+def get_status_str(raw_status):
+    raw_status_lower = raw_status.lower() if raw_status else "None"
+    return STATUS_STRINGS.get(raw_status_lower, raw_status)
+def get_additional_info_line(new_pass_count, new_tests):
+    result=[]
+    if new_tests:
+        result.append(f'+{new_tests} test(s) present')
+    if new_pass_count:
+        result.append(f'+{new_pass_count} test(s) now passing')
+    if not result:
+        return ""
+    return '    -> ' + ', '.join(result) + '\n'
+def compare_result(logger, base_name, target_name, base_result, target_result, display_limit=None):
    base_result = base_result.get('result')
    target_result = target_result.get('result')
    result = {}
+    new_tests = 0
+    regressions = {}
+    resultstring = ""
+    new_tests = 0
+    new_pass_count = 0
+    display_limit = int(display_limit) if display_limit else REGRESSIONS_DISPLAY_LIMIT
    if base_result and target_result:
        for k in base_result:
+            if k in ['ptestresult.rawlogs', 'ptestresult.sections']:
+                continue
            base_testcase = base_result[k]
            base_status = base_testcase.get('status')
            if base_status:
@@ -27,12 +223,47 @@ def compare_result(logger, base_name, target_name, base_result, target_result):
                    result[k] = {'base': base_status, 'target': target_status}
            else:
                logger.error('Failed to retrieved base test case status: %s' % k)
+        # Also count new tests that were not present in base results: it
+        # could be newly added tests, but it could also highlights some tests
+        # renames or fixed faulty ptests
+        for k in target_result:
+            if k not in base_result:
+                new_tests += 1
    if result:
-        resultstring = "Regression: %s\n            %s\n" % (base_name, target_name)
+        new_pass_count = sum(test['target'] is not None and test['target'].startswith("PASS") for test in result.values())
-        for k in sorted(result):
+        # Print a regression report only if at least one test has a regression status (FAIL, SKIPPED, absent...)
-            resultstring += '    %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
+        if new_pass_count < len(result):
+            resultstring = "Regression:  %s\n             %s\n" % (base_name, target_name)
+            for k in sorted(result):
+                if not result[k]['target'] or not result[k]['target'].startswith("PASS"):
+                    # Differentiate each ptest kind when listing regressions
+                    key_parts = k.split('.')
+                    key = '.'.join(key_parts[:2]) if k.startswith('ptest') else key_parts[0]
+                    # Append new regression to corresponding test family
+                    regressions[key] = regressions.setdefault(key, []) + ['        %s: %s -> %s\n' % (k, get_status_str(result[k]['base']), get_status_str(result[k]['target']))]
+            resultstring += f"    Total: {sum([len(regressions[r]) for r in regressions])} new regression(s):\n"
+            for k in regressions:
+                resultstring += f"    {len(regressions[k])} regression(s) for {k}\n"
+                count_to_print=min([display_limit, len(regressions[k])]) if display_limit > 0 else len(regressions[k])
+                resultstring += ''.join(regressions[k][:count_to_print])
+                if count_to_print < len(regressions[k]):
+                    resultstring+='        [...]\n'
+            if new_pass_count > 0:
+                resultstring += f'    Additionally, {new_pass_count} previously failing test(s) is/are now passing\n'
+            if new_tests > 0:
+                resultstring += f'    Additionally, {new_tests} new test(s) is/are present\n'
+        else:
+            resultstring = "%s\n%s\n" % (base_name, target_name)
+            result = None
    else:
-        resultstring = "Match: %s\n       %s" % (base_name, target_name)
+        resultstring = "%s\n%s\n" % (base_name, target_name)
+    if not result:
+        additional_info = get_additional_info_line(new_pass_count, new_tests)
+        if additional_info:
+            resultstring += additional_info
    return result, resultstring
 def get_results(logger, source):
@@ -44,12 +275,38 @@ def regression(args, logger):
    regression_common(args, logger, base_results, target_results)
+# Some test case naming is poor and contains random strings, particularly lttng/babeltrace.
+# Truncating the test names works since they contain file and line number identifiers
+# which allows us to match them without the random components.
+def fixup_ptest_names(results, logger):
+    for r in results:
+        for i in results[r]:
+            tests = list(results[r][i]['result'].keys())
+            for test in tests:
+                new = None
+                if test.startswith(("ptestresult.lttng-tools.", "ptestresult.babeltrace.", "ptestresult.babeltrace2")) and "_-_" in test:
+                    new = test.split("_-_")[0]
+                elif test.startswith(("ptestresult.curl.")) and "__" in test:
+                    new = test.split("__")[0]
+                elif test.startswith(("ptestresult.dbus.")) and "__" in test:
+                    new = test.split("__")[0]
+                elif test.startswith("ptestresult.binutils") and "build-st-" in test:
+                    new = test.split(" ")[0]
+                elif test.startswith("ptestresult.gcc") and "/tmp/runtest." in test:
+                    new = ".".join(test.split(".")[:2])
+                if new:
+                    results[r][i]['result'][new] = results[r][i]['result'][test]
+                    del results[r][i]['result'][test]
 def regression_common(args, logger, base_results, target_results):
    if args.base_result_id:
        base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
    if args.target_result_id:
        target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
+    fixup_ptest_names(base_results, logger)
+    fixup_ptest_names(target_results, logger)
    matches = []
    regressions = []
    notfound = []
@@ -62,7 +319,9 @@ def regression_common(args, logger, base_results, target_results):
            # removing any pairs which match
            for c in base.copy():
                for b in target.copy():
-                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
+                    if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
+                        continue
+                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
                    if not res:
                        matches.append(resstr)
                        base.remove(c)
@@ -71,15 +330,18 @@ def regression_common(args, logger, base_results, target_results):
            # Should only now see regressions, we may not be able to match multiple pairs directly
            for c in base:
                for b in target:
-                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
+                    if not can_be_compared(logger, base_results[a][c], target_results[a][b]):
+                        continue
+                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b], args.limit)
                    if res:
                        regressions.append(resstr)
        else:
            notfound.append("%s not found in target" % a)
-    print("\n".join(sorted(matches)))
    print("\n".join(sorted(regressions)))
+    print("\n" + MISSING_TESTS_BANNER + "\n")
    print("\n".join(sorted(notfound)))
+    print("\n" + ADDITIONAL_DATA_BANNER + "\n")
+    print("\n".join(sorted(matches)))
    return 0
 def regression_git(args, logger):
@@ -162,6 +424,7 @@ def register_commands(subparsers):
                              help='(optional) filter the base results to this result ID')
    parser_build.add_argument('-t', '--target-result-id', default='',
                              help='(optional) filter the target results to this result ID')
+    parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes")
    parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
                                         description='regression analysis comparing base result set to target '
@@ -183,4 +446,5 @@ def register_commands(subparsers):
    parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
    parser_build.add_argument('--commit2', help="Revision to compare with")
    parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
+    parser_build.add_argument('-l', '--limit', default=REGRESSIONS_DISPLAY_LIMIT, help="Maximum number of changes to display per test. Can be set to 0 to print all changes")
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index f0ca50ebe2..1c100b00ab 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -176,7 +176,10 @@ class ResultsTextReport(object):
            vals['sort'] = line['testseries'] + "_" + line['result_id']
            vals['failed_testcases'] = line['failed_testcases']
            for k in cols:
-                vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
+                if total_tested:
+                    vals[k] = "%d (%s%%)" % (line[k], format(line[k] / total_tested * 100, '.0f'))
+                else:
+                    vals[k] = "0 (0%)"
            for k in maxlen:
                if k in vals and len(vals[k]) > maxlen[k]:
                    maxlen[k] = len(vals[k])
@@ -253,7 +256,7 @@ class ResultsTextReport(object):
                if selected_test_case_only:
                    print_selected_testcase_result(raw_results, selected_test_case_only)
                else:
-                    print(json.dumps(raw_results, sort_keys=True, indent=4))
+                    print(json.dumps(raw_results, sort_keys=True, indent=1))
            else:
                print('Could not find raw test result for %s' % raw_test)
            return 0
diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py
index 8917022d36..b8fc79a6ac 100644
--- a/scripts/lib/resulttool/resultutils.py
+++ b/scripts/lib/resulttool/resultutils.py
@@ -14,8 +14,11 @@ import scriptpath
 import copy
 import urllib.request
 import posixpath
+import logging
 scriptpath.add_oe_lib_path()
+logger = logging.getLogger('resulttool')
 flatten_map = {
    "oeselftest": [],
    "runtime": [],
@@ -31,13 +34,19 @@ regression_map = {
    "manual": ['TEST_TYPE', 'TEST_MODULE', 'IMAGE_BASENAME', 'MACHINE']
 }
 store_map = {
-    "oeselftest": ['TEST_TYPE'],
+    "oeselftest": ['TEST_TYPE', 'TESTSERIES', 'MACHINE'],
    "runtime": ['TEST_TYPE', 'DISTRO', 'MACHINE', 'IMAGE_BASENAME'],
    "sdk": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
    "sdkext": ['TEST_TYPE', 'MACHINE', 'SDKMACHINE', 'IMAGE_BASENAME'],
    "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME']
 }
+rawlog_sections = {
+    "ptestresult.rawlogs": "ptest",
+    "ltpresult.rawlogs": "ltp",
+    "ltpposixresult.rawlogs": "ltpposix"
+}
 def is_url(p):
    """
    Helper for determining if the given path is a URL
@@ -58,7 +67,11 @@ def append_resultsdata(results, f, configmap=store_map, configvars=extra_configv
            testseries = posixpath.basename(posixpath.dirname(url.path))
        else:
            with open(f, "r") as filedata:
-                data = json.load(filedata)
+                try:
+                    data = json.load(filedata)
+                except json.decoder.JSONDecodeError:
+                    print("Cannot decode {}. Possible corruption. Skipping.".format(f))
+                    data = ""
            testseries = os.path.basename(os.path.dirname(f))
    else:
        data = f
@@ -104,21 +117,57 @@ def filter_resultsdata(results, resultid):
                 newresults[r][i] = results[r][i]
    return newresults
-def strip_ptestresults(results):
+def strip_logs(results):
    newresults = copy.deepcopy(results)
-    #for a in newresults2:
-    #  newresults = newresults2[a]
    for res in newresults:
        if 'result' not in newresults[res]:
            continue
-        if 'ptestresult.rawlogs' in newresults[res]['result']:
+        for logtype in rawlog_sections:
-            del newresults[res]['result']['ptestresult.rawlogs']
+            if logtype in newresults[res]['result']:
+                del newresults[res]['result'][logtype]
        if 'ptestresult.sections' in newresults[res]['result']:
            for i in newresults[res]['result']['ptestresult.sections']:
                if 'log' in newresults[res]['result']['ptestresult.sections'][i]:
                    del newresults[res]['result']['ptestresult.sections'][i]['log']
    return newresults
+# For timing numbers, crazy amounts of precision don't make sense and just confuse
+# the logs. For numbers over 1, trim to 3 decimal places, for numbers less than 1,
+# trim to 4 significant digits
+def trim_durations(results):
+    for res in results:
+        if 'result' not in results[res]:
+            continue
+        for entry in results[res]['result']:
+            if 'duration' in results[res]['result'][entry]:
+                duration = results[res]['result'][entry]['duration']
+                if duration > 1:
+                    results[res]['result'][entry]['duration'] = float("%.3f" % duration)
+                elif duration < 1:
+                    results[res]['result'][entry]['duration'] = float("%.4g" % duration)
+    return results
+def handle_cleanups(results):
+    # Remove pointless path duplication from old format reproducibility results
+    for res2 in results:
+        try:
+            section = results[res2]['result']['reproducible']['files']
+            for pkgtype in section:
+                for filelist in section[pkgtype].copy():
+                    if section[pkgtype][filelist] and type(section[pkgtype][filelist][0]) == dict:
+                        newlist = []
+                        for entry in section[pkgtype][filelist]:
+                            newlist.append(entry["reference"].split("/./")[1])
+                        section[pkgtype][filelist] = newlist
+        except KeyError:
+            pass
+    # Remove pointless duplicate rawlogs data
+    try:
+        del results[res2]['result']['reproducible.rawlogs']
+    except KeyError:
+        pass
 def decode_log(logdata):
    if isinstance(logdata, str):
        return logdata
@@ -142,7 +191,7 @@ def generic_get_log(sectionname, results, section):
    return decode_log(ptest['log'])
 def ptestresult_get_log(results, section):
-    return generic_get_log('ptestresuls.sections', results, section)
+    return generic_get_log('ptestresult.sections', results, section)
 def generic_get_rawlogs(sectname, results):
    if sectname not in results:
@@ -151,9 +200,6 @@ def generic_get_rawlogs(sectname, results):
        return None
    return decode_log(results[sectname]['log'])
-def ptestresult_get_rawlogs(results):
-    return generic_get_rawlogs('ptestresult.rawlogs', results)
 def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, ptestlogs=False):
    for res in results:
        if res:
@@ -163,16 +209,20 @@ def save_resultsdata(results, destdir, fn="testresults.json", ptestjson=False, p
        os.makedirs(os.path.dirname(dst), exist_ok=True)
        resultsout = results[res]
        if not ptestjson:
-            resultsout = strip_ptestresults(results[res])
+            resultsout = strip_logs(results[res])
+        trim_durations(resultsout)
+        handle_cleanups(resultsout)
        with open(dst, 'w') as f:
-            f.write(json.dumps(resultsout, sort_keys=True, indent=4))
+            f.write(json.dumps(resultsout, sort_keys=True, indent=1))
        for res2 in results[res]:
            if ptestlogs and 'result' in results[res][res2]:
                seriesresults = results[res][res2]['result']
-                rawlogs = ptestresult_get_rawlogs(seriesresults)
+                for logtype in rawlog_sections:
-                if rawlogs is not None:
+                    logdata = generic_get_rawlogs(logtype, seriesresults)
-                    with open(dst.replace(fn, "ptest-raw.log"), "w+") as f:
+                    if logdata is not None:
-                        f.write(rawlogs)
+                        logger.info("Extracting " + rawlog_sections[logtype] + "-raw.log")
+                        with open(dst.replace(fn, rawlog_sections[logtype] + "-raw.log"), "w+") as f:
+                            f.write(logdata)
                if 'ptestresult.sections' in seriesresults:
                    for i in seriesresults['ptestresult.sections']:
                        sectionlog = ptestresult_get_log(seriesresults, i)
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py
index e0951f0a8f..b143334e69 100644
--- a/scripts/lib/resulttool/store.py
+++ b/scripts/lib/resulttool/store.py
@@ -65,18 +65,35 @@ def store(args, logger):
        for r in revisions:
            results = revisions[r]
+            if args.revision and r[0] != args.revision:
+                logger.info('skipping %s as non-matching' % r[0])
+                continue
            keywords = {'commit': r[0], 'branch': r[1], "commit_count": r[2]}
-            subprocess.check_call(["find", tempdir, "!", "-path", "./.git/*", "-delete"])
+            subprocess.check_call(["find", tempdir, "-name", "testresults.json", "!", "-path", "./.git/*", "-delete"])
            resultutils.save_resultsdata(results, tempdir, ptestlogs=True)
            logger.info('Storing test result into git repository %s' % args.git_dir)
-            gitarchive.gitarchive(tempdir, args.git_dir, False, False,
+            excludes = []
+            if args.logfile_archive:
+                excludes = ['*.log', "*.log.zst"]
+            tagname = gitarchive.gitarchive(tempdir, args.git_dir, False, False,
                                  "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}",
                                  False, "{branch}/{commit_count}-g{commit}/{tag_number}",
                                  'Test run #{tag_number} of {branch}:{commit}', '',
-                                  [], [], False, keywords, logger)
+                                  excludes, [], False, keywords, logger)
+            if args.logfile_archive:
+                logdir = args.logfile_archive + "/" + tagname
+                shutil.copytree(tempdir, logdir)
+                os.chmod(logdir, 0o755)
+                for root, dirs,  files in os.walk(logdir):
+                    for name in files:
+                        if not name.endswith(".log"):
+                            continue
+                        f = os.path.join(root, name)
+                        subprocess.run(["zstd", f, "--rm"], check=True, capture_output=True)
    finally:
        subprocess.check_call(["rm", "-rf",  tempdir])
@@ -102,3 +119,7 @@ def register_commands(subparsers):
                              help='add executed-by configuration to each result file')
    parser_build.add_argument('-t', '--extra-test-env', default='',
                              help='add extra test environment data to each result file configuration')
+    parser_build.add_argument('-r', '--revision', default='',
+                              help='only store data for the specified revision')
+    parser_build.add_argument('-l', '--logfile-archive', default='',
+                              help='directory to separately archive log files along with a copy of the results')
diff --git a/scripts/lib/scriptutils.py b/scripts/lib/scriptutils.py
index f92255d8dc..32e749dbb1 100644
--- a/scripts/lib/scriptutils.py
+++ b/scripts/lib/scriptutils.py
@@ -5,7 +5,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
-import argparse
 import glob
 import logging
 import os
@@ -18,13 +17,14 @@ import sys
 import tempfile
 import threading
 import importlib
-from importlib import machinery
+import importlib.machinery
+import importlib.util
 class KeepAliveStreamHandler(logging.StreamHandler):
    def __init__(self, keepalive=True, **kwargs):
        super().__init__(**kwargs)
        if keepalive is True:
-            keepalive = 5000 # default timeout
+            keepalive = 5000  # default timeout
        self._timeout = threading.Condition()
        self._stop = False
@@ -35,9 +35,9 @@ class KeepAliveStreamHandler(logging.StreamHandler):
                with self._timeout:
                    if not self._timeout.wait(keepalive):
                        self.emit(logging.LogRecord("keepalive", logging.INFO,
-                            None, None, "Keepalive message", None, None))
+                                                    None, None, "Keepalive message", None, None))
-        self._thread = threading.Thread(target = thread, daemon = True)
+        self._thread = threading.Thread(target=thread, daemon=True)
        self._thread.start()
    def close(self):
@@ -71,18 +71,19 @@ def logger_setup_color(logger, color='auto'):
    for handler in logger.handlers:
        if (isinstance(handler, logging.StreamHandler) and
-            isinstance(handler.formatter, BBLogFormatter)):
+                isinstance(handler.formatter, BBLogFormatter)):
            if color == 'always' or (color == 'auto' and handler.stream.isatty()):
                handler.formatter.enable_color()
 def load_plugins(logger, plugins, pluginpath):
    def load_plugin(name):
        logger.debug('Loading plugin %s' % name)
-        spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
+        spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath])
        if spec:
-            return spec.loader.load_module()
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
    def plugin_name(filename):
        return os.path.splitext(os.path.basename(filename))[0]
@@ -176,9 +177,15 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
                f.write('BB_STRICT_CHECKSUM = "ignore"\n')
                f.write('SRC_URI = "%s"\n' % srcuri)
                f.write('SRCREV = "%s"\n' % srcrev)
+                f.write('PV = "0.0+"\n')
                f.write('WORKDIR = "%s"\n' % tmpworkdir)
+                f.write('UNPACKDIR = "%s"\n' % destdir)
                # Set S out of the way so it doesn't get created under the workdir
-                f.write('S = "%s"\n' % os.path.join(tmpdir, 'emptysrc'))
+                s_dir = os.path.join(tmpdir, 'emptysrc')
+                bb.utils.mkdirhier(s_dir)
+                f.write('S = "%s"\n' % s_dir)
                if not mirrors:
                    # We do not need PREMIRRORS since we are almost certainly
                    # fetching new source rather than something that has already
@@ -215,7 +222,8 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
                pathvars = ['T', 'RECIPE_SYSROOT', 'RECIPE_SYSROOT_NATIVE']
                for pathvar in pathvars:
                    path = rd.getVar(pathvar)
-                    shutil.rmtree(path)
+                    if os.path.exists(path):
+                        shutil.rmtree(path)
        finally:
            if fetchrecipe:
                try:
@@ -229,10 +237,6 @@ def fetch_url(tinfoil, srcuri, srcrev, destdir, logger, preserve_tmp=False, mirr
                if e.errno != errno.ENOTEMPTY:
                    raise
-        bb.utils.mkdirhier(destdir)
-        for fn in os.listdir(tmpworkdir):
-            shutil.move(os.path.join(tmpworkdir, fn), destdir)
    finally:
        if not preserve_tmp:
            shutil.rmtree(tmpdir)
@@ -268,12 +272,3 @@ def is_src_url(param):
        return True
    return False
-def filter_src_subdirs(pth):
-    """
-    Filter out subdirectories of initial unpacked source trees that we do not care about.
-    Used by devtool and recipetool.
-    """
-    dirlist = os.listdir(pth)
-    filterout = ['git.indirectionsymlink', 'source-date-epoch']
-    dirlist = [x for x in dirlist if x not in filterout]
-    return dirlist
diff --git a/scripts/lib/wic/canned-wks/common.wks.inc b/scripts/lib/wic/canned-wks/common.wks.inc
index 4fd29fa8c1..4a440ddafe 100644
--- a/scripts/lib/wic/canned-wks/common.wks.inc
+++ b/scripts/lib/wic/canned-wks/common.wks.inc
@@ -1,3 +1,3 @@
 # This file is included into 3 canned wks files from this directory
-part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part /boot --source bootimg_pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --use-uuid --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024
+part / --source rootfs --use-uuid --fstype=ext4 --label platform --align 1024
diff --git a/scripts/lib/wic/canned-wks/directdisk-gpt.wks b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
index cf16c0c30b..cb640056f1 100644
--- a/scripts/lib/wic/canned-wks/directdisk-gpt.wks
+++ b/scripts/lib/wic/canned-wks/directdisk-gpt.wks
@@ -3,8 +3,8 @@
 # can directly dd to boot media.
-part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part /boot --source bootimg_pcbios --ondisk sda --label boot --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default" --label platform --align 1024 --use-uuid
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
 bootloader  --ptable gpt --timeout=0  --append="rootwait rootfstype=ext4 video=vesafb vga=0x318 console=tty0 console=ttyS0,115200n8"
diff --git a/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
index f61d941d6d..4fd1999ffb 100644
--- a/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
+++ b/scripts/lib/wic/canned-wks/directdisk-multi-rootfs.wks
@@ -15,7 +15,7 @@
 #
 #   - or any combinations of -r and --rootfs command line options
-part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+part /boot --source bootimg_pcbios --ondisk sda --label boot --active --align 1024
 part / --source rootfs --rootfs-dir=rootfs1 --ondisk sda --fstype=ext4 --label platform --align 1024
 part /rescue --source rootfs --rootfs-dir=rootfs2 --ondisk sda --fstype=ext4 --label secondary --align 1024
diff --git a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
index 7300e65e32..5211972955 100644
--- a/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
+++ b/scripts/lib/wic/canned-wks/efi-bootdisk.wks.in
@@ -1,3 +1,3 @@
 bootloader --ptable gpt
-part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.0
+part /boot --source rootfs --rootfs-dir=${IMAGE_ROOTFS}/boot --fstype=vfat --label boot --active --align 1024 --use-uuid --overhead-factor 1.2
 part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/
diff --git a/scripts/lib/wic/canned-wks/efi-uki-bootdisk.wks.in b/scripts/lib/wic/canned-wks/efi-uki-bootdisk.wks.in
new file mode 100644
index 0000000000..cac0fa32cd
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/efi-uki-bootdisk.wks.in
@@ -0,0 +1,3 @@
+bootloader --ptable gpt --timeout=5
+part /boot --source bootimg_efi --sourceparams="loader=${EFI_PROVIDER}" --label boot --active --align 1024 --use-uuid --part-name="ESP" --part-type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B --overhead-factor=1
+part / --source rootfs --fstype=ext4 --label root --align 1024 --exclude-path boot/
diff --git a/scripts/lib/wic/canned-wks/mkefidisk.wks b/scripts/lib/wic/canned-wks/mkefidisk.wks
index d1878e23e5..16dfe76dfe 100644
--- a/scripts/lib/wic/canned-wks/mkefidisk.wks
+++ b/scripts/lib/wic/canned-wks/mkefidisk.wks
@@ -2,10 +2,10 @@
 # long-description: Creates a partitioned EFI disk image that the user
 # can directly dd to boot media.
-part /boot --source bootimg-efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
+part /boot --source bootimg_efi --sourceparams="loader=grub-efi" --ondisk sda --label msdos --active --align 1024
-part / --source rootfs --ondisk sda --fstype=ext4 --mkfs-extraopts "-T default"  --label platform --align 1024 --use-uuid
+part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
 part swap --ondisk sda --size 44 --label swap1 --fstype=swap
-bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=ttyS0,115200 console=tty0"
+bootloader --ptable gpt --timeout=5 --append="rootfstype=ext4 console=${KERNEL_CONSOLE} console=tty0"
diff --git a/scripts/lib/wic/canned-wks/mkhybridiso.wks b/scripts/lib/wic/canned-wks/mkhybridiso.wks
index 48c5ac4791..c3a030e5b4 100644
--- a/scripts/lib/wic/canned-wks/mkhybridiso.wks
+++ b/scripts/lib/wic/canned-wks/mkhybridiso.wks
@@ -2,6 +2,6 @@
 # long-description: Creates an EFI and legacy bootable hybrid ISO image
 # which can be used on optical media as well as USB media.
-part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO
+part /boot --source isoimage_isohybrid --sourceparams="loader=grub-efi,image_name=HYBRID_ISO_IMG" --ondisk cd --label HYBRIDISO
 bootloader  --timeout=15  --append=""
diff --git a/scripts/lib/wic/canned-wks/qemuloongarch.wks b/scripts/lib/wic/canned-wks/qemuloongarch.wks
new file mode 100644
index 0000000000..8465c7a8c0
--- /dev/null
+++ b/scripts/lib/wic/canned-wks/qemuloongarch.wks
@@ -0,0 +1,3 @@
+# short-description: Create qcow2 image for LoongArch QEMU machines
+part / --source rootfs --fstype=ext4 --label root --align 4096 --size 5G
diff --git a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
index 22b45217f1..808997611a 100644
--- a/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
+++ b/scripts/lib/wic/canned-wks/qemux86-directdisk.wks
@@ -4,5 +4,5 @@
 include common.wks.inc
-bootloader  --timeout=0  --append="rw oprofile.timer=1 rootfstype=ext4 "
+bootloader  --timeout=0  --append="rw oprofile.timer=1 rootfstype=ext4 console=tty console=ttyS0 "
diff --git a/scripts/lib/wic/canned-wks/sdimage-bootpart.wks b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
index 63bc4dab6a..f9f8044f7d 100644
--- a/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
+++ b/scripts/lib/wic/canned-wks/sdimage-bootpart.wks
@@ -2,5 +2,5 @@
 # long-description: Creates a partitioned SD card image. Boot files
 # are located in the first vfat partition.
-part /boot --source bootimg-partition --ondisk mmcblk0 --fstype=vfat --label boot --active --align 4 --size 16
+part /boot --source bootimg_partition --ondisk mmcblk0 --fstype=vfat --label boot --active --align 4 --size 16
 part / --source rootfs --ondisk mmcblk0 --fstype=ext4 --label root --align 4
diff --git a/scripts/lib/wic/canned-wks/systemd-bootdisk.wks b/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
index 95d7b97a60..3fb2c0e35f 100644
--- a/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
+++ b/scripts/lib/wic/canned-wks/systemd-bootdisk.wks
@@ -2,7 +2,7 @@
 # long-description: Creates a partitioned EFI disk image that the user
 # can directly dd to boot media. The selected bootloader is systemd-boot.
-part /boot --source bootimg-efi --sourceparams="loader=systemd-boot" --ondisk sda --label msdos --active --align 1024 --use-uuid
+part /boot --source bootimg_efi --sourceparams="loader=systemd-boot" --ondisk sda --label msdos --active --align 1024 --use-uuid
 part / --source rootfs --ondisk sda --fstype=ext4 --label platform --align 1024 --use-uuid
diff --git a/scripts/lib/wic/engine.py b/scripts/lib/wic/engine.py
index 018815b966..b9e60cbe4e 100644
--- a/scripts/lib/wic/engine.py
+++ b/scripts/lib/wic/engine.py
@@ -19,10 +19,10 @@ import os
 import tempfile
 import json
 import subprocess
+import shutil
 import re
 from collections import namedtuple, OrderedDict
-from distutils.spawn import find_executable
 from wic import WicError
 from wic.filemap import sparse_copy
@@ -180,6 +180,8 @@ def wic_create(wks_file, rootfs_dir, bootimg_dir, kernel_dir,
        os.makedirs(options.outdir)
    pname = options.imager
+    # Don't support '-' in plugin names
+    pname = pname.replace("-", "_")
    plugin_class = PluginMgr.get_plugins('imager').get(pname)
    if not plugin_class:
        raise WicError('Unknown plugin: %s' % pname)
@@ -232,6 +234,16 @@ class Disk:
        self._psector_size = None
        self._ptable_format = None
+        # define sector size
+        sector_size_str = get_bitbake_var('WIC_SECTOR_SIZE')
+        if sector_size_str is not None:
+            try:
+                self.sector_size = int(sector_size_str)
+            except ValueError:
+                self.sector_size = None
+        else:
+            self.sector_size = None
        # find parted
        # read paths from $PATH environment variable
        # if it fails, use hardcoded paths
@@ -245,7 +257,7 @@ class Disk:
            for path in pathlist.split(':'):
                self.paths = "%s%s:%s" % (native_sysroot, path, self.paths)
-        self.parted = find_executable("parted", self.paths)
+        self.parted = shutil.which("parted", path=self.paths)
        if not self.parted:
            raise WicError("Can't find executable parted")
@@ -258,7 +270,13 @@ class Disk:
    def get_partitions(self):
        if self._partitions is None:
            self._partitions = OrderedDict()
-            out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
+            if self.sector_size is not None:
+                out = exec_cmd("export PARTED_SECTOR_SIZE=%d; %s -sm %s unit B print" % \
+                           (self.sector_size, self.parted, self.imagepath), True)
+            else:
+                out = exec_cmd("%s -sm %s unit B print" % (self.parted, self.imagepath))
            parttype = namedtuple("Part", "pnum start end size fstype")
            splitted = out.splitlines()
            # skip over possible errors in exec_cmd output
@@ -283,7 +301,7 @@ class Disk:
                    "resize2fs", "mkswap", "mkdosfs", "debugfs","blkid"):
            aname = "_%s" % name
            if aname not in self.__dict__:
-                setattr(self, aname, find_executable(name, self.paths))
+                setattr(self, aname, shutil.which(name, path=self.paths))
                if aname not in self.__dict__ or self.__dict__[aname] is None:
                    raise WicError("Can't find executable '{}'".format(name))
            return self.__dict__[aname]
@@ -359,7 +377,7 @@ class Disk:
        Remove files/dirs and their contents from the partition.
        This only applies to ext* partition.
        """
-        abs_path = re.sub('\/\/+', '/', path)
+        abs_path = re.sub(r'\/\/+', '/', path)
        cmd = "{} {} -wR 'rm \"{}\"'".format(self.debugfs,
                                            self._get_part_image(pnum),
                                            abs_path)
diff --git a/scripts/lib/wic/filemap.py b/scripts/lib/wic/filemap.py
index 4d9da28172..85b39d5d74 100644
--- a/scripts/lib/wic/filemap.py
+++ b/scripts/lib/wic/filemap.py
@@ -46,6 +46,13 @@ def get_block_size(file_obj):
            bsize = stat.st_blksize
        else:
            raise IOError("Unable to determine block size")
+    # The logic in this script only supports a maximum of a 4KB
+    # block size
+    max_block_size = 4 * 1024
+    if bsize > max_block_size:
+        bsize = max_block_size
    return bsize
 class ErrorNotSupp(Exception):
diff --git a/scripts/lib/wic/help.py b/scripts/lib/wic/help.py
index bd3a2b97df..2e3061f343 100644
--- a/scripts/lib/wic/help.py
+++ b/scripts/lib/wic/help.py
@@ -544,18 +544,18 @@ DESCRIPTION
    the --source param given to that partition.  For example, if the
    partition is set up like this:
-      part /boot --source bootimg-pcbios   ...
+      part /boot --source bootimg_pcbios   ...
    then the methods defined as class members of the plugin having the
-    matching bootimg-pcbios .name class member would be used.
+    matching bootimg_pcbios .name class member would be used.
    To be more concrete, here's the plugin definition that would match
-    a '--source bootimg-pcbios' usage, along with an example method
+    a '--source bootimg_pcbios' usage, along with an example method
    that would be called by the wic implementation when it needed to
    invoke an implementation-specific partition-preparation function:
    class BootimgPcbiosPlugin(SourcePlugin):
-        name = 'bootimg-pcbios'
+        name = 'bootimg_pcbios'
    @classmethod
        def do_prepare_partition(self, part, ...)
@@ -637,7 +637,7 @@ DESCRIPTION
    oe-core: directdisk.bbclass and mkefidisk.sh.  The difference
    between wic and those examples is that with wic the functionality
    of those scripts is implemented by a general-purpose partitioning
-    'language' based on Redhat kickstart syntax).
+    'language' based on Red Hat kickstart syntax).
    The initial motivation and design considerations that lead to the
    current tool are described exhaustively in Yocto Bug #3847
@@ -794,7 +794,7 @@ DESCRIPTION
     Here is a content of test.wks:
-     part /boot --source bootimg-pcbios --ondisk sda --label boot --active --align 1024
+     part /boot --source bootimg_pcbios --ondisk sda --label boot --active --align 1024
     part / --source rootfs --ondisk sda --fstype=ext3 --label platform --align 1024
     bootloader  --timeout=0  --append="rootwait rootfstype=ext3 video=vesafb vga=0x318 console=tty0"
@@ -840,8 +840,8 @@ DESCRIPTION
    meanings. The commands are based on the Fedora kickstart
    documentation but with modifications to reflect wic capabilities.
-      http://fedoraproject.org/wiki/Anaconda/Kickstart#part_or_partition
+      https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#part-or-partition
-      http://fedoraproject.org/wiki/Anaconda/Kickstart#bootloader
+      https://pykickstart.readthedocs.io/en/latest/kickstart-docs.html#bootloader
  Commands
@@ -916,6 +916,10 @@ DESCRIPTION
                   will create empty partition. --size parameter has
                   to be used to specify size of empty partition.
+         --sourceparams: This option is specific to wic. Supply additional
+                         parameters to the source plugin in
+                         key1=value1,key2 format.
         --ondisk or --ondrive: Forces the partition to be created on
                                a particular disk.
@@ -930,7 +934,9 @@ DESCRIPTION
             ext4
             btrfs
             squashfs
+             erofs
             swap
+             none
         --fsoptions: Specifies a free-form string of options to be
                      used when mounting the filesystem. This string
@@ -939,6 +945,12 @@ DESCRIPTION
                      quotes.  If not specified, the default string is
                      "defaults".
+         --fspassno: Specifies the order in which filesystem checks are done
+                     at boot time by fsck.  See fs_passno parameter of
+                     fstab(5).  This parameter will be copied into the
+                     /etc/fstab file of the installed system.  If not
+                     specified the default value of "0" will be used.
         --label label: Specifies the label to give to the filesystem
                        to be made on the partition. If the given
                        label is already in use by another filesystem,
@@ -958,6 +970,14 @@ DESCRIPTION
                              to start a partition on an x KBytes
                              boundary.
+         --offset: This option is specific to wic that says to place a partition
+                   at exactly the specified offset. If the partition cannot be
+                   placed at the specified offset, the image build will fail.
+                   Specify as an integer value optionally followed by one of the
+                   units s/S for 512 byte sector, k/K for kibibyte, M for
+                   mebibyte and G for gibibyte. The default unit if none is
+                   given is k.
         --no-table: This option is specific to wic. Space will be
                     reserved for the partition and it will be
                     populated but it will not be added to the
@@ -990,6 +1010,9 @@ DESCRIPTION
                             multiple partitions and we want to keep the right
                             permissions and usernames in all the partitions.
+         --no-fstab-update: This option is specific to wic. It does not update the
+                            '/etc/fstab' stock file for the given partition.
         --extra-space: This option is specific to wic. It adds extra
                        space after the space filled by the content
                        of the partition. The final size can go
@@ -1035,6 +1058,18 @@ DESCRIPTION
                           not take effect when --mkfs-extraopts is used. This should be taken into
                           account when using --mkfs-extraopts.
+         --type: This option is specific to wic. Valid values are 'primary',
+                 'logical'. For msdos partition tables, this option specifies
+                 the partition type.
+         --hidden: This option is specific to wic. This option sets the
+                   RequiredPartition bit (bit 0) on GPT partitions.
+         --mbr: This option is specific to wic. This option is used with the
+                gpt-hybrid partition type that uses both a GPT partition and
+                an MBR header. Partitions with this flag will be included in
+                this MBR header.
    * bootloader
      This command allows the user to specify various bootloader
@@ -1053,6 +1088,13 @@ DESCRIPTION
                      file. Using this option will override any other
                      bootloader option.
+        --ptable: Specifies the partition table format. Valid values are
+                  'msdos', 'gpt', 'gpt-hybrid'.
+        --source: Specifies the source plugin. If not specified, the
+                  --source value will be copied from the partition that has
+                  /boot as mountpoint.
      Note that bootloader functionality and boot partitions are
      implemented by the various --source plugins that implement
      bootloader functionality; the bootloader command essentially
@@ -1108,7 +1150,7 @@ COMMAND:
 TOPIC:
    overview  - Presents an overall overview of Wic
    plugins   - Presents an overview and API for Wic plugins
-    kickstart - Presents a Wic kicstart file reference
+    kickstart - Presents a Wic kickstart file reference
 Examples:
diff --git a/scripts/lib/wic/ksparser.py b/scripts/lib/wic/ksparser.py
index 3eb669da39..7ef3dc83dd 100644
--- a/scripts/lib/wic/ksparser.py
+++ b/scripts/lib/wic/ksparser.py
@@ -155,9 +155,11 @@ class KickStart():
        part.add_argument('--change-directory')
        part.add_argument("--extra-space", type=sizetype("M"))
        part.add_argument('--fsoptions', dest='fsopts')
+        part.add_argument('--fspassno', dest='fspassno')
        part.add_argument('--fstype', default='vfat',
                          choices=('ext2', 'ext3', 'ext4', 'btrfs',
-                                   'squashfs', 'vfat', 'msdos', 'swap'))
+                                   'squashfs', 'vfat', 'msdos', 'erofs',
+                                   'swap', 'none'))
        part.add_argument('--mkfs-extraopts', default='')
        part.add_argument('--label')
        part.add_argument('--use-label', action='store_true')
@@ -169,6 +171,7 @@ class KickStart():
        part.add_argument('--rootfs-dir')
        part.add_argument('--type', default='primary',
                choices = ('primary', 'logical'))
+        part.add_argument('--hidden', action='store_true')
        # --size and --fixed-size cannot be specified together; options
        # ----extra-space and --overhead-factor should also raise a parser
@@ -184,11 +187,13 @@ class KickStart():
        part.add_argument('--use-uuid', action='store_true')
        part.add_argument('--uuid')
        part.add_argument('--fsuuid')
+        part.add_argument('--no-fstab-update', action='store_true')
+        part.add_argument('--mbr', action='store_true')
        bootloader = subparsers.add_parser('bootloader')
        bootloader.add_argument('--append')
        bootloader.add_argument('--configfile')
-        bootloader.add_argument('--ptable', choices=('msdos', 'gpt'),
+        bootloader.add_argument('--ptable', choices=('msdos', 'gpt', 'gpt-hybrid'),
                                default='msdos')
        bootloader.add_argument('--timeout', type=int)
        bootloader.add_argument('--source')
@@ -229,6 +234,10 @@ class KickStart():
                                err = "%s:%d: SquashFS does not support LABEL" \
                                       % (confpath, lineno)
                                raise KickStartError(err)
+                        # erofs does not support filesystem labels
+                        if parsed.fstype == 'erofs' and parsed.label:
+                            err = "%s:%d: erofs does not support LABEL" % (confpath, lineno)
+                            raise KickStartError(err)
                        if parsed.fstype == 'msdos' or parsed.fstype == 'vfat':
                            if parsed.fsuuid:
                                if parsed.fsuuid.upper().startswith('0X'):
diff --git a/scripts/lib/wic/misc.py b/scripts/lib/wic/misc.py
index 57c042c503..1a7c140fa6 100644
--- a/scripts/lib/wic/misc.py
+++ b/scripts/lib/wic/misc.py
@@ -16,16 +16,16 @@ import logging
 import os
 import re
 import subprocess
+import shutil
 from collections import defaultdict
-from distutils import spawn
 from wic import WicError
 logger = logging.getLogger('wic')
 # executable -> recipe pairs for exec_native_cmd
-NATIVE_RECIPES = {"bmaptool": "bmap-tools",
+NATIVE_RECIPES = {"bmaptool": "bmaptool",
                  "dumpe2fs": "e2fsprogs",
                  "grub-mkimage": "grub-efi",
                  "isohybrid": "syslinux",
@@ -36,6 +36,7 @@ NATIVE_RECIPES = {"bmaptool": "bmap-tools",
                  "mkdosfs": "dosfstools",
                  "mkisofs": "cdrtools",
                  "mkfs.btrfs": "btrfs-tools",
+                  "mkfs.erofs": "erofs-utils",
                  "mkfs.ext2": "e2fsprogs",
                  "mkfs.ext3": "e2fsprogs",
                  "mkfs.ext4": "e2fsprogs",
@@ -122,7 +123,7 @@ def find_executable(cmd, paths):
    if provided and "%s-native" % recipe in provided:
        return True
-    return spawn.find_executable(cmd, paths)
+    return shutil.which(cmd, path=paths)
 def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
    """
@@ -140,11 +141,12 @@ def exec_native_cmd(cmd_and_args, native_sysroot, pseudo=""):
        cmd_and_args = pseudo + cmd_and_args
    hosttools_dir = get_bitbake_var("HOSTTOOLS_DIR")
+    target_sys = get_bitbake_var("TARGET_SYS")
-    native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/bin:%s" % \
+    native_paths = "%s/sbin:%s/usr/sbin:%s/usr/bin:%s/usr/bin/%s:%s/bin:%s" % \
                   (native_sysroot, native_sysroot,
-                    native_sysroot, native_sysroot,
+                    native_sysroot, native_sysroot, target_sys,
-                    hosttools_dir)
+                    native_sysroot, hosttools_dir)
    native_cmd_and_args = "export PATH=%s:$PATH;%s" % \
                   (native_paths, cmd_and_args)
diff --git a/scripts/lib/wic/partition.py b/scripts/lib/wic/partition.py
index 76d144d12d..b34691d313 100644
--- a/scripts/lib/wic/partition.py
+++ b/scripts/lib/wic/partition.py
@@ -33,6 +33,7 @@ class Partition():
        self.include_path = args.include_path
        self.change_directory = args.change_directory
        self.fsopts = args.fsopts
+        self.fspassno = args.fspassno
        self.fstype = args.fstype
        self.label = args.label
        self.use_label = args.use_label
@@ -54,9 +55,12 @@ class Partition():
        self.uuid = args.uuid
        self.fsuuid = args.fsuuid
        self.type = args.type
+        self.no_fstab_update = args.no_fstab_update
        self.updated_fstab_path = None
        self.has_fstab = False
        self.update_fstab_in_rootfs = False
+        self.hidden = args.hidden
+        self.mbr = args.mbr
        self.lineno = lineno
        self.source_file = ""
@@ -104,7 +108,7 @@ class Partition():
                extra_blocks = self.extra_space
            rootfs_size = actual_rootfs_size + extra_blocks
-            rootfs_size *= self.overhead_factor
+            rootfs_size = int(rootfs_size * self.overhead_factor)
            logger.debug("Added %d extra blocks to %s to get to %d total blocks",
                         extra_blocks, self.mountpoint, rootfs_size)
@@ -131,6 +135,8 @@ class Partition():
            self.update_fstab_in_rootfs = True
        if not self.source:
+            if self.fstype == "none" or self.no_table:
+                return
            if not self.size and not self.fixed_size:
                raise WicError("The %s partition has a size of zero. Please "
                               "specify a non-zero --size/--fixed-size for that "
@@ -141,9 +147,9 @@ class Partition():
                                            native_sysroot)
                self.source_file = "%s/fs.%s" % (cr_workdir, self.fstype)
            else:
-                if self.fstype == 'squashfs':
+                if self.fstype in ('squashfs', 'erofs'):
-                    raise WicError("It's not possible to create empty squashfs "
+                    raise WicError("It's not possible to create empty %s "
-                                   "partition '%s'" % (self.mountpoint))
+                                   "partition '%s'" % (self.fstype, self.mountpoint))
                rootfs = "%s/fs_%s.%s.%s" % (cr_workdir, self.label,
                                             self.lineno, self.fstype)
@@ -158,6 +164,9 @@ class Partition():
        plugins = PluginMgr.get_plugins('source')
+        # Don't support '-' in plugin names
+        self.source = self.source.replace("-", "_")
        if self.source not in plugins:
            raise WicError("The '%s' --source specified for %s doesn't exist.\n\t"
                           "See 'wic list source-plugins' for a list of available"
@@ -170,9 +179,9 @@ class Partition():
            # Split sourceparams string of the form key1=val1[,key2=val2,...]
            # into a dict.  Also accepts valueless keys i.e. without =
            splitted = self.sourceparams.split(',')
-            srcparams_dict = dict(par.split('=', 1) for par in splitted if par)
+            srcparams_dict = dict((par.split('=', 1) + [None])[:2] for par in splitted if par)
-        plugin = PluginMgr.get_plugins('source')[self.source]
+        plugin = plugins[self.source]
        plugin.do_configure_partition(self, srcparams_dict, creator,
                                      cr_workdir, oe_builddir, bootimg_dir,
                                      kernel_dir, native_sysroot)
@@ -216,19 +225,19 @@ class Partition():
        if (pseudo_dir):
            # Canonicalize the ignore paths. This corresponds to
            # calling oe.path.canonicalize(), which is used in bitbake.conf.
-            ignore_paths = [rootfs] + (get_bitbake_var("PSEUDO_IGNORE_PATHS") or "").split(",")
+            include_paths = [rootfs_dir] + (get_bitbake_var("PSEUDO_INCLUDE_PATHS") or "").split(",")
            canonical_paths = []
-            for path in ignore_paths:
+            for path in include_paths:
                if "$" not in path:
                    trailing_slash = path.endswith("/") and "/" or ""
                    canonical_paths.append(os.path.realpath(path) + trailing_slash)
-            ignore_paths = ",".join(canonical_paths)
+            include_paths = ",".join(canonical_paths)
            pseudo = "export PSEUDO_PREFIX=%s;" % p_prefix
            pseudo += "export PSEUDO_LOCALSTATEDIR=%s;" % pseudo_dir
            pseudo += "export PSEUDO_PASSWD=%s;" % rootfs_dir
            pseudo += "export PSEUDO_NOSYMLINKEXP=1;"
-            pseudo += "export PSEUDO_IGNORE_PATHS=%s;" % ignore_paths
+            pseudo += "export PSEUDO_INCLUDE_PATHS=%s;" % include_paths
            pseudo += "%s " % get_bitbake_var("FAKEROOTCMD")
        else:
            pseudo = None
@@ -238,7 +247,7 @@ class Partition():
            # from bitbake variable
            rsize_bb = get_bitbake_var('ROOTFS_SIZE')
            rdir = get_bitbake_var('IMAGE_ROOTFS')
-            if rsize_bb and rdir == rootfs_dir:
+            if rsize_bb and (rdir == rootfs_dir or (rootfs_dir.split('/')[-2] == "tmp-wic" and rootfs_dir.split('/')[-1][:6] == "rootfs")):
                # Bitbake variable ROOTFS_SIZE is calculated in
                # Image._get_rootfs_size method from meta/lib/oe/image.py
                # using IMAGE_ROOTFS_SIZE, IMAGE_ROOTFS_ALIGNMENT,
@@ -278,6 +287,9 @@ class Partition():
        extraopts = self.mkfs_extraopts or "-F -i 8192"
+        # use hash_seed to generate reproducible ext4 images
+        (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, pseudo)
        label_str = ""
        if self.label:
            label_str = "-L %s" % self.label
@@ -286,7 +298,7 @@ class Partition():
            (self.fstype, extraopts, rootfs, label_str, self.fsuuid, rootfs_dir)
        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
-        if self.updated_fstab_path and self.has_fstab:
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
            debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
            with open(debugfs_script_path, "w") as f:
                f.write("cd etc\n")
@@ -298,8 +310,49 @@ class Partition():
        mkfs_cmd = "fsck.%s -pvfD %s" % (self.fstype, rootfs)
        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            sde_time = hex(int(os.getenv('SOURCE_DATE_EPOCH')))
+            debugfs_script_path = os.path.join(cr_workdir, "debugfs_script")
+            files = []
+            for root, dirs, others in os.walk(rootfs_dir):
+                base = root.replace(rootfs_dir, "").rstrip(os.sep)
+                files += [ "/" if base == "" else base ]
+                files += [ base + "/" + n for n in dirs + others ]
+            with open(debugfs_script_path, "w") as f:
+                f.write("set_current_time %s\n" % (sde_time))
+                if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
+                    f.write("set_inode_field /etc/fstab mtime %s\n" % (sde_time))
+                    f.write("set_inode_field /etc/fstab mtime_extra 0\n")
+                for file in set(files):
+                    for time in ["atime", "ctime", "crtime"]:
+                        f.write("set_inode_field \"%s\" %s %s\n" % (file, time, sde_time))
+                        f.write("set_inode_field \"%s\" %s_extra 0\n" % (file, time))
+                for time in ["wtime", "mkfs_time", "lastcheck"]:
+                    f.write("set_super_value %s %s\n" % (time, sde_time))
+                for time in ["mtime", "first_error_time", "last_error_time"]:
+                    f.write("set_super_value %s 0\n" % (time))
+            debugfs_cmd = "debugfs -w -f %s %s" % (debugfs_script_path, rootfs)
+            exec_native_cmd(debugfs_cmd, native_sysroot)
        self.check_for_Y2038_problem(rootfs, native_sysroot)
+    def get_hash_seed_ext4(self, extraopts, pseudo):
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            sde_time = int(os.getenv('SOURCE_DATE_EPOCH'))
+            if pseudo:
+                pseudo = "export E2FSPROGS_FAKE_TIME=%s;%s " % (sde_time, pseudo)
+            else:
+                pseudo = "export E2FSPROGS_FAKE_TIME=%s; " % sde_time
+            # Set hash_seed to generate deterministic directory indexes
+            namespace = uuid.UUID("e7429877-e7b3-4a68-a5c9-2f2fdf33d460")
+            if self.fsuuid:
+                namespace = uuid.UUID(self.fsuuid)
+            hash_seed = str(uuid.uuid5(namespace, str(sde_time)))
+            extraopts += " -E hash_seed=%s" % hash_seed
+        return (extraopts, pseudo)
    def prepare_rootfs_btrfs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
                             native_sysroot, pseudo):
        """
@@ -350,8 +403,8 @@ class Partition():
        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (rootfs, rootfs_dir)
        exec_native_cmd(mcopy_cmd, native_sysroot)
-        if self.updated_fstab_path and self.has_fstab:
+        if self.updated_fstab_path and self.has_fstab and not self.no_fstab_update:
-            mcopy_cmd = "mcopy -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
+            mcopy_cmd = "mcopy -m -i %s %s ::/etc/fstab" % (rootfs, self.updated_fstab_path)
            exec_native_cmd(mcopy_cmd, native_sysroot)
        chmod_cmd = "chmod 644 %s" % rootfs
@@ -369,6 +422,19 @@ class Partition():
                       (rootfs_dir, rootfs, extraopts)
        exec_native_cmd(squashfs_cmd, native_sysroot, pseudo=pseudo)
+    def prepare_rootfs_erofs(self, rootfs, cr_workdir, oe_builddir, rootfs_dir,
+                             native_sysroot, pseudo):
+        """
+        Prepare content for a erofs rootfs partition.
+        """
+        extraopts = self.mkfs_extraopts or ''
+        erofs_cmd = "mkfs.erofs %s -U %s %s %s" % \
+                       (extraopts, self.fsuuid, rootfs, rootfs_dir)
+        exec_native_cmd(erofs_cmd, native_sysroot, pseudo=pseudo)
+    def prepare_empty_partition_none(self, rootfs, oe_builddir, native_sysroot):
+        pass
    def prepare_empty_partition_ext(self, rootfs, oe_builddir,
                                    native_sysroot):
        """
@@ -380,13 +446,16 @@ class Partition():
        extraopts = self.mkfs_extraopts or "-i 8192"
+        # use hash_seed to generate reproducible ext4 images
+        (extraopts, pseudo) = self.get_hash_seed_ext4(extraopts, None)
        label_str = ""
        if self.label:
            label_str = "-L %s" % self.label
        mkfs_cmd = "mkfs.%s -F %s %s -U %s %s" % \
            (self.fstype, extraopts, label_str, self.fsuuid, rootfs)
-        exec_native_cmd(mkfs_cmd, native_sysroot)
+        exec_native_cmd(mkfs_cmd, native_sysroot, pseudo=pseudo)
        self.check_for_Y2038_problem(rootfs, native_sysroot)
diff --git a/scripts/lib/wic/pluginbase.py b/scripts/lib/wic/pluginbase.py
index d9b4e57747..640da292d3 100644
--- a/scripts/lib/wic/pluginbase.py
+++ b/scripts/lib/wic/pluginbase.py
@@ -9,9 +9,11 @@ __all__ = ['ImagerPlugin', 'SourcePlugin']
 import os
 import logging
+import types
 from collections import defaultdict
-from importlib.machinery import SourceFileLoader
+import importlib
+import importlib.util
 from wic import WicError
 from wic.misc import get_bitbake_var
@@ -42,7 +44,7 @@ class PluginMgr:
                    path = os.path.join(layer_path, script_plugin_dir)
                    path = os.path.abspath(os.path.expanduser(path))
                    if path not in cls._plugin_dirs and os.path.isdir(path):
-                        cls._plugin_dirs.insert(0, path)
+                        cls._plugin_dirs.append(path)
        if ptype not in PLUGINS:
            # load all ptype plugins
@@ -54,7 +56,9 @@ class PluginMgr:
                            mname = fname[:-3]
                            mpath = os.path.join(ppath, fname)
                            logger.debug("loading plugin module %s", mpath)
-                            SourceFileLoader(mname, mpath).load_module()
+                            spec = importlib.util.spec_from_file_location(mname, mpath)
+                            module = importlib.util.module_from_spec(spec)
+                            spec.loader.exec_module(module)
        return PLUGINS.get(ptype)
diff --git a/scripts/lib/wic/plugins/imager/direct.py b/scripts/lib/wic/plugins/imager/direct.py
index ea709e8c54..6e1f1c8cba 100644
--- a/scripts/lib/wic/plugins/imager/direct.py
+++ b/scripts/lib/wic/plugins/imager/direct.py
@@ -77,7 +77,8 @@ class DirectPlugin(ImagerPlugin):
        image_path = self._full_path(self.workdir, self.parts[0].disk, "direct")
        self._image = PartitionedImage(image_path, self.ptable_format,
-                                       self.parts, self.native_sysroot)
+                                       self.parts, self.native_sysroot,
+                                       options.extra_space)
    def setup_workdir(self, workdir):
        if workdir:
@@ -116,7 +117,7 @@ class DirectPlugin(ImagerPlugin):
        updated = False
        for part in self.parts:
            if not part.realnum or not part.mountpoint \
-               or part.mountpoint == "/":
+               or part.mountpoint == "/" or not (part.mountpoint.startswith('/') or part.mountpoint == "swap"):
                continue
            if part.use_uuid:
@@ -137,8 +138,9 @@ class DirectPlugin(ImagerPlugin):
                device_name = "/dev/%s%s%d" % (part.disk, prefix, part.realnum)
            opts = part.fsopts if part.fsopts else "defaults"
+            passno = part.fspassno if part.fspassno else "0"
            line = "\t".join([device_name, part.mountpoint, part.fstype,
-                              opts, "0", "0"]) + "\n"
+                              opts, "0", passno]) + "\n"
            fstab_lines.append(line)
            updated = True
@@ -147,6 +149,9 @@ class DirectPlugin(ImagerPlugin):
            self.updated_fstab_path = os.path.join(self.workdir, "fstab")
            with open(self.updated_fstab_path, "w") as f:
                f.writelines(fstab_lines)
+            if os.getenv('SOURCE_DATE_EPOCH'):
+                fstab_time = int(os.getenv('SOURCE_DATE_EPOCH'))
+                os.utime(self.updated_fstab_path, (fstab_time, fstab_time))
    def _full_path(self, path, name, extention):
        """ Construct full file path to a file we generate. """
@@ -198,6 +203,8 @@ class DirectPlugin(ImagerPlugin):
        source_plugin = self.ks.bootloader.source
        disk_name = self.parts[0].disk
        if source_plugin:
+            # Don't support '-' in plugin names
+            source_plugin = source_plugin.replace("-", "_")
            plugin = PluginMgr.get_plugins('source')[source_plugin]
            plugin.do_install_disk(self._image, disk_name, self, self.workdir,
                                   self.oe_builddir, self.bootimg_dir,
@@ -258,6 +265,8 @@ class DirectPlugin(ImagerPlugin):
            if part.mountpoint == "/":
                if part.uuid:
                    return "PARTUUID=%s" % part.uuid
+                elif part.label and self.ptable_format != 'msdos':
+                    return "PARTLABEL=%s" % part.label
                else:
                    suffix = 'p' if part.disk.startswith('mmcblk') else ''
                    return "/dev/%s%s%-d" % (part.disk, suffix, part.realnum)
@@ -293,7 +302,7 @@ class PartitionedImage():
    Partitioned image in a file.
    """
-    def __init__(self, path, ptable_format, partitions, native_sysroot=None):
+    def __init__(self, path, ptable_format, partitions, native_sysroot=None, extra_space=0):
        self.path = path  # Path to the image file
        self.numpart = 0  # Number of allocated partitions
        self.realpart = 0 # Number of partitions in the partition table
@@ -306,14 +315,26 @@ class PartitionedImage():
                          # all partitions (in bytes)
        self.ptable_format = ptable_format  # Partition table format
        # Disk system identifier
-        self.identifier = random.SystemRandom().randint(1, 0xffffffff)
+        if os.getenv('SOURCE_DATE_EPOCH'):
+            self.identifier = random.Random(int(os.getenv('SOURCE_DATE_EPOCH'))).randint(1, 0xffffffff)
+        else:
+            self.identifier = random.SystemRandom().randint(1, 0xffffffff)
        self.partitions = partitions
        self.partimages = []
        # Size of a sector used in calculations
-        self.sector_size = SECTOR_SIZE
+        sector_size_str = get_bitbake_var('WIC_SECTOR_SIZE')
+        if sector_size_str is not None:
+            try:
+                self.sector_size = int(sector_size_str)
+            except ValueError:
+                self.sector_size = SECTOR_SIZE
+        else:
+            self.sector_size = SECTOR_SIZE
        self.native_sysroot = native_sysroot
        num_real_partitions = len([p for p in self.partitions if not p.no_table])
+        self.extra_space = extra_space
        # calculate the real partition number, accounting for partitions not
        # in the partition table and logical partitions
@@ -331,7 +352,7 @@ class PartitionedImage():
        # generate parition and filesystem UUIDs
        for part in self.partitions:
            if not part.uuid and part.use_uuid:
-                if self.ptable_format == 'gpt':
+                if self.ptable_format in ('gpt', 'gpt-hybrid'):
                    part.uuid = str(uuid.uuid4())
                else: # msdos partition table
                    part.uuid = '%08x-%02d' % (self.identifier, part.realnum)
@@ -387,6 +408,10 @@ class PartitionedImage():
                raise WicError("setting custom partition type is not " \
                               "implemented for msdos partitions")
+            if part.mbr and self.ptable_format != 'gpt-hybrid':
+                raise WicError("Partition may only be included in MBR with " \
+                               "a gpt-hybrid partition table")
            # Get the disk where the partition is located
            self.numpart += 1
            if not part.no_table:
@@ -395,7 +420,7 @@ class PartitionedImage():
            if self.numpart == 1:
                if self.ptable_format == "msdos":
                    overhead = MBR_OVERHEAD
-                elif self.ptable_format == "gpt":
+                elif self.ptable_format in ("gpt", "gpt-hybrid"):
                    overhead = GPT_OVERHEAD
                # Skip one sector required for the partitioning scheme overhead
@@ -479,10 +504,11 @@ class PartitionedImage():
        # Once all the partitions have been layed out, we can calculate the
        # minumim disk size
        self.min_size = self.offset
-        if self.ptable_format == "gpt":
+        if self.ptable_format in ("gpt", "gpt-hybrid"):
            self.min_size += GPT_OVERHEAD
        self.min_size *= self.sector_size
+        self.min_size += self.extra_space
    def _create_partition(self, device, parttype, fstype, start, size):
        """ Create a partition on an image described by the 'device' object. """
@@ -492,29 +518,58 @@ class PartitionedImage():
        logger.debug("Added '%s' partition, sectors %d-%d, size %d sectors",
                     parttype, start, end, size)
-        cmd = "parted -s %s unit s mkpart %s" % (device, parttype)
+        cmd = "export PARTED_SECTOR_SIZE=%d; parted -s %s unit s mkpart %s" % \
+                     (self.sector_size, device, parttype)
        if fstype:
            cmd += " %s" % fstype
        cmd += " %d %d" % (start, end)
        return exec_native_cmd(cmd, self.native_sysroot)
+    def _write_identifier(self, device, identifier):
+        logger.debug("Set disk identifier %x", identifier)
+        with open(device, 'r+b') as img:
+            img.seek(0x1B8)
+            img.write(identifier.to_bytes(4, 'little'))
+    def _make_disk(self, device, ptable_format, min_size):
+        logger.debug("Creating sparse file %s", device)
+        with open(device, 'w') as sparse:
+            os.ftruncate(sparse.fileno(), min_size)
+        logger.debug("Initializing partition table for %s", device)
+        exec_native_cmd("export PARTED_SECTOR_SIZE=%d; parted -s %s mklabel %s" %
+                        (self.sector_size, device, ptable_format), self.native_sysroot)
+    def _write_disk_guid(self):
+        if self.ptable_format in ('gpt', 'gpt-hybrid'):
+            if os.getenv('SOURCE_DATE_EPOCH'):
+                self.disk_guid = uuid.UUID(int=int(os.getenv('SOURCE_DATE_EPOCH')))
+            else:
+                self.disk_guid = uuid.uuid4()
+            logger.debug("Set disk guid %s", self.disk_guid)
+            sfdisk_cmd = "sfdisk --sector-size %s --disk-id %s %s" % \
+                        (self.sector_size, self.path, self.disk_guid)
+            exec_native_cmd(sfdisk_cmd, self.native_sysroot)
    def create(self):
-        logger.debug("Creating sparse file %s", self.path)
+        self._make_disk(self.path,
-        with open(self.path, 'w') as sparse:
+                        "gpt" if self.ptable_format == "gpt-hybrid" else self.ptable_format,
-            os.ftruncate(sparse.fileno(), self.min_size)
+                        self.min_size)
-        logger.debug("Initializing partition table for %s", self.path)
+        self._write_identifier(self.path, self.identifier)
-        exec_native_cmd("parted -s %s mklabel %s" %
+        self._write_disk_guid()
-                        (self.path, self.ptable_format), self.native_sysroot)
-        logger.debug("Set disk identifier %x", self.identifier)
+        if self.ptable_format == "gpt-hybrid":
-        with open(self.path, 'r+b') as img:
+            mbr_path = self.path + ".mbr"
-            img.seek(0x1B8)
+            self._make_disk(mbr_path, "msdos", self.min_size)
-            img.write(self.identifier.to_bytes(4, 'little'))
+            self._write_identifier(mbr_path, self.identifier)
        logger.debug("Creating partitions")
+        hybrid_mbr_part_num = 0
        for part in self.partitions:
            if part.num == 0:
                continue
@@ -559,46 +614,77 @@ class PartitionedImage():
            self._create_partition(self.path, part.type,
                                   parted_fs_type, part.start, part.size_sec)
-            if part.part_name:
+            if self.ptable_format == "gpt-hybrid" and part.mbr:
-                logger.debug("partition %d: set name to %s",
+                hybrid_mbr_part_num += 1
-                             part.num, part.part_name)
+                if hybrid_mbr_part_num > 4:
-                exec_native_cmd("sgdisk --change-name=%d:%s %s" % \
+                    raise WicError("Extended MBR partitions are not supported in hybrid MBR")
-                                         (part.num, part.part_name,
+                self._create_partition(mbr_path, "primary",
-                                          self.path), self.native_sysroot)
+                                       parted_fs_type, part.start, part.size_sec)
+            if self.ptable_format in ("gpt", "gpt-hybrid") and (part.part_name or part.label):
+                partition_label = part.part_name if part.part_name else part.label
+                logger.debug("partition %d: set name to %s",
+                             part.num, partition_label)
+                exec_native_cmd("sfdisk --sector-size %s --part-label %s %d %s" % \
+                                         (self.sector_size, self.path, part.num,
+                                          partition_label), self.native_sysroot)
            if part.part_type:
                logger.debug("partition %d: set type UID to %s",
                             part.num, part.part_type)
-                exec_native_cmd("sgdisk --typecode=%d:%s %s" % \
+                exec_native_cmd("sfdisk --sector-size %s --part-type %s %d %s" % \
-                                         (part.num, part.part_type,
+                                         (self.sector_size, self.path, part.num,
-                                          self.path), self.native_sysroot)
+                                          part.part_type), self.native_sysroot)
-            if part.uuid and self.ptable_format == "gpt":
+            if part.uuid and self.ptable_format in ("gpt", "gpt-hybrid"):
                logger.debug("partition %d: set UUID to %s",
                             part.num, part.uuid)
-                exec_native_cmd("sgdisk --partition-guid=%d:%s %s" % \
+                exec_native_cmd("sfdisk --sector-size %s --part-uuid %s %d %s" % \
-                                (part.num, part.uuid, self.path),
+                                (self.sector_size, self.path, part.num, part.uuid),
-                                self.native_sysroot)
-            if part.label and self.ptable_format == "gpt":
-                logger.debug("partition %d: set name to %s",
-                             part.num, part.label)
-                exec_native_cmd("parted -s %s name %d %s" % \
-                                (self.path, part.num, part.label),
                                self.native_sysroot)
            if part.active:
-                flag_name = "legacy_boot" if self.ptable_format == 'gpt' else "boot"
+                flag_name = "legacy_boot" if self.ptable_format in ('gpt', 'gpt-hybrid') else "boot"
                logger.debug("Set '%s' flag for partition '%s' on disk '%s'",
                             flag_name, part.num, self.path)
-                exec_native_cmd("parted -s %s set %d %s on" % \
+                exec_native_cmd("export PARTED_SECTOR_SIZE=%d; parted -s %s set %d %s on" % \
-                                (self.path, part.num, flag_name),
+                                (self.sector_size, self.path, part.num, flag_name),
                                self.native_sysroot)
+                if self.ptable_format == 'gpt-hybrid' and part.mbr:
+                    exec_native_cmd("export PARTED_SECTOR_SIZE=%d; parted -s %s set %d %s on" % \
+                                    (self.sector_size, mbr_path, hybrid_mbr_part_num, "boot"),
+                                    self.native_sysroot)
            if part.system_id:
-                exec_native_cmd("sfdisk --part-type %s %s %s" % \
+                exec_native_cmd("sfdisk --sector-size %s --part-type %s %s %s" % \
-                                (self.path, part.num, part.system_id),
+                                (self.sector_size, self.path, part.num, part.system_id),
                                self.native_sysroot)
+            if part.hidden and self.ptable_format == "gpt":
+                logger.debug("Set hidden attribute for partition '%s' on disk '%s'",
+                             part.num, self.path)
+                exec_native_cmd("sfdisk --sector-size %s --part-attrs %s %s RequiredPartition" % \
+                                (self.sector_size, self.path, part.num),
+                                self.native_sysroot)
+        if self.ptable_format == "gpt-hybrid":
+            # Write a protective GPT partition
+            hybrid_mbr_part_num += 1
+            if hybrid_mbr_part_num > 4:
+                raise WicError("Extended MBR partitions are not supported in hybrid MBR")
+            # parted cannot directly create a protective GPT partition, so
+            # create with an arbitrary type, then change it to the correct type
+            # with sfdisk
+            self._create_partition(mbr_path, "primary", "fat32", 1, GPT_OVERHEAD)
+            exec_native_cmd("sfdisk --sector-size %s --part-type %s %d 0xee" % \
+                            (self.sector_size, mbr_path, hybrid_mbr_part_num),
+                            self.native_sysroot)
+            # Copy hybrid MBR
+            with open(mbr_path, "rb") as mbr_file:
+                with open(self.path, "r+b") as image_file:
+                    mbr = mbr_file.read(512)
+                    image_file.write(mbr)
    def cleanup(self):
        pass
diff --git a/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py b/scripts/lib/wic/plugins/source/bootimg_biosplusefi.py
index 5bd7390680..4279ddded8 100644
--- a/scripts/lib/wic/plugins/source/bootimg-biosplusefi.py
+++ b/scripts/lib/wic/plugins/source/bootimg_biosplusefi.py
@@ -13,7 +13,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # DESCRIPTION
-# This implements the 'bootimg-biosplusefi' source plugin class for 'wic'
+# This implements the 'bootimg_biosplusefi' source plugin class for 'wic'
 #
 # AUTHORS
 # William Bourque <wbourque [at) gmail.com>
@@ -34,7 +34,7 @@ class BootimgBiosPlusEFIPlugin(SourcePlugin):
    Note it is possible to create an image that can boot from both
    legacy BIOS and EFI by defining two partitions : one with arg
-    --source bootimg-efi  and another one with --source bootimg-pcbios.
+    --source bootimg_efi  and another one with --source bootimg_pcbios.
    However, this method has the obvious downside that it requires TWO
    partitions to be created on the storage device.
    Both partitions will also be marked as "bootable" which does not work on
@@ -45,7 +45,7 @@ class BootimgBiosPlusEFIPlugin(SourcePlugin):
    the first partition will be duplicated into the second, even though it
    will not be used at all.
-    Also, unlike "isoimage-isohybrid" that also does BIOS and EFI, this plugin
+    Also, unlike "isoimage_isohybrid" that also does BIOS and EFI, this plugin
    allows you to have more than only a single rootfs partitions and does
    not turn the rootfs into an initramfs RAM image.
@@ -53,32 +53,32 @@ class BootimgBiosPlusEFIPlugin(SourcePlugin):
    does not have the limitations listed above.
    The plugin is made so it does tries not to reimplement what's already
-    been done in other plugins; as such it imports "bootimg-pcbios"
+    been done in other plugins; as such it imports "bootimg_pcbios"
-    and "bootimg-efi".
+    and "bootimg_efi".
-    Plugin "bootimg-pcbios" is used to generate legacy BIOS boot.
+    Plugin "bootimg_pcbios" is used to generate legacy BIOS boot.
-    Plugin "bootimg-efi" is used to generate the UEFI boot. Note that it
+    Plugin "bootimg_efi" is used to generate the UEFI boot. Note that it
    requires a --sourceparams argument to know which loader to use; refer
-    to "bootimg-efi" code/documentation for the list of loader.
+    to "bootimg_efi" code/documentation for the list of loader.
    Imports are handled with "SourceFileLoader" from importlib as it is
    otherwise very difficult to import module that has hyphen "-" in their
    filename.
    The SourcePlugin() methods used in the plugins (do_install_disk,
    do_configure_partition, do_prepare_partition) are then called on both,
-    beginning by "bootimg-efi".
+    beginning by "bootimg_efi".
    Plugin options, such as "--sourceparams" can still be passed to a
    plugin, as long they does not cause issue in the other plugin.
    Example wic configuration:
-    part /boot --source bootimg-biosplusefi --sourceparams="loader=grub-efi"\\
+    part /boot --source bootimg_biosplusefi --sourceparams="loader=grub-efi"\\
               --ondisk sda --label os_boot --active --align 1024 --use-uuid
    """
-    name = 'bootimg-biosplusefi'
+    name = 'bootimg_biosplusefi'
-    __PCBIOS_MODULE_NAME = "bootimg-pcbios"
+    __PCBIOS_MODULE_NAME = "bootimg_pcbios"
-    __EFI_MODULE_NAME = "bootimg-efi"
+    __EFI_MODULE_NAME = "bootimg_efi"
    __imgEFIObj = None
    __imgBiosObj = None
@@ -100,7 +100,7 @@ class BootimgBiosPlusEFIPlugin(SourcePlugin):
        """
-        # Import bootimg-pcbios (class name "BootimgPcbiosPlugin")
+        # Import bootimg_pcbios (class name "BootimgPcbiosPlugin")
        modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  cls.__PCBIOS_MODULE_NAME + ".py")
        loader = SourceFileLoader(cls.__PCBIOS_MODULE_NAME, modulePath)
@@ -108,7 +108,7 @@ class BootimgBiosPlusEFIPlugin(SourcePlugin):
        loader.exec_module(mod)
        cls.__imgBiosObj = mod.BootimgPcbiosPlugin()
-        # Import bootimg-efi (class name "BootimgEFIPlugin")
+        # Import bootimg_efi (class name "BootimgEFIPlugin")
        modulePath = os.path.join(os.path.dirname(os.path.realpath(__file__)),
                                  cls.__EFI_MODULE_NAME + ".py")
        loader = SourceFileLoader(cls.__EFI_MODULE_NAME, modulePath)
diff --git a/scripts/lib/wic/plugins/source/bootimg-efi.py b/scripts/lib/wic/plugins/source/bootimg_efi.py
index cdc72543c2..cf16705a28 100644
--- a/scripts/lib/wic/plugins/source/bootimg-efi.py
+++ b/scripts/lib/wic/plugins/source/bootimg_efi.py
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # DESCRIPTION
-# This implements the 'bootimg-efi' source plugin class for 'wic'
+# This implements the 'bootimg_efi' source plugin class for 'wic'
 #
 # AUTHORS
 # Tom Zanussi <tom.zanussi (at] linux.intel.com>
@@ -12,6 +12,7 @@
 import logging
 import os
+import tempfile
 import shutil
 import re
@@ -31,7 +32,29 @@ class BootimgEFIPlugin(SourcePlugin):
    This plugin supports GRUB 2 and systemd-boot bootloaders.
    """
-    name = 'bootimg-efi'
+    name = 'bootimg_efi'
+    @classmethod
+    def _copy_additional_files(cls, hdddir, initrd, dtb):
+        bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
+        if not bootimg_dir:
+            raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
+        if initrd:
+            initrds = initrd.split(';')
+            for rd in initrds:
+                cp_cmd = "cp -v -p %s/%s %s" % (bootimg_dir, rd, hdddir)
+                out = exec_cmd(cp_cmd, True)
+                logger.debug("initrd files:\n%s" % (out))
+        else:
+            logger.debug("Ignoring missing initrd")
+        if dtb:
+            if ';' in dtb:
+                raise WicError("Only one DTB supported, exiting")
+            cp_cmd = "cp -v -p %s/%s %s" % (bootimg_dir, dtb, hdddir)
+            out = exec_cmd(cp_cmd, True)
+            logger.debug("dtb files:\n%s" % (out))
    @classmethod
    def do_configure_grubefi(cls, hdddir, creator, cr_workdir, source_params):
@@ -52,18 +75,9 @@ class BootimgEFIPlugin(SourcePlugin):
                               "get it from %s." % configfile)
        initrd = source_params.get('initrd')
+        dtb = source_params.get('dtb')
-        if initrd:
+        cls._copy_additional_files(hdddir, initrd, dtb)
-            bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
-            if not bootimg_dir:
-                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
-            initrds = initrd.split(';')
-            for rd in initrds:
-                cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
-                exec_cmd(cp_cmd, True)
-        else:
-            logger.debug("Ignoring missing initrd")
        if not custom_cfg:
            # Create grub configuration using parameters from wks file
@@ -97,6 +111,9 @@ class BootimgEFIPlugin(SourcePlugin):
                    grubefi_conf += " /%s" % rd
                grubefi_conf += "\n"
+            if dtb:
+                grubefi_conf += "devicetree /%s\n" % dtb
            grubefi_conf += "}\n"
        logger.debug("Writing grubefi config %s/hdd/boot/EFI/BOOT/grub.cfg",
@@ -108,8 +125,16 @@ class BootimgEFIPlugin(SourcePlugin):
    @classmethod
    def do_configure_systemdboot(cls, hdddir, creator, cr_workdir, source_params):
        """
-        Create loader-specific systemd-boot/gummiboot config
+        Create loader-specific systemd-boot/gummiboot config. Unified Kernel Image (uki)
+        support is done in image recipe with uki.bbclass and only systemd-boot loader config
+        and ESP partition structure is created here.
        """
+        # detect uki.bbclass usage
+        image_classes = get_bitbake_var("IMAGE_CLASSES").split()
+        unified_image = False
+        if "uki" in image_classes:
+            unified_image = True
        install_cmd = "install -d %s/loader" % hdddir
        exec_cmd(install_cmd)
@@ -117,34 +142,26 @@ class BootimgEFIPlugin(SourcePlugin):
        exec_cmd(install_cmd)
        bootloader = creator.ks.bootloader
        loader_conf = ""
-        loader_conf += "default boot\n"
-        loader_conf += "timeout %d\n" % bootloader.timeout
-        initrd = source_params.get('initrd')
-        if initrd:
-            # obviously we need to have a common common deploy var
-            bootimg_dir = get_bitbake_var("DEPLOY_DIR_IMAGE")
-            if not bootimg_dir:
-                raise WicError("Couldn't find DEPLOY_DIR_IMAGE, exiting")
-            initrds = initrd.split(';')
+        # 5 seconds is a sensible default timeout
-            for rd in initrds:
+        loader_conf += "timeout %d\n" % (bootloader.timeout or 5)
-                cp_cmd = "cp %s/%s %s" % (bootimg_dir, rd, hdddir)
-                exec_cmd(cp_cmd, True)
-        else:
-            logger.debug("Ignoring missing initrd")
        logger.debug("Writing systemd-boot config "
                     "%s/hdd/boot/loader/loader.conf", cr_workdir)
        cfg = open("%s/hdd/boot/loader/loader.conf" % cr_workdir, "w")
        cfg.write(loader_conf)
+        logger.debug("loader.conf:\n%s" % (loader_conf))
        cfg.close()
+        initrd = source_params.get('initrd')
+        dtb = source_params.get('dtb')
+        if not unified_image:
+            cls._copy_additional_files(hdddir, initrd, dtb)
        configfile = creator.ks.bootloader.configfile
        custom_cfg = None
+        boot_conf = ""
        if configfile:
            custom_cfg = get_custom_config(configfile)
            if custom_cfg:
@@ -155,8 +172,7 @@ class BootimgEFIPlugin(SourcePlugin):
            else:
                raise WicError("configfile is specified but failed to "
                               "get it from %s.", configfile)
+        else:
-        if not custom_cfg:
            # Create systemd-boot configuration using parameters from wks file
            kernel = get_bitbake_var("KERNEL_IMAGETYPE")
            if get_bitbake_var("INITRAMFS_IMAGE_BUNDLE") == "1":
@@ -166,7 +182,6 @@ class BootimgEFIPlugin(SourcePlugin):
            title = source_params.get('title')
-            boot_conf = ""
            boot_conf += "title %s\n" % (title if title else "boot")
            boot_conf += "linux /%s\n" % kernel
@@ -183,11 +198,16 @@ class BootimgEFIPlugin(SourcePlugin):
                for rd in initrds:
                    boot_conf += "initrd /%s\n" % rd
-        logger.debug("Writing systemd-boot config "
+            if dtb:
-                     "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
+                boot_conf += "devicetree /%s\n" % dtb
-        cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
-        cfg.write(boot_conf)
+        if not unified_image:
-        cfg.close()
+            logger.debug("Writing systemd-boot config "
+                         "%s/hdd/boot/loader/entries/boot.conf", cr_workdir)
+            cfg = open("%s/hdd/boot/loader/entries/boot.conf" % cr_workdir, "w")
+            cfg.write(boot_conf)
+            logger.debug("boot.conf:\n%s" % (boot_conf))
+            cfg.close()
    @classmethod
@@ -207,10 +227,12 @@ class BootimgEFIPlugin(SourcePlugin):
                cls.do_configure_grubefi(hdddir, creator, cr_workdir, source_params)
            elif source_params['loader'] == 'systemd-boot':
                cls.do_configure_systemdboot(hdddir, creator, cr_workdir, source_params)
+            elif source_params['loader'] == 'uefi-kernel':
+                pass
            else:
-                raise WicError("unrecognized bootimg-efi loader: %s" % source_params['loader'])
+                raise WicError("unrecognized bootimg_efi loader: %s" % source_params['loader'])
        except KeyError:
-            raise WicError("bootimg-efi requires a loader, none specified")
+            raise WicError("bootimg_efi requires a loader, none specified")
        if get_bitbake_var("IMAGE_EFI_BOOT_FILES") is None:
            logger.debug('No boot files defined in IMAGE_EFI_BOOT_FILES')
@@ -230,7 +252,7 @@ class BootimgEFIPlugin(SourcePlugin):
            # list of tuples (src_name, dst_name)
            deploy_files = []
-            for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files):
+            for src_entry in re.findall(r'[\w;\-\.\+/\*]+', boot_files):
                if ';' in src_entry:
                    dst_entry = tuple(src_entry.split(';'))
                    if not dst_entry[0] or not dst_entry[1]:
@@ -288,40 +310,84 @@ class BootimgEFIPlugin(SourcePlugin):
                kernel = "%s-%s.bin" % \
                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
-        install_cmd = "install -m 0644 %s/%s %s/%s" % \
+        if source_params.get('create-unified-kernel-image') == "true":
-            (staging_kernel_dir, kernel, hdddir, kernel)
+            raise WicError("create-unified-kernel-image is no longer supported. Please use uki.bbclass.")
-        exec_cmd(install_cmd)
+        if source_params.get('install-kernel-into-boot-dir') != 'false':
+            install_cmd = "install -v -p -m 0644 %s/%s %s/%s" % \
+                (staging_kernel_dir, kernel, hdddir, kernel)
+            out = exec_cmd(install_cmd)
+            logger.debug("Installed kernel files:\n%s" % out)
        if get_bitbake_var("IMAGE_EFI_BOOT_FILES"):
            for src_path, dst_path in cls.install_task:
-                install_cmd = "install -m 0644 -D %s %s" \
+                install_cmd = "install -v -p -m 0644 -D %s %s" \
                              % (os.path.join(kernel_dir, src_path),
                                 os.path.join(hdddir, dst_path))
-                exec_cmd(install_cmd)
+                out = exec_cmd(install_cmd)
+                logger.debug("Installed IMAGE_EFI_BOOT_FILES:\n%s" % out)
        try:
            if source_params['loader'] == 'grub-efi':
                shutil.copyfile("%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir,
                                "%s/grub.cfg" % cr_workdir)
                for mod in [x for x in os.listdir(kernel_dir) if x.startswith("grub-efi-")]:
-                    cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:])
+                    cp_cmd = "cp -v -p %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[9:])
                    exec_cmd(cp_cmd, True)
                shutil.move("%s/grub.cfg" % cr_workdir,
                            "%s/hdd/boot/EFI/BOOT/grub.cfg" % cr_workdir)
            elif source_params['loader'] == 'systemd-boot':
                for mod in [x for x in os.listdir(kernel_dir) if x.startswith("systemd-")]:
-                    cp_cmd = "cp %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
+                    cp_cmd = "cp -v -p %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, mod[8:])
-                    exec_cmd(cp_cmd, True)
+                    out = exec_cmd(cp_cmd, True)
+                    logger.debug("systemd-boot files:\n%s" % out)
+            elif source_params['loader'] == 'uefi-kernel':
+                kernel = get_bitbake_var("KERNEL_IMAGETYPE")
+                if not kernel:
+                    raise WicError("Empty KERNEL_IMAGETYPE")
+                target = get_bitbake_var("TARGET_SYS")
+                if not target:
+                    raise WicError("Empty TARGET_SYS")
+                if re.match("x86_64", target):
+                    kernel_efi_image = "bootx64.efi"
+                elif re.match('i.86', target):
+                    kernel_efi_image = "bootia32.efi"
+                elif re.match('aarch64', target):
+                    kernel_efi_image = "bootaa64.efi"
+                elif re.match('arm', target):
+                    kernel_efi_image = "bootarm.efi"
+                else:
+                    raise WicError("UEFI stub kernel is incompatible with target %s" % target)
+                for mod in [x for x in os.listdir(kernel_dir) if x.startswith(kernel)]:
+                    cp_cmd = "cp -v -p %s/%s %s/EFI/BOOT/%s" % (kernel_dir, mod, hdddir, kernel_efi_image)
+                    out = exec_cmd(cp_cmd, True)
+                    logger.debug("uefi-kernel files:\n%s" % out)
            else:
-                raise WicError("unrecognized bootimg-efi loader: %s" %
+                raise WicError("unrecognized bootimg_efi loader: %s" %
                               source_params['loader'])
+            # must have installed at least one EFI bootloader
+            out = glob(os.path.join(hdddir, 'EFI', 'BOOT', 'boot*.efi'))
+            logger.debug("Installed EFI loader files:\n%s" % out)
+            if not out:
+                raise WicError("No EFI loaders installed to ESP partition. Check that grub-efi, systemd-boot or similar is installed.")
        except KeyError:
-            raise WicError("bootimg-efi requires a loader, none specified")
+            raise WicError("bootimg_efi requires a loader, none specified")
        startup = os.path.join(kernel_dir, "startup.nsh")
        if os.path.exists(startup):
-            cp_cmd = "cp %s %s/" % (startup, hdddir)
+            cp_cmd = "cp -v -p %s %s/" % (startup, hdddir)
-            exec_cmd(cp_cmd, True)
+            out = exec_cmd(cp_cmd, True)
+            logger.debug("startup files:\n%s" % out)
+        for paths in part.include_path or []:
+            for path in paths:
+                cp_cmd = "cp -v -p -r %s %s/" % (path, hdddir)
+                exec_cmd(cp_cmd, True)
+                logger.debug("include_path files:\n%s" % out)
        du_cmd = "du -bks %s" % hdddir
        out = exec_cmd(du_cmd)
@@ -337,17 +403,26 @@ class BootimgEFIPlugin(SourcePlugin):
        logger.debug("Added %d extra blocks to %s to get to %d total blocks",
                     extra_blocks, part.mountpoint, blocks)
+        # required for compatibility with certain devices expecting file system
+        # block count to be equal to partition block count
+        if blocks < part.fixed_size:
+            blocks = part.fixed_size
+            logger.debug("Overriding %s to %d total blocks for compatibility",
+                     part.mountpoint, blocks)
        # dosfs image, created by mkdosfs
        bootimg = "%s/boot.img" % cr_workdir
        label = part.label if part.label else "ESP"
-        dosfs_cmd = "mkdosfs -n %s -i %s -C %s %d" % \
+        dosfs_cmd = "mkdosfs -v -n %s -i %s -C %s %d" % \
                    (label, part.fsuuid, bootimg, blocks)
        exec_native_cmd(dosfs_cmd, native_sysroot)
+        logger.debug("mkdosfs:\n%s" % (str(out)))
-        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
+        mcopy_cmd = "mcopy -v -p -i %s -s %s/* ::/" % (bootimg, hdddir)
-        exec_native_cmd(mcopy_cmd, native_sysroot)
+        out = exec_native_cmd(mcopy_cmd, native_sysroot)
+        logger.debug("mcopy:\n%s" % (str(out)))
        chmod_cmd = "chmod 644 %s" % bootimg
        exec_cmd(chmod_cmd)
diff --git a/scripts/lib/wic/plugins/source/bootimg-partition.py b/scripts/lib/wic/plugins/source/bootimg_partition.py
index 5dbe2558d2..cc121a78f0 100644
--- a/scripts/lib/wic/plugins/source/bootimg-partition.py
+++ b/scripts/lib/wic/plugins/source/bootimg_partition.py
@@ -1,8 +1,10 @@
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # DESCRIPTION
-# This implements the 'bootimg-partition' source plugin class for
+# This implements the 'bootimg_partition' source plugin class for
 # 'wic'. The plugin creates an image of boot partition, copying over
 # files listed in IMAGE_BOOT_FILES bitbake variable.
 #
@@ -14,7 +16,7 @@ import logging
 import os
 import re
-from glob import glob
+from oe.bootfiles import get_boot_files
 from wic import WicError
 from wic.engine import get_custom_config
@@ -29,7 +31,8 @@ class BootimgPartitionPlugin(SourcePlugin):
    listed in IMAGE_BOOT_FILES bitbake variable.
    """
-    name = 'bootimg-partition'
+    name = 'bootimg_partition'
+    image_boot_files_var_name = 'IMAGE_BOOT_FILES'
    @classmethod
    def do_configure_partition(cls, part, source_params, cr, cr_workdir,
@@ -54,51 +57,16 @@ class BootimgPartitionPlugin(SourcePlugin):
            else:
                var = ""
-            boot_files = get_bitbake_var("IMAGE_BOOT_FILES" + var)
+            boot_files = get_bitbake_var(cls.image_boot_files_var_name + var)
            if boot_files is not None:
                break
        if boot_files is None:
-            raise WicError('No boot files defined, IMAGE_BOOT_FILES unset for entry #%d' % part.lineno)
+            raise WicError('No boot files defined, %s unset for entry #%d' % (cls.image_boot_files_var_name, part.lineno))
        logger.debug('Boot files: %s', boot_files)
-        # list of tuples (src_name, dst_name)
+        cls.install_task = get_boot_files(kernel_dir, boot_files)
-        deploy_files = []
-        for src_entry in re.findall(r'[\w;\-\./\*]+', boot_files):
-            if ';' in src_entry:
-                dst_entry = tuple(src_entry.split(';'))
-                if not dst_entry[0] or not dst_entry[1]:
-                    raise WicError('Malformed boot file entry: %s' % src_entry)
-            else:
-                dst_entry = (src_entry, src_entry)
-            logger.debug('Destination entry: %r', dst_entry)
-            deploy_files.append(dst_entry)
-        cls.install_task = [];
-        for deploy_entry in deploy_files:
-            src, dst = deploy_entry
-            if '*' in src:
-                # by default install files under their basename
-                entry_name_fn = os.path.basename
-                if dst != src:
-                    # unless a target name was given, then treat name
-                    # as a directory and append a basename
-                    entry_name_fn = lambda name: \
-                                    os.path.join(dst,
-                                                 os.path.basename(name))
-                srcs = glob(os.path.join(kernel_dir, src))
-                logger.debug('Globbed sources: %s', ', '.join(srcs))
-                for entry in srcs:
-                    src = os.path.relpath(entry, kernel_dir)
-                    entry_dst_name = entry_name_fn(entry)
-                    cls.install_task.append((src, entry_dst_name))
-            else:
-                cls.install_task.append((src, dst))
        if source_params.get('loader') != "u-boot":
            return
@@ -110,7 +78,7 @@ class BootimgPartitionPlugin(SourcePlugin):
                # Use a custom configuration for extlinux.conf
                extlinux_conf = custom_cfg
                logger.debug("Using custom configuration file "
-                             "%s for extlinux.cfg", configfile)
+                             "%s for extlinux.conf", configfile)
            else:
                raise WicError("configfile is specified but failed to "
                               "get it from %s." % configfile)
diff --git a/scripts/lib/wic/plugins/source/bootimg-pcbios.py b/scripts/lib/wic/plugins/source/bootimg_pcbios.py
index f2639e7004..21f41e00bb 100644
--- a/scripts/lib/wic/plugins/source/bootimg-pcbios.py
+++ b/scripts/lib/wic/plugins/source/bootimg_pcbios.py
@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # DESCRIPTION
-# This implements the 'bootimg-pcbios' source plugin class for 'wic'
+# This implements the 'bootimg_pcbios' source plugin class for 'wic'
 #
 # AUTHORS
 # Tom Zanussi <tom.zanussi (at] linux.intel.com>
@@ -27,7 +27,7 @@ class BootimgPcbiosPlugin(SourcePlugin):
    Create MBR boot partition and install syslinux on it.
    """
-    name = 'bootimg-pcbios'
+    name = 'bootimg_pcbios'
    @classmethod
    def _get_bootimg_dir(cls, bootimg_dir, dirname):
@@ -122,7 +122,7 @@ class BootimgPcbiosPlugin(SourcePlugin):
            syslinux_conf += "DEFAULT boot\n"
            syslinux_conf += "LABEL boot\n"
-            kernel = "/vmlinuz"
+            kernel = "/" + get_bitbake_var("KERNEL_IMAGETYPE")
            syslinux_conf += "KERNEL " + kernel + "\n"
            syslinux_conf += "APPEND label=boot root=%s %s\n" % \
@@ -155,8 +155,8 @@ class BootimgPcbiosPlugin(SourcePlugin):
                kernel = "%s-%s.bin" % \
                    (get_bitbake_var("KERNEL_IMAGETYPE"), get_bitbake_var("INITRAMFS_LINK_NAME"))
-        cmds = ("install -m 0644 %s/%s %s/vmlinuz" %
+        cmds = ("install -m 0644 %s/%s %s/%s" %
-                (staging_kernel_dir, kernel, hdddir),
+                (staging_kernel_dir, kernel, hdddir, get_bitbake_var("KERNEL_IMAGETYPE")),
                "install -m 444 %s/syslinux/ldlinux.sys %s/ldlinux.sys" %
                (bootimg_dir, hdddir),
                "install -m 0644 %s/syslinux/vesamenu.c32 %s/vesamenu.c32" %
@@ -186,8 +186,10 @@ class BootimgPcbiosPlugin(SourcePlugin):
        # dosfs image, created by mkdosfs
        bootimg = "%s/boot%s.img" % (cr_workdir, part.lineno)
-        dosfs_cmd = "mkdosfs -n boot -i %s -S 512 -C %s %d" % \
+        label = part.label if part.label else "boot"
-                    (part.fsuuid, bootimg, blocks)
+        dosfs_cmd = "mkdosfs -n %s -i %s -S 512 -C %s %d" % \
+                    (label, part.fsuuid, bootimg, blocks)
        exec_native_cmd(dosfs_cmd, native_sysroot)
        mcopy_cmd = "mcopy -i %s -s %s/* ::/" % (bootimg, hdddir)
diff --git a/scripts/lib/wic/plugins/source/empty.py b/scripts/lib/wic/plugins/source/empty.py
index 041617d648..4178912377 100644
--- a/scripts/lib/wic/plugins/source/empty.py
+++ b/scripts/lib/wic/plugins/source/empty.py
@@ -1,4 +1,6 @@
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: MIT
 #
@@ -7,9 +9,19 @@
 # To use it you must pass "empty" as argument for the "--source" parameter in
 # the wks file. For example:
 # part foo --source empty --ondisk sda --size="1024" --align 1024
+#
+# The plugin supports writing zeros to the start of the
+# partition. This is useful to overwrite old content like
+# filesystem signatures which may be re-recognized otherwise.
+# This feature can be enabled with
+# '--sourceparams="[fill|size=<N>[S|s|K|k|M|G]][,][bs=<N>[S|s|K|k|M|G]]"'
+# Conflicting or missing options throw errors.
 import logging
+import os
+from wic import WicError
+from wic.ksparser import sizetype
 from wic.pluginbase import SourcePlugin
 logger = logging.getLogger('wic')
@@ -17,6 +29,16 @@ logger = logging.getLogger('wic')
 class EmptyPartitionPlugin(SourcePlugin):
    """
    Populate unformatted empty partition.
+    The following sourceparams are supported:
+    - fill
+      Fill the entire partition with zeros. Requires '--fixed-size' option
+      to be set.
+    - size=<N>[S|s|K|k|M|G]
+      Set the first N bytes of the partition to zero. Default unit is 'K'.
+    - bs=<N>[S|s|K|k|M|G]
+      Write at most N bytes at a time during source file creation.
+      Defaults to '1M'. Default unit is 'K'.
    """
    name = 'empty'
@@ -29,4 +51,39 @@ class EmptyPartitionPlugin(SourcePlugin):
        Called to do the actual content population for a partition i.e. it
        'prepares' the partition to be incorporated into the image.
        """
-        return
+        get_byte_count = sizetype('K', True)
+        size = 0
+        if 'fill' in source_params and 'size' in source_params:
+            raise WicError("Conflicting source parameters 'fill' and 'size' specified, exiting.")
+        # Set the size of the zeros to be written to the partition
+        if 'fill' in source_params:
+            if part.fixed_size == 0:
+                raise WicError("Source parameter 'fill' only works with the '--fixed-size' option, exiting.")
+            size = get_byte_count(part.fixed_size)
+        elif 'size' in source_params:
+            size = get_byte_count(source_params['size'])
+        if size == 0:
+            # Nothing to do, create empty partition
+            return
+        if 'bs' in source_params:
+            bs = get_byte_count(source_params['bs'])
+        else:
+            bs = get_byte_count('1M')
+        # Create a binary file of the requested size filled with zeros
+        source_file = os.path.join(cr_workdir, 'empty-plugin-zeros%s.bin' % part.lineno)
+        if not os.path.exists(os.path.dirname(source_file)):
+            os.makedirs(os.path.dirname(source_file))
+        quotient, remainder = divmod(size, bs)
+        with open(source_file, 'wb') as file:
+            for _ in range(quotient):
+                file.write(bytearray(bs))
+            file.write(bytearray(remainder))
+        part.size = (size + 1024 - 1) // 1024  # size in KB rounded up
+        part.source_file = source_file
diff --git a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py b/scripts/lib/wic/plugins/source/isoimage_isohybrid.py
index afc9ea0f8f..5d42eb5d3e 100644
--- a/scripts/lib/wic/plugins/source/isoimage-isohybrid.py
+++ b/scripts/lib/wic/plugins/source/isoimage_isohybrid.py
@@ -1,8 +1,10 @@
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 # DESCRIPTION
-# This implements the 'isoimage-isohybrid' source plugin class for 'wic'
+# This implements the 'isoimage_isohybrid' source plugin class for 'wic'
 #
 # AUTHORS
 # Mihaly Varga <mihaly.varga (at] ni.com>
@@ -33,7 +35,7 @@ class IsoImagePlugin(SourcePlugin):
    bootloader files.
    Example kickstart file:
-    part /boot --source isoimage-isohybrid --sourceparams="loader=grub-efi, \\
+    part /boot --source isoimage_isohybrid --sourceparams="loader=grub-efi, \\
    image_name= IsoImage" --ondisk cd --label LIVECD
    bootloader  --timeout=10  --append=" "
@@ -43,7 +45,7 @@ class IsoImagePlugin(SourcePlugin):
    extension added by direct imeger plugin) and a file named IsoImage-cd.iso
    """
-    name = 'isoimage-isohybrid'
+    name = 'isoimage_isohybrid'
    @classmethod
    def do_configure_syslinux(cls, creator, cr_workdir):
@@ -338,10 +340,10 @@ class IsoImagePlugin(SourcePlugin):
                    cls.do_configure_grubefi(part, creator, target_dir)
            else:
-                raise WicError("unrecognized bootimg-efi loader: %s" %
+                raise WicError("unrecognized bootimg_efi loader: %s" %
                               source_params['loader'])
        except KeyError:
-            raise WicError("bootimg-efi requires a loader, none specified")
+            raise WicError("bootimg_efi requires a loader, none specified")
        # Create efi.img that contains bootloader files for EFI booting
        # if ISODIR didn't exist or didn't contains it
diff --git a/scripts/lib/wic/plugins/source/rawcopy.py b/scripts/lib/wic/plugins/source/rawcopy.py
index 3c4997d8ba..21903c2f23 100644
--- a/scripts/lib/wic/plugins/source/rawcopy.py
+++ b/scripts/lib/wic/plugins/source/rawcopy.py
@@ -1,9 +1,13 @@
 #
+# Copyright OpenEmbedded Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 import logging
 import os
+import signal
+import subprocess
 from wic import WicError
 from wic.pluginbase import SourcePlugin
@@ -21,6 +25,10 @@ class RawCopyPlugin(SourcePlugin):
    @staticmethod
    def do_image_label(fstype, dst, label):
+        # don't create label when fstype is none
+        if fstype == 'none':
+            return
        if fstype.startswith('ext'):
            cmd = 'tune2fs -L %s %s' % (label, dst)
        elif fstype in ('msdos', 'vfat'):
@@ -29,15 +37,35 @@ class RawCopyPlugin(SourcePlugin):
            cmd = 'btrfs filesystem label %s %s' % (dst, label)
        elif fstype == 'swap':
            cmd = 'mkswap -L %s %s' % (label, dst)
-        elif fstype == 'squashfs':
+        elif fstype in ('squashfs', 'erofs'):
-            raise WicError("It's not possible to update a squashfs "
+            raise WicError("It's not possible to update a %s "
-                           "filesystem label '%s'" % (label))
+                           "filesystem label '%s'" % (fstype, label))
        else:
            raise WicError("Cannot update filesystem label: "
                           "Unknown fstype: '%s'" % (fstype))
        exec_cmd(cmd)
+    @staticmethod
+    def do_image_uncompression(src, dst, workdir):
+        def subprocess_setup():
+            # Python installs a SIGPIPE handler by default. This is usually not what
+            # non-Python subprocesses expect.
+            # SIGPIPE errors are known issues with gzip/bash
+            signal.signal(signal.SIGPIPE, signal.SIG_DFL)
+        extension = os.path.splitext(src)[1]
+        decompressor = {
+            ".bz2": "bzip2",
+            ".gz": "gzip",
+            ".xz": "xz",
+            ".zst": "zstd -f",
+        }.get(extension)
+        if not decompressor:
+            raise WicError("Not supported compressor filename extension: %s" % extension)
+        cmd = "%s -dc %s > %s" % (decompressor, src, dst)
+        subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True, cwd=workdir)
    @classmethod
    def do_prepare_partition(cls, part, source_params, cr, cr_workdir,
                             oe_builddir, bootimg_dir, kernel_dir,
@@ -56,7 +84,13 @@ class RawCopyPlugin(SourcePlugin):
        if 'file' not in source_params:
            raise WicError("No file specified")
-        src = os.path.join(kernel_dir, source_params['file'])
+        if 'unpack' in source_params:
+            img = os.path.join(kernel_dir, source_params['file'])
+            src = os.path.join(cr_workdir, os.path.splitext(source_params['file'])[0])
+            RawCopyPlugin.do_image_uncompression(img, src, cr_workdir)
+        else:
+            src = os.path.join(kernel_dir, source_params['file'])
        dst = os.path.join(cr_workdir, "%s.%s" % (os.path.basename(source_params['file']), part.lineno))
        if not os.path.exists(os.path.dirname(dst)):
diff --git a/scripts/lib/wic/plugins/source/rootfs.py b/scripts/lib/wic/plugins/source/rootfs.py
index 96d940a91d..06fce06bb1 100644
--- a/scripts/lib/wic/plugins/source/rootfs.py
+++ b/scripts/lib/wic/plugins/source/rootfs.py
@@ -35,22 +35,22 @@ class RootfsPlugin(SourcePlugin):
    @staticmethod
    def __validate_path(cmd, rootfs_dir, path):
        if os.path.isabs(path):
-            logger.error("%s: Must be relative: %s" % (cmd, orig_path))
+            logger.error("%s: Must be relative: %s" % (cmd, path))
            sys.exit(1)
        # Disallow climbing outside of parent directory using '..',
        # because doing so could be quite disastrous (we will delete the
        # directory, or modify a directory outside OpenEmbedded).
-        full_path = os.path.realpath(os.path.join(rootfs_dir, path))
+        full_path = os.path.abspath(os.path.join(rootfs_dir, path))
        if not full_path.startswith(os.path.realpath(rootfs_dir)):
-            logger.error("%s: Must point inside the rootfs:" % (cmd, path))
+            logger.error("%s: Must point inside the rootfs: %s" % (cmd, path))
            sys.exit(1)
        return full_path
    @staticmethod
    def __get_rootfs_dir(rootfs_dir):
-        if os.path.isdir(rootfs_dir):
+        if rootfs_dir and os.path.isdir(rootfs_dir):
            return os.path.realpath(rootfs_dir)
        image_rootfs_dir = get_bitbake_var("IMAGE_ROOTFS", rootfs_dir)
@@ -97,6 +97,9 @@ class RootfsPlugin(SourcePlugin):
        part.has_fstab = os.path.exists(os.path.join(part.rootfs_dir, "etc/fstab"))
        pseudo_dir = os.path.join(part.rootfs_dir, "../pseudo")
        if not os.path.lexists(pseudo_dir):
+            pseudo_dir = os.path.join(cls.__get_rootfs_dir(None), '../pseudo')
+        if not os.path.lexists(pseudo_dir):
            logger.warn("%s folder does not exist. "
                        "Usernames and permissions will be invalid " % pseudo_dir)
            pseudo_dir = None
@@ -218,10 +221,10 @@ class RootfsPlugin(SourcePlugin):
            # Update part.has_fstab here as fstab may have been added or
            # removed by the above modifications.
            part.has_fstab = os.path.exists(os.path.join(new_rootfs, "etc/fstab"))
-            if part.update_fstab_in_rootfs and part.has_fstab:
+            if part.update_fstab_in_rootfs and part.has_fstab and not part.no_fstab_update:
                fstab_path = os.path.join(new_rootfs, "etc/fstab")
                # Assume that fstab should always be owned by root with fixed permissions
-                install_cmd = "install -m 0644 %s %s" % (part.updated_fstab_path, fstab_path)
+                install_cmd = "install -m 0644 -p %s %s" % (part.updated_fstab_path, fstab_path)
                if new_pseudo:
                    pseudo = cls.__get_pseudo(native_sysroot, new_rootfs, new_pseudo)
                else: