3 files changed, 352 insertions, 3 deletions

diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index f4571c4ef1..ad10af5826 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -8,7 +8,7 @@ import glob
 
 import oeqa.utils.ftools as ftools
 from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import runCmd, bitbake, get_bb_var
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
 from oeqa.utils.decorators import testcase
 
 class DevtoolBase(oeSelfTest):
@@ -31,6 +31,35 @@ class DevtoolBase(oeSelfTest):
         for inherit in checkinherits:
             self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)
 
+    def _check_bbappend(self, testrecipe, recipefile, appenddir):
+        result = runCmd('bitbake-layers show-appends', cwd=self.builddir)
+        resultlines = result.output.splitlines()
+        inrecipe = False
+        bbappends = []
+        bbappendfile = None
+        for line in resultlines:
+            if inrecipe:
+                if line.startswith(' '):
+                    bbappends.append(line.strip())
+                else:
+                    break
+            elif line == '%s:' % os.path.basename(recipefile):
+                inrecipe = True
+        self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n  %s' % (testrecipe, '\n  '.join(bbappends)))
+        for bbappend in bbappends:
+            if bbappend.startswith(appenddir):
+                bbappendfile = bbappend
+                break
+        else:
+            self.assertTrue(False, 'bbappend for recipe %s does not seem to be created in test layer' % testrecipe)
+        return bbappendfile
+
+    def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'):
+        create_temp_layer(templayerdir, templayername, priority, recipepathspec)
+        if addlayer:
+            self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)
+            result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)
+
 
 class DevtoolTests(DevtoolBase):
 
diff --git a/meta/lib/oeqa/selftest/recipetool.py b/meta/lib/oeqa/selftest/recipetool.py
index 832fb7b16a..f3ad493457 100644
--- a/meta/lib/oeqa/selftest/recipetool.py
+++ b/meta/lib/oeqa/selftest/recipetool.py
@@ -6,16 +6,326 @@ import tempfile
 
 import oeqa.utils.ftools as ftools
 from oeqa.selftest.base import oeSelfTest
-from oeqa.utils.commands import runCmd, bitbake, get_bb_var
+from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
 from oeqa.utils.decorators import testcase
 from oeqa.selftest.devtool import DevtoolBase
 
 
+templayerdir = ''
+
+def setUpModule():
+    global templayerdir
+    templayerdir = tempfile.mkdtemp(prefix='recipetoolqa')
+    create_temp_layer(templayerdir, 'selftestrecipetool')
+    result = runCmd('bitbake-layers add-layer %s' % templayerdir)
+    # Ensure we have the right data in shlibs/pkgdata
+    logger = logging.getLogger("selftest")
+    logger.info('Running bitbake to generate pkgdata')
+    bitbake('base-files coreutils busybox selftest-recipetool-appendfile')
+
+def tearDownModule():
+    runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True)
+    runCmd('rm -rf %s' % templayerdir)
+    # Shouldn't leave any traces of this artificial recipe behind
+    bitbake('-c cleansstate selftest-recipetool-appendfile')
+
+
 class RecipetoolTests(DevtoolBase):
 
     def setUpLocal(self):
         self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa')
         self.track_for_cleanup(self.tempdir)
+        self.testfile = os.path.join(self.tempdir, 'testfile')
+        with open(self.testfile, 'w') as f:
+            f.write('Test file\n')
+
+    def tearDownLocal(self):
+        runCmd('rm -rf %s/recipes-*' % templayerdir)
+
+    def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles):
+        result = runCmd('recipetool appendfile %s %s %s %s' % (templayerdir, destfile, newfile, options))
+        self.assertNotIn('Traceback', result.output)
+        # Check the bbappend was created and applies properly
+        recipefile = get_bb_var('FILE', testrecipe)
+        bbappendfile = self._check_bbappend(testrecipe, recipefile, templayerdir)
+        # Check the bbappend contents
+        with open(bbappendfile, 'r') as f:
+            self.assertEqual(expectedlines, f.readlines())
+        # Check file was copied
+        filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe)
+        for expectedfile in expectedfiles:
+            self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile)
+        # Check no other files created
+        createdfiles = []
+        for root, _, files in os.walk(filesdir):
+            for f in files:
+                createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir))
+        self.assertTrue(sorted(createdfiles), sorted(expectedfiles))
+        return bbappendfile, result.output
+
+    def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror):
+        cmd = 'recipetool appendfile %s %s %s' % (templayerdir, destfile, newfile)
+        result = runCmd(cmd, ignore_status=True)
+        self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd)
+        self.assertNotIn('Traceback', result.output)
+        for errorstr in checkerror:
+            self.assertIn(errorstr, result.output)
+
+
+    def test_recipetool_appendfile_basic(self):
+        # Basic test
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                        '\n']
+        _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_invalid(self):
+        # Test some commands that should error
+        self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers'])
+        self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool'])
+        self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool'])
+
+    def test_recipetool_appendfile_alternatives(self):
+        # Now try with a file we know should be an alternative
+        # (this is very much a fake example, but one we know is reliably an alternative)
+        self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox'])
+        corebase = get_bb_var('COREBASE')
+        # Need a test file - should be executable
+        testfile2 = os.path.join(corebase, 'oe-init-build-env')
+        testfile2name = os.path.basename(testfile2)
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://%s"\n' % testfile2name,
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${base_bindir}\n',
+                         '    install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name,
+                         '}\n']
+        self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name])
+        # Now try bbappending the same file again, contents should not change
+        bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name])
+        # But file should have
+        copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name)
+        result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True)
+        self.assertNotEqual(result.status, 0, 'New file should have been copied but was not')
+
+    def test_recipetool_appendfile_binary(self):
+        # Try appending a binary file
+        result = runCmd('recipetool appendfile %s /bin/ls /bin/ls -r coreutils' % templayerdir)
+        self.assertIn('WARNING: ', result.output)
+        self.assertIn('is a binary', result.output)
+
+    def test_recipetool_appendfile_add(self):
+        corebase = get_bb_var('COREBASE')
+        # Try arbitrary file add to a recipe
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
+                         '}\n']
+        self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile'])
+        # Try adding another file, this time where the source file is executable
+        # (so we're testing that, plus modifying an existing bbappend)
+        testfile2 = os.path.join(corebase, 'oe-init-build-env')
+        testfile2name = os.path.basename(testfile2)
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile \\\n',
+                         '            file://%s \\\n' % testfile2name,
+                         '            "\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
+                         '    install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name,
+                         '}\n']
+        self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name])
+
+    def test_recipetool_appendfile_add_bindir(self):
+        # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${bindir}\n',
+                         '    install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_add_machine(self):
+        # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'PACKAGE_ARCH = "${MACHINE_ARCH}"\n',
+                         '\n',
+                         'SRC_URI_append_mymachine = " file://testfile"\n',
+                         '\n',
+                         'do_install_append_mymachine() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_orig(self):
+        # A file that's in SRC_URI and in do_install with the same name
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_todir(self):
+        # A file that's in SRC_URI and in do_install with destination directory rather than file
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_renamed(self):
+        # A file that's in SRC_URI with a different name to the destination file
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_subdir(self):
+        # A file that's in SRC_URI in a subdir
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_src_glob(self):
+        # A file that's in SRC_URI as a glob
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-src-globfile\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-src-globfile', self.testfile, '', expectedlines, ['testfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_inst_glob(self):
+        # A file that's in do_install as a glob
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_inst_todir_glob(self):
+        # A file that's in do_install as a glob with destination as a directory
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_patch(self):
+        # A file that's added by a patch in SRC_URI
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${sysconfdir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile'])
+        for line in output.splitlines():
+            if line.startswith('WARNING: '):
+                self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line)
+                break
+        else:
+            self.assertTrue(False, 'Patch warning not found in output:\n%s' % output)
+
+    def test_recipetool_appendfile_script(self):
+        # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install)
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_inst_func(self):
+        # A file that's installed from a function called by do_install
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func'])
+        self.assertNotIn('WARNING: ', output)
+
+    def test_recipetool_appendfile_postinstall(self):
+        # A file that's created by a postinstall script (and explicitly mentioned in it)
+        # First try without specifying recipe
+        self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile'])
+        # Now specify recipe
+        expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
+                         '\n',
+                         'SRC_URI += "file://testfile"\n',
+                         '\n',
+                         'do_install_append() {\n',
+                         '    install -d ${D}${datadir}\n',
+                         '    install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n',
+                         '}\n']
+        _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile'])
+
+    def test_recipetool_appendfile_extlayer(self):
+        # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure
+        exttemplayerdir = os.path.join(self.tempdir, 'extlayer')
+        self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*')
+        result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile))
+        self.assertNotIn('Traceback', result.output)
+        createdfiles = []
+        for root, _, files in os.walk(exttemplayerdir):
+            for f in files:
+                createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir))
+        createdfiles.remove('conf/layer.conf')
+        expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend',
+                         'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig']
+        self.assertEqual(sorted(createdfiles), sorted(expectedfiles))
+
+    def test_recipetool_appendfile_wildcard(self):
+
+        def try_appendfile_wc(options):
+            result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (templayerdir, self.testfile, options))
+            self.assertNotIn('Traceback', result.output)
+            bbappendfile = None
+            for root, _, files in os.walk(templayerdir):
+                for f in files:
+                    if f.endswith('.bbappend'):
+                        bbappendfile = f
+                        break
+            if not bbappendfile:
+                self.assertTrue(False, 'No bbappend file created')
+            runCmd('rm -rf %s/recipes-*' % templayerdir)
+            return bbappendfile
+
+        # Check without wildcard option
+        recipefn = os.path.basename(get_bb_var('FILE', 'base-files'))
+        filename = try_appendfile_wc('')
+        self.assertEqual(filename, recipefn.replace('.bb', '.bbappend'))
+        # Now check with wildcard option
+        filename = try_appendfile_wc('-w')
+        self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend')
+
+
 
     def test_recipetool_create(self):
         # Try adding a recipe
@@ -52,4 +362,3 @@ class RecipetoolTests(DevtoolBase):
         checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
         inherits = ['autotools', 'pkgconfig']
         self._test_recipe_contents(recipefile, checkvars, inherits)
-
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 663e4e7f41..dc8a9836e7 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -162,3 +162,14 @@ def get_test_layer():
             testlayer = l
             break
     return testlayer
+
+def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
+    os.makedirs(os.path.join(templayerdir, 'conf'))
+    with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
+        f.write('BBPATH .= ":${LAYERDIR}"\n')
+        f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec)
+        f.write('            ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec)
+        f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername)
+        f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername)
+        f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority))
+        f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername)