summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta-selftest/recipes-test/recipetool/files/add-file.patch8
-rw-r--r--meta-selftest/recipes-test/recipetool/files/file12
-rw-r--r--meta-selftest/recipes-test/recipetool/files/installscript.sh3
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-func1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-globfile1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-todir-globfile1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-orig1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-src-globfile1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/selftest-replaceme-todir1
-rw-r--r--meta-selftest/recipes-test/recipetool/files/subdir/fileinsubdir1
-rw-r--r--meta-selftest/recipes-test/recipetool/selftest-recipetool-appendfile.bb42
-rw-r--r--meta/lib/oe/patch.py63
-rw-r--r--meta/lib/oe/recipeutils.py328
-rw-r--r--meta/lib/oeqa/selftest/devtool.py31
-rw-r--r--meta/lib/oeqa/selftest/recipetool.py313
-rw-r--r--meta/lib/oeqa/utils/commands.py11
-rw-r--r--scripts/lib/recipetool/append.py360
-rwxr-xr-xscripts/recipetool6
18 files changed, 1167 insertions, 7 deletions
diff --git a/meta-selftest/recipes-test/recipetool/files/add-file.patch b/meta-selftest/recipes-test/recipetool/files/add-file.patch
new file mode 100644
index 0000000000..bdc99c94f0
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/add-file.patch
@@ -0,0 +1,8 @@
1diff --git a/file2 b/file2
2new file mode 100644
3index 0000000..049b42e
4--- /dev/null
5+++ b/file2
6@@ -0,0 +1,2 @@
7+Test file 2
8+456
diff --git a/meta-selftest/recipes-test/recipetool/files/file1 b/meta-selftest/recipes-test/recipetool/files/file1
new file mode 100644
index 0000000000..7571aa7a88
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/file1
@@ -0,0 +1,2 @@
1First test file
2123
diff --git a/meta-selftest/recipes-test/recipetool/files/installscript.sh b/meta-selftest/recipes-test/recipetool/files/installscript.sh
new file mode 100644
index 0000000000..9de30d69ca
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/installscript.sh
@@ -0,0 +1,3 @@
1#!/bin/sh
2echo "Third file" > $1/selftest-replaceme-scripted
3
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-func b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-func
new file mode 100644
index 0000000000..2802bb348b
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-func
@@ -0,0 +1 @@
A file installed by a function called by do_install
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-globfile b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-globfile
new file mode 100644
index 0000000000..996298bf1f
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-globfile
@@ -0,0 +1 @@
A file matched by a glob in do_install
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-todir-globfile b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-todir-globfile
new file mode 100644
index 0000000000..585ae3e9b0
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-inst-todir-globfile
@@ -0,0 +1 @@
A file matched by a glob in do_install to a directory
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-orig b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-orig
new file mode 100644
index 0000000000..593d6a0bb4
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-orig
@@ -0,0 +1 @@
Straight through with same nam
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-src-globfile b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-src-globfile
new file mode 100644
index 0000000000..1e20a2b03e
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-src-globfile
@@ -0,0 +1 @@
A file matched by a glob in SRC_URI
diff --git a/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-todir b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-todir
new file mode 100644
index 0000000000..85bd5eba46
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/selftest-replaceme-todir
@@ -0,0 +1 @@
File in SRC_URI installed just to directory path
diff --git a/meta-selftest/recipes-test/recipetool/files/subdir/fileinsubdir b/meta-selftest/recipes-test/recipetool/files/subdir/fileinsubdir
new file mode 100644
index 0000000000..d516b4951b
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/files/subdir/fileinsubdir
@@ -0,0 +1 @@
A file in a subdirectory
diff --git a/meta-selftest/recipes-test/recipetool/selftest-recipetool-appendfile.bb b/meta-selftest/recipes-test/recipetool/selftest-recipetool-appendfile.bb
new file mode 100644
index 0000000000..7d0a040beb
--- /dev/null
+++ b/meta-selftest/recipes-test/recipetool/selftest-recipetool-appendfile.bb
@@ -0,0 +1,42 @@
1SUMMARY = "Test recipe for recipetool appendfile"
2LICENSE = "MIT"
3LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302"
4
5INHIBIT_DEFAULT_DEPS = "1"
6
7SRC_URI = "file://installscript.sh \
8 file://selftest-replaceme-orig \
9 file://selftest-replaceme-todir \
10 file://file1 \
11 file://add-file.patch \
12 file://subdir \
13 file://selftest-replaceme-src-glob* \
14 file://selftest-replaceme-inst-globfile \
15 file://selftest-replaceme-inst-todir-globfile \
16 file://selftest-replaceme-inst-func"
17
18install_extrafunc() {
19 install -m 0644 ${WORKDIR}/selftest-replaceme-inst-func ${D}${datadir}/selftest-replaceme-inst-func
20}
21
22do_install() {
23 install -d ${D}${datadir}/
24 install -m 0644 ${WORKDIR}/selftest-replaceme-orig ${D}${datadir}/selftest-replaceme-orig
25 install -m 0644 ${WORKDIR}/selftest-replaceme-todir ${D}${datadir}
26 install -m 0644 ${WORKDIR}/file1 ${D}${datadir}/selftest-replaceme-renamed
27 install -m 0644 ${WORKDIR}/subdir/fileinsubdir ${D}${datadir}/selftest-replaceme-subdir
28 install -m 0644 ${WORKDIR}/selftest-replaceme-src-globfile ${D}${datadir}/selftest-replaceme-src-globfile
29 cp ${WORKDIR}/selftest-replaceme-inst-glob* ${D}${datadir}/selftest-replaceme-inst-globfile
30 cp ${WORKDIR}/selftest-replaceme-inst-todir-glob* ${D}${datadir}
31 install -d ${D}${sysconfdir}
32 install -m 0644 ${S}/file2 ${D}${sysconfdir}/selftest-replaceme-patched
33 sh ${WORKDIR}/installscript.sh ${D}${datadir}
34 install_extrafunc
35}
36
37pkg_postinst_${PN} () {
38 echo "Test file installed by postinst" > $D${datadir}/selftest-replaceme-postinst
39}
40
41FILES_${PN} += "${datadir}"
42
diff --git a/meta/lib/oe/patch.py b/meta/lib/oe/patch.py
index e1f1c53bef..afb0013a4b 100644
--- a/meta/lib/oe/patch.py
+++ b/meta/lib/oe/patch.py
@@ -92,6 +92,69 @@ class PatchSet(object):
92 def Refresh(self, remote = None, all = None): 92 def Refresh(self, remote = None, all = None):
93 raise NotImplementedError() 93 raise NotImplementedError()
94 94
95 @staticmethod
96 def getPatchedFiles(patchfile, striplevel, srcdir=None):
97 """
98 Read a patch file and determine which files it will modify.
99 Params:
100 patchfile: the patch file to read
101 striplevel: the strip level at which the patch is going to be applied
102 srcdir: optional path to join onto the patched file paths
103 Returns:
104 A list of tuples of file path and change mode ('A' for add,
105 'D' for delete or 'M' for modify)
106 """
107
108 def patchedpath(patchline):
109 filepth = patchline.split()[1]
110 if filepth.endswith('/dev/null'):
111 return '/dev/null'
112 filesplit = filepth.split(os.sep)
113 if striplevel > len(filesplit):
114 bb.error('Patch %s has invalid strip level %d' % (patchfile, striplevel))
115 return None
116 return os.sep.join(filesplit[striplevel:])
117
118 copiedmode = False
119 filelist = []
120 with open(patchfile) as f:
121 for line in f:
122 if line.startswith('--- '):
123 patchpth = patchedpath(line)
124 if not patchpth:
125 break
126 if copiedmode:
127 addedfile = patchpth
128 else:
129 removedfile = patchpth
130 elif line.startswith('+++ '):
131 addedfile = patchedpath(line)
132 if not addedfile:
133 break
134 elif line.startswith('*** '):
135 copiedmode = True
136 removedfile = patchedpath(line)
137 if not removedfile:
138 break
139 else:
140 removedfile = None
141 addedfile = None
142
143 if addedfile and removedfile:
144 if removedfile == '/dev/null':
145 mode = 'A'
146 elif addedfile == '/dev/null':
147 mode = 'D'
148 else:
149 mode = 'M'
150 if srcdir:
151 fullpath = os.path.abspath(os.path.join(srcdir, addedfile))
152 else:
153 fullpath = addedfile
154 filelist.append((fullpath, mode))
155
156 return filelist
157
95 158
96class PatchTree(PatchSet): 159class PatchTree(PatchSet):
97 def __init__(self, dir, d): 160 def __init__(self, dir, d):
diff --git a/meta/lib/oe/recipeutils.py b/meta/lib/oe/recipeutils.py
index 0689fb0c71..f05b6c06ba 100644
--- a/meta/lib/oe/recipeutils.py
+++ b/meta/lib/oe/recipeutils.py
@@ -2,7 +2,7 @@
2# 2#
3# Some code borrowed from the OE layer index 3# Some code borrowed from the OE layer index
4# 4#
5# Copyright (C) 2013-2014 Intel Corporation 5# Copyright (C) 2013-2015 Intel Corporation
6# 6#
7 7
8import sys 8import sys
@@ -14,6 +14,7 @@ import difflib
14import utils 14import utils
15import shutil 15import shutil
16import re 16import re
17import fnmatch
17from collections import OrderedDict, defaultdict 18from collections import OrderedDict, defaultdict
18 19
19 20
@@ -289,6 +290,27 @@ def get_recipe_patches(d):
289 return patchfiles 290 return patchfiles
290 291
291 292
293def get_recipe_patched_files(d):
294 """
295 Get the list of patches for a recipe along with the files each patch modifies.
296 Params:
297 d: the datastore for the recipe
298 Returns:
299 a dict mapping patch file path to a list of tuples of changed files and
300 change mode ('A' for add, 'D' for delete or 'M' for modify)
301 """
302 import oe.patch
303 # Execute src_patches() defined in patch.bbclass - this works since that class
304 # is inherited globally
305 patches = bb.utils.exec_flat_python_func('src_patches', d)
306 patchedfiles = {}
307 for patch in patches:
308 _, _, patchfile, _, _, parm = bb.fetch.decodeurl(patch)
309 striplevel = int(parm['striplevel'])
310 patchedfiles[patchfile] = oe.patch.PatchSet.getPatchedFiles(patchfile, striplevel, os.path.join(d.getVar('S', True), parm.get('patchdir', '')))
311 return patchedfiles
312
313
292def validate_pn(pn): 314def validate_pn(pn):
293 """Perform validation on a recipe name (PN) for a new recipe.""" 315 """Perform validation on a recipe name (PN) for a new recipe."""
294 reserved_names = ['forcevariable', 'append', 'prepend', 'remove'] 316 reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
@@ -300,3 +322,307 @@ def validate_pn(pn):
300 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn 322 return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
301 return '' 323 return ''
302 324
325
326def get_bbappend_path(d, destlayerdir, wildcardver=False):
327 """Determine how a bbappend for a recipe should be named and located within another layer"""
328
329 import bb.cookerdata
330
331 destlayerdir = os.path.abspath(destlayerdir)
332 recipefile = d.getVar('FILE', True)
333 recipefn = os.path.splitext(os.path.basename(recipefile))[0]
334 if wildcardver and '_' in recipefn:
335 recipefn = recipefn.split('_', 1)[0] + '_%'
336 appendfn = recipefn + '.bbappend'
337
338 # Parse the specified layer's layer.conf file directly, in case the layer isn't in bblayers.conf
339 confdata = d.createCopy()
340 confdata.setVar('BBFILES', '')
341 confdata.setVar('LAYERDIR', destlayerdir)
342 destlayerconf = os.path.join(destlayerdir, "conf", "layer.conf")
343 confdata = bb.cookerdata.parse_config_file(destlayerconf, confdata)
344
345 origlayerdir = find_layerdir(recipefile)
346 if not origlayerdir:
347 return (None, False)
348 # Now join this to the path where the bbappend is going and check if it is covered by BBFILES
349 appendpath = os.path.join(destlayerdir, os.path.relpath(os.path.dirname(recipefile), origlayerdir), appendfn)
350 closepath = ''
351 pathok = True
352 for bbfilespec in confdata.getVar('BBFILES', True).split():
353 if fnmatch.fnmatchcase(appendpath, bbfilespec):
354 # Our append path works, we're done
355 break
356 elif bbfilespec.startswith(destlayerdir) and fnmatch.fnmatchcase('test.bbappend', os.path.basename(bbfilespec)):
357 # Try to find the longest matching path
358 if len(bbfilespec) > len(closepath):
359 closepath = bbfilespec
360 else:
361 # Unfortunately the bbappend layer and the original recipe's layer don't have the same structure
362 if closepath:
363 # bbappend layer's layer.conf at least has a spec that picks up .bbappend files
364 # Now we just need to substitute out any wildcards
365 appendsubdir = os.path.relpath(os.path.dirname(closepath), destlayerdir)
366 if 'recipes-*' in appendsubdir:
367 # Try to copy this part from the original recipe path
368 res = re.search('/recipes-[^/]+/', recipefile)
369 if res:
370 appendsubdir = appendsubdir.replace('/recipes-*/', res.group(0))
371 # This is crude, but we have to do something
372 appendsubdir = appendsubdir.replace('*', recipefn.split('_')[0])
373 appendsubdir = appendsubdir.replace('?', 'a')
374 appendpath = os.path.join(destlayerdir, appendsubdir, appendfn)
375 else:
376 pathok = False
377 return (appendpath, pathok)
378
379
380def bbappend_recipe(rd, destlayerdir, srcfiles, install=None, wildcardver=False, machine=None, extralines=None, removevalues=None):
381 """
382 Writes a bbappend file for a recipe
383 Parameters:
384 rd: data dictionary for the recipe
385 destlayerdir: base directory of the layer to place the bbappend in
386 (subdirectory path from there will be determined automatically)
387 srcfiles: dict of source files to add to SRC_URI, where the value
388 is the full path to the file to be added, and the value is the
389 original filename as it would appear in SRC_URI or None if it
390 isn't already present. You may pass None for this parameter if
391 you simply want to specify your own content via the extralines
392 parameter.
393 install: dict mapping entries in srcfiles to a tuple of two elements:
394 install path (*without* ${D} prefix) and permission value (as a
395 string, e.g. '0644').
396 wildcardver: True to use a % wildcard in the bbappend filename, or
397 False to make the bbappend specific to the recipe version.
398 machine:
399 If specified, make the changes in the bbappend specific to this
400 machine. This will also cause PACKAGE_ARCH = "${MACHINE_ARCH}"
401 to be added to the bbappend.
402 extralines:
403 Extra lines to add to the bbappend. This may be a dict of name
404 value pairs, or simply a list of the lines.
405 removevalues:
406 Variable values to remove - a dict of names/values.
407 """
408
409 if not removevalues:
410 removevalues = {}
411
412 # Determine how the bbappend should be named
413 appendpath, pathok = get_bbappend_path(rd, destlayerdir, wildcardver)
414 if not appendpath:
415 bb.error('Unable to determine layer directory containing %s' % recipefile)
416 return (None, None)
417 if not pathok:
418 bb.warn('Unable to determine correct subdirectory path for bbappend file - check that what %s adds to BBFILES also matches .bbappend files. Using %s for now, but until you fix this the bbappend will not be applied.' % (os.path.join(destlayerdir, 'conf', 'layer.conf'), os.path.dirname(appendpath)))
419
420 appenddir = os.path.dirname(appendpath)
421 bb.utils.mkdirhier(appenddir)
422
423 # FIXME check if the bbappend doesn't get overridden by a higher priority layer?
424
425 layerdirs = [os.path.abspath(layerdir) for layerdir in rd.getVar('BBLAYERS', True).split()]
426 if not os.path.abspath(destlayerdir) in layerdirs:
427 bb.warn('Specified layer is not currently enabled in bblayers.conf, you will need to add it before this bbappend will be active')
428
429 bbappendlines = []
430 if extralines:
431 if isinstance(extralines, dict):
432 for name, value in extralines.iteritems():
433 bbappendlines.append((name, '=', value))
434 else:
435 # Do our best to split it
436 for line in extralines:
437 if line[-1] == '\n':
438 line = line[:-1]
439 splitline = line.split(maxsplit=2)
440 if len(splitline) == 3:
441 bbappendlines.append(tuple(splitline))
442 else:
443 raise Exception('Invalid extralines value passed')
444
445 def popline(varname):
446 for i in xrange(0, len(bbappendlines)):
447 if bbappendlines[i][0] == varname:
448 line = bbappendlines.pop(i)
449 return line
450 return None
451
452 def appendline(varname, op, value):
453 for i in xrange(0, len(bbappendlines)):
454 item = bbappendlines[i]
455 if item[0] == varname:
456 bbappendlines[i] = (item[0], item[1], item[2] + ' ' + value)
457 break
458 else:
459 bbappendlines.append((varname, op, value))
460
461 destsubdir = rd.getVar('PN', True)
462 if srcfiles:
463 bbappendlines.append(('FILESEXTRAPATHS_prepend', ':=', '${THISDIR}/${PN}:'))
464
465 appendoverride = ''
466 if machine:
467 bbappendlines.append(('PACKAGE_ARCH', '=', '${MACHINE_ARCH}'))
468 appendoverride = '_%s' % machine
469 copyfiles = {}
470 if srcfiles:
471 instfunclines = []
472 for newfile, origsrcfile in srcfiles.iteritems():
473 srcfile = origsrcfile
474 srcurientry = None
475 if not srcfile:
476 srcfile = os.path.basename(newfile)
477 srcurientry = 'file://%s' % srcfile
478 # Double-check it's not there already
479 # FIXME do we care if the entry is added by another bbappend that might go away?
480 if not srcurientry in rd.getVar('SRC_URI', True).split():
481 if machine:
482 appendline('SRC_URI_append%s' % appendoverride, '=', ' ' + srcurientry)
483 else:
484 appendline('SRC_URI', '+=', srcurientry)
485 copyfiles[newfile] = srcfile
486 if install:
487 institem = install.pop(newfile, None)
488 if institem:
489 (destpath, perms) = institem
490 instdestpath = replace_dir_vars(destpath, rd)
491 instdirline = 'install -d ${D}%s' % os.path.dirname(instdestpath)
492 if not instdirline in instfunclines:
493 instfunclines.append(instdirline)
494 instfunclines.append('install -m %s ${WORKDIR}/%s ${D}%s' % (perms, os.path.basename(srcfile), instdestpath))
495 if instfunclines:
496 bbappendlines.append(('do_install_append%s()' % appendoverride, '', instfunclines))
497
498 bb.note('Writing append file %s' % appendpath)
499
500 if os.path.exists(appendpath):
501 # Work around lack of nonlocal in python 2
502 extvars = {'destsubdir': destsubdir}
503
504 def appendfile_varfunc(varname, origvalue, op, newlines):
505 if varname == 'FILESEXTRAPATHS_prepend':
506 if origvalue.startswith('${THISDIR}/'):
507 popline('FILESEXTRAPATHS_prepend')
508 extvars['destsubdir'] = rd.expand(origvalue.split('${THISDIR}/', 1)[1].rstrip(':'))
509 elif varname == 'PACKAGE_ARCH':
510 if machine:
511 popline('PACKAGE_ARCH')
512 return (machine, None, 4, False)
513 elif varname.startswith('do_install_append'):
514 func = popline(varname)
515 if func:
516 instfunclines = [line.strip() for line in origvalue.strip('\n').splitlines()]
517 for line in func[2]:
518 if not line in instfunclines:
519 instfunclines.append(line)
520 return (instfunclines, None, 4, False)
521 else:
522 splitval = origvalue.split()
523 changed = False
524 removevar = varname
525 if varname in ['SRC_URI', 'SRC_URI_append%s' % appendoverride]:
526 removevar = 'SRC_URI'
527 line = popline(varname)
528 if line:
529 if line[2] not in splitval:
530 splitval.append(line[2])
531 changed = True
532 else:
533 line = popline(varname)
534 if line:
535 splitval = [line[2]]
536 changed = True
537
538 if removevar in removevalues:
539 remove = removevalues[removevar]
540 if isinstance(remove, basestring):
541 if remove in splitval:
542 splitval.remove(remove)
543 changed = True
544 else:
545 for removeitem in remove:
546 if removeitem in splitval:
547 splitval.remove(removeitem)
548 changed = True
549
550 if changed:
551 newvalue = splitval
552 if len(newvalue) == 1:
553 # Ensure it's written out as one line
554 if '_append' in varname:
555 newvalue = ' ' + newvalue[0]
556 else:
557 newvalue = newvalue[0]
558 if not newvalue and (op in ['+=', '.='] or '_append' in varname):
559 # There's no point appending nothing
560 newvalue = None
561 if varname.endswith('()'):
562 indent = 4
563 else:
564 indent = -1
565 return (newvalue, None, indent, True)
566 return (origvalue, None, 4, False)
567
568 varnames = [item[0] for item in bbappendlines]
569 if removevalues:
570 varnames.extend(removevalues.keys())
571
572 with open(appendpath, 'r') as f:
573 (updated, newlines) = bb.utils.edit_metadata(f, varnames, appendfile_varfunc)
574
575 destsubdir = extvars['destsubdir']
576 else:
577 updated = False
578 newlines = []
579
580 if bbappendlines:
581 for line in bbappendlines:
582 if line[0].endswith('()'):
583 newlines.append('%s {\n %s\n}\n' % (line[0], '\n '.join(line[2])))
584 else:
585 newlines.append('%s %s "%s"\n\n' % line)
586 updated = True
587
588 if updated:
589 with open(appendpath, 'w') as f:
590 f.writelines(newlines)
591
592 if copyfiles:
593 if machine:
594 destsubdir = os.path.join(destsubdir, machine)
595 for newfile, srcfile in copyfiles.iteritems():
596 filedest = os.path.join(appenddir, destsubdir, os.path.basename(srcfile))
597 if os.path.abspath(newfile) != os.path.abspath(filedest):
598 bb.note('Copying %s to %s' % (newfile, filedest))
599 bb.utils.mkdirhier(os.path.dirname(filedest))
600 shutil.copyfile(newfile, filedest)
601
602 return (appendpath, os.path.join(appenddir, destsubdir))
603
604
605def find_layerdir(fn):
606 """ Figure out relative path to base of layer for a file (e.g. a recipe)"""
607 pth = os.path.dirname(fn)
608 layerdir = ''
609 while pth:
610 if os.path.exists(os.path.join(pth, 'conf', 'layer.conf')):
611 layerdir = pth
612 break
613 pth = os.path.dirname(pth)
614 return layerdir
615
616
617def replace_dir_vars(path, d):
618 """Replace common directory paths with appropriate variable references (e.g. /etc becomes ${sysconfdir})"""
619 dirvars = {}
620 for var in d:
621 if var.endswith('dir') and var.lower() == var:
622 value = d.getVar(var, True)
623 if value.startswith('/') and not '\n' in value:
624 dirvars[value] = var
625 for dirpath in sorted(dirvars.keys(), reverse=True):
626 path = path.replace(dirpath, '${%s}' % dirvars[dirpath])
627 return path
628
diff --git a/meta/lib/oeqa/selftest/devtool.py b/meta/lib/oeqa/selftest/devtool.py
index f4571c4ef1..ad10af5826 100644
--- a/meta/lib/oeqa/selftest/devtool.py
+++ b/meta/lib/oeqa/selftest/devtool.py
@@ -8,7 +8,7 @@ import glob
8 8
9import oeqa.utils.ftools as ftools 9import oeqa.utils.ftools as ftools
10from oeqa.selftest.base import oeSelfTest 10from oeqa.selftest.base import oeSelfTest
11from oeqa.utils.commands import runCmd, bitbake, get_bb_var 11from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
12from oeqa.utils.decorators import testcase 12from oeqa.utils.decorators import testcase
13 13
14class DevtoolBase(oeSelfTest): 14class DevtoolBase(oeSelfTest):
@@ -31,6 +31,35 @@ class DevtoolBase(oeSelfTest):
31 for inherit in checkinherits: 31 for inherit in checkinherits:
32 self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit) 32 self.assertIn(inherit, inherits, 'Missing inherit of %s' % inherit)
33 33
34 def _check_bbappend(self, testrecipe, recipefile, appenddir):
35 result = runCmd('bitbake-layers show-appends', cwd=self.builddir)
36 resultlines = result.output.splitlines()
37 inrecipe = False
38 bbappends = []
39 bbappendfile = None
40 for line in resultlines:
41 if inrecipe:
42 if line.startswith(' '):
43 bbappends.append(line.strip())
44 else:
45 break
46 elif line == '%s:' % os.path.basename(recipefile):
47 inrecipe = True
48 self.assertLessEqual(len(bbappends), 2, '%s recipe is being bbappended by another layer - bbappends found:\n %s' % (testrecipe, '\n '.join(bbappends)))
49 for bbappend in bbappends:
50 if bbappend.startswith(appenddir):
51 bbappendfile = bbappend
52 break
53 else:
54 self.assertTrue(False, 'bbappend for recipe %s does not seem to be created in test layer' % testrecipe)
55 return bbappendfile
56
57 def _create_temp_layer(self, templayerdir, addlayer, templayername, priority=999, recipepathspec='recipes-*/*'):
58 create_temp_layer(templayerdir, templayername, priority, recipepathspec)
59 if addlayer:
60 self.add_command_to_tearDown('bitbake-layers remove-layer %s || true' % templayerdir)
61 result = runCmd('bitbake-layers add-layer %s' % templayerdir, cwd=self.builddir)
62
34 63
35class DevtoolTests(DevtoolBase): 64class DevtoolTests(DevtoolBase):
36 65
diff --git a/meta/lib/oeqa/selftest/recipetool.py b/meta/lib/oeqa/selftest/recipetool.py
index 832fb7b16a..f3ad493457 100644
--- a/meta/lib/oeqa/selftest/recipetool.py
+++ b/meta/lib/oeqa/selftest/recipetool.py
@@ -6,16 +6,326 @@ import tempfile
6 6
7import oeqa.utils.ftools as ftools 7import oeqa.utils.ftools as ftools
8from oeqa.selftest.base import oeSelfTest 8from oeqa.selftest.base import oeSelfTest
9from oeqa.utils.commands import runCmd, bitbake, get_bb_var 9from oeqa.utils.commands import runCmd, bitbake, get_bb_var, create_temp_layer
10from oeqa.utils.decorators import testcase 10from oeqa.utils.decorators import testcase
11from oeqa.selftest.devtool import DevtoolBase 11from oeqa.selftest.devtool import DevtoolBase
12 12
13 13
14templayerdir = ''
15
16def setUpModule():
17 global templayerdir
18 templayerdir = tempfile.mkdtemp(prefix='recipetoolqa')
19 create_temp_layer(templayerdir, 'selftestrecipetool')
20 result = runCmd('bitbake-layers add-layer %s' % templayerdir)
21 # Ensure we have the right data in shlibs/pkgdata
22 logger = logging.getLogger("selftest")
23 logger.info('Running bitbake to generate pkgdata')
24 bitbake('base-files coreutils busybox selftest-recipetool-appendfile')
25
26def tearDownModule():
27 runCmd('bitbake-layers remove-layer %s' % templayerdir, ignore_status=True)
28 runCmd('rm -rf %s' % templayerdir)
29 # Shouldn't leave any traces of this artificial recipe behind
30 bitbake('-c cleansstate selftest-recipetool-appendfile')
31
32
14class RecipetoolTests(DevtoolBase): 33class RecipetoolTests(DevtoolBase):
15 34
16 def setUpLocal(self): 35 def setUpLocal(self):
17 self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa') 36 self.tempdir = tempfile.mkdtemp(prefix='recipetoolqa')
18 self.track_for_cleanup(self.tempdir) 37 self.track_for_cleanup(self.tempdir)
38 self.testfile = os.path.join(self.tempdir, 'testfile')
39 with open(self.testfile, 'w') as f:
40 f.write('Test file\n')
41
42 def tearDownLocal(self):
43 runCmd('rm -rf %s/recipes-*' % templayerdir)
44
45 def _try_recipetool_appendfile(self, testrecipe, destfile, newfile, options, expectedlines, expectedfiles):
46 result = runCmd('recipetool appendfile %s %s %s %s' % (templayerdir, destfile, newfile, options))
47 self.assertNotIn('Traceback', result.output)
48 # Check the bbappend was created and applies properly
49 recipefile = get_bb_var('FILE', testrecipe)
50 bbappendfile = self._check_bbappend(testrecipe, recipefile, templayerdir)
51 # Check the bbappend contents
52 with open(bbappendfile, 'r') as f:
53 self.assertEqual(expectedlines, f.readlines())
54 # Check file was copied
55 filesdir = os.path.join(os.path.dirname(bbappendfile), testrecipe)
56 for expectedfile in expectedfiles:
57 self.assertTrue(os.path.isfile(os.path.join(filesdir, expectedfile)), 'Expected file %s to be copied next to bbappend, but it wasn\'t' % expectedfile)
58 # Check no other files created
59 createdfiles = []
60 for root, _, files in os.walk(filesdir):
61 for f in files:
62 createdfiles.append(os.path.relpath(os.path.join(root, f), filesdir))
63 self.assertTrue(sorted(createdfiles), sorted(expectedfiles))
64 return bbappendfile, result.output
65
66 def _try_recipetool_appendfile_fail(self, destfile, newfile, checkerror):
67 cmd = 'recipetool appendfile %s %s %s' % (templayerdir, destfile, newfile)
68 result = runCmd(cmd, ignore_status=True)
69 self.assertNotEqual(result.status, 0, 'Command "%s" should have failed but didn\'t' % cmd)
70 self.assertNotIn('Traceback', result.output)
71 for errorstr in checkerror:
72 self.assertIn(errorstr, result.output)
73
74
75 def test_recipetool_appendfile_basic(self):
76 # Basic test
77 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
78 '\n']
79 _, output = self._try_recipetool_appendfile('base-files', '/etc/motd', self.testfile, '', expectedlines, ['motd'])
80 self.assertNotIn('WARNING: ', output)
81
82 def test_recipetool_appendfile_invalid(self):
83 # Test some commands that should error
84 self._try_recipetool_appendfile_fail('/etc/passwd', self.testfile, ['ERROR: /etc/passwd cannot be handled by this tool', 'useradd', 'extrausers'])
85 self._try_recipetool_appendfile_fail('/etc/timestamp', self.testfile, ['ERROR: /etc/timestamp cannot be handled by this tool'])
86 self._try_recipetool_appendfile_fail('/dev/console', self.testfile, ['ERROR: /dev/console cannot be handled by this tool'])
87
88 def test_recipetool_appendfile_alternatives(self):
89 # Now try with a file we know should be an alternative
90 # (this is very much a fake example, but one we know is reliably an alternative)
91 self._try_recipetool_appendfile_fail('/bin/ls', self.testfile, ['ERROR: File /bin/ls is an alternative possibly provided by the following recipes:', 'coreutils', 'busybox'])
92 corebase = get_bb_var('COREBASE')
93 # Need a test file - should be executable
94 testfile2 = os.path.join(corebase, 'oe-init-build-env')
95 testfile2name = os.path.basename(testfile2)
96 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
97 '\n',
98 'SRC_URI += "file://%s"\n' % testfile2name,
99 '\n',
100 'do_install_append() {\n',
101 ' install -d ${D}${base_bindir}\n',
102 ' install -m 0755 ${WORKDIR}/%s ${D}${base_bindir}/ls\n' % testfile2name,
103 '}\n']
104 self._try_recipetool_appendfile('coreutils', '/bin/ls', testfile2, '-r coreutils', expectedlines, [testfile2name])
105 # Now try bbappending the same file again, contents should not change
106 bbappendfile, _ = self._try_recipetool_appendfile('coreutils', '/bin/ls', self.testfile, '-r coreutils', expectedlines, [testfile2name])
107 # But file should have
108 copiedfile = os.path.join(os.path.dirname(bbappendfile), 'coreutils', testfile2name)
109 result = runCmd('diff -q %s %s' % (testfile2, copiedfile), ignore_status=True)
110 self.assertNotEqual(result.status, 0, 'New file should have been copied but was not')
111
112 def test_recipetool_appendfile_binary(self):
113 # Try appending a binary file
114 result = runCmd('recipetool appendfile %s /bin/ls /bin/ls -r coreutils' % templayerdir)
115 self.assertIn('WARNING: ', result.output)
116 self.assertIn('is a binary', result.output)
117
118 def test_recipetool_appendfile_add(self):
119 corebase = get_bb_var('COREBASE')
120 # Try arbitrary file add to a recipe
121 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
122 '\n',
123 'SRC_URI += "file://testfile"\n',
124 '\n',
125 'do_install_append() {\n',
126 ' install -d ${D}${datadir}\n',
127 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
128 '}\n']
129 self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase', expectedlines, ['testfile'])
130 # Try adding another file, this time where the source file is executable
131 # (so we're testing that, plus modifying an existing bbappend)
132 testfile2 = os.path.join(corebase, 'oe-init-build-env')
133 testfile2name = os.path.basename(testfile2)
134 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
135 '\n',
136 'SRC_URI += "file://testfile \\\n',
137 ' file://%s \\\n' % testfile2name,
138 ' "\n',
139 '\n',
140 'do_install_append() {\n',
141 ' install -d ${D}${datadir}\n',
142 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
143 ' install -m 0755 ${WORKDIR}/%s ${D}${datadir}/scriptname\n' % testfile2name,
144 '}\n']
145 self._try_recipetool_appendfile('netbase', '/usr/share/scriptname', testfile2, '-r netbase', expectedlines, ['testfile', testfile2name])
146
147 def test_recipetool_appendfile_add_bindir(self):
148 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
149 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
150 '\n',
151 'SRC_URI += "file://testfile"\n',
152 '\n',
153 'do_install_append() {\n',
154 ' install -d ${D}${bindir}\n',
155 ' install -m 0755 ${WORKDIR}/testfile ${D}${bindir}/selftest-recipetool-testbin\n',
156 '}\n']
157 _, output = self._try_recipetool_appendfile('netbase', '/usr/bin/selftest-recipetool-testbin', self.testfile, '-r netbase', expectedlines, ['testfile'])
158 self.assertNotIn('WARNING: ', output)
159
160 def test_recipetool_appendfile_add_machine(self):
161 # Try arbitrary file add to a recipe, this time to a location such that should be installed as executable
162 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
163 '\n',
164 'PACKAGE_ARCH = "${MACHINE_ARCH}"\n',
165 '\n',
166 'SRC_URI_append_mymachine = " file://testfile"\n',
167 '\n',
168 'do_install_append_mymachine() {\n',
169 ' install -d ${D}${datadir}\n',
170 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/something\n',
171 '}\n']
172 _, output = self._try_recipetool_appendfile('netbase', '/usr/share/something', self.testfile, '-r netbase -m mymachine', expectedlines, ['mymachine/testfile'])
173 self.assertNotIn('WARNING: ', output)
174
175 def test_recipetool_appendfile_orig(self):
176 # A file that's in SRC_URI and in do_install with the same name
177 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
178 '\n']
179 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-orig', self.testfile, '', expectedlines, ['selftest-replaceme-orig'])
180 self.assertNotIn('WARNING: ', output)
181
182 def test_recipetool_appendfile_todir(self):
183 # A file that's in SRC_URI and in do_install with destination directory rather than file
184 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
185 '\n']
186 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-todir', self.testfile, '', expectedlines, ['selftest-replaceme-todir'])
187 self.assertNotIn('WARNING: ', output)
188
189 def test_recipetool_appendfile_renamed(self):
190 # A file that's in SRC_URI with a different name to the destination file
191 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
192 '\n']
193 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-renamed', self.testfile, '', expectedlines, ['file1'])
194 self.assertNotIn('WARNING: ', output)
195
196 def test_recipetool_appendfile_subdir(self):
197 # A file that's in SRC_URI in a subdir
198 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
199 '\n',
200 'SRC_URI += "file://testfile"\n',
201 '\n',
202 'do_install_append() {\n',
203 ' install -d ${D}${datadir}\n',
204 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-subdir\n',
205 '}\n']
206 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-subdir', self.testfile, '', expectedlines, ['testfile'])
207 self.assertNotIn('WARNING: ', output)
208
209 def test_recipetool_appendfile_src_glob(self):
210 # A file that's in SRC_URI as a glob
211 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
212 '\n',
213 'SRC_URI += "file://testfile"\n',
214 '\n',
215 'do_install_append() {\n',
216 ' install -d ${D}${datadir}\n',
217 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-src-globfile\n',
218 '}\n']
219 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-src-globfile', self.testfile, '', expectedlines, ['testfile'])
220 self.assertNotIn('WARNING: ', output)
221
222 def test_recipetool_appendfile_inst_glob(self):
223 # A file that's in do_install as a glob
224 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
225 '\n']
226 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-globfile'])
227 self.assertNotIn('WARNING: ', output)
228
229 def test_recipetool_appendfile_inst_todir_glob(self):
230 # A file that's in do_install as a glob with destination as a directory
231 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
232 '\n']
233 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-todir-globfile', self.testfile, '', expectedlines, ['selftest-replaceme-inst-todir-globfile'])
234 self.assertNotIn('WARNING: ', output)
235
236 def test_recipetool_appendfile_patch(self):
237 # A file that's added by a patch in SRC_URI
238 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
239 '\n',
240 'SRC_URI += "file://testfile"\n',
241 '\n',
242 'do_install_append() {\n',
243 ' install -d ${D}${sysconfdir}\n',
244 ' install -m 0644 ${WORKDIR}/testfile ${D}${sysconfdir}/selftest-replaceme-patched\n',
245 '}\n']
246 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/etc/selftest-replaceme-patched', self.testfile, '', expectedlines, ['testfile'])
247 for line in output.splitlines():
248 if line.startswith('WARNING: '):
249 self.assertIn('add-file.patch', line, 'Unexpected warning found in output:\n%s' % line)
250 break
251 else:
252 self.assertTrue(False, 'Patch warning not found in output:\n%s' % output)
253
254 def test_recipetool_appendfile_script(self):
255 # Now, a file that's in SRC_URI but installed by a script (so no mention in do_install)
256 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
257 '\n',
258 'SRC_URI += "file://testfile"\n',
259 '\n',
260 'do_install_append() {\n',
261 ' install -d ${D}${datadir}\n',
262 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-scripted\n',
263 '}\n']
264 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-scripted', self.testfile, '', expectedlines, ['testfile'])
265 self.assertNotIn('WARNING: ', output)
266
267 def test_recipetool_appendfile_inst_func(self):
268 # A file that's installed from a function called by do_install
269 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
270 '\n']
271 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-inst-func', self.testfile, '', expectedlines, ['selftest-replaceme-inst-func'])
272 self.assertNotIn('WARNING: ', output)
273
274 def test_recipetool_appendfile_postinstall(self):
275 # A file that's created by a postinstall script (and explicitly mentioned in it)
276 # First try without specifying recipe
277 self._try_recipetool_appendfile_fail('/usr/share/selftest-replaceme-postinst', self.testfile, ['File /usr/share/selftest-replaceme-postinst may be written out in a pre/postinstall script of the following recipes:', 'selftest-recipetool-appendfile'])
278 # Now specify recipe
279 expectedlines = ['FILESEXTRAPATHS_prepend := "${THISDIR}/${PN}:"\n',
280 '\n',
281 'SRC_URI += "file://testfile"\n',
282 '\n',
283 'do_install_append() {\n',
284 ' install -d ${D}${datadir}\n',
285 ' install -m 0644 ${WORKDIR}/testfile ${D}${datadir}/selftest-replaceme-postinst\n',
286 '}\n']
287 _, output = self._try_recipetool_appendfile('selftest-recipetool-appendfile', '/usr/share/selftest-replaceme-postinst', self.testfile, '-r selftest-recipetool-appendfile', expectedlines, ['testfile'])
288
289 def test_recipetool_appendfile_extlayer(self):
290 # Try creating a bbappend in a layer that's not in bblayers.conf and has a different structure
291 exttemplayerdir = os.path.join(self.tempdir, 'extlayer')
292 self._create_temp_layer(exttemplayerdir, False, 'oeselftestextlayer', recipepathspec='metadata/recipes/recipes-*/*')
293 result = runCmd('recipetool appendfile %s /usr/share/selftest-replaceme-orig %s' % (exttemplayerdir, self.testfile))
294 self.assertNotIn('Traceback', result.output)
295 createdfiles = []
296 for root, _, files in os.walk(exttemplayerdir):
297 for f in files:
298 createdfiles.append(os.path.relpath(os.path.join(root, f), exttemplayerdir))
299 createdfiles.remove('conf/layer.conf')
300 expectedfiles = ['metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile.bbappend',
301 'metadata/recipes/recipes-test/selftest-recipetool-appendfile/selftest-recipetool-appendfile/selftest-replaceme-orig']
302 self.assertEqual(sorted(createdfiles), sorted(expectedfiles))
303
304 def test_recipetool_appendfile_wildcard(self):
305
306 def try_appendfile_wc(options):
307 result = runCmd('recipetool appendfile %s /etc/profile %s %s' % (templayerdir, self.testfile, options))
308 self.assertNotIn('Traceback', result.output)
309 bbappendfile = None
310 for root, _, files in os.walk(templayerdir):
311 for f in files:
312 if f.endswith('.bbappend'):
313 bbappendfile = f
314 break
315 if not bbappendfile:
316 self.assertTrue(False, 'No bbappend file created')
317 runCmd('rm -rf %s/recipes-*' % templayerdir)
318 return bbappendfile
319
320 # Check without wildcard option
321 recipefn = os.path.basename(get_bb_var('FILE', 'base-files'))
322 filename = try_appendfile_wc('')
323 self.assertEqual(filename, recipefn.replace('.bb', '.bbappend'))
324 # Now check with wildcard option
325 filename = try_appendfile_wc('-w')
326 self.assertEqual(filename, recipefn.split('_')[0] + '_%.bbappend')
327
328
19 329
20 def test_recipetool_create(self): 330 def test_recipetool_create(self):
21 # Try adding a recipe 331 # Try adding a recipe
@@ -52,4 +362,3 @@ class RecipetoolTests(DevtoolBase):
52 checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg' 362 checkvars['DEPENDS'] = 'libpng pango libx11 libxext jpeg'
53 inherits = ['autotools', 'pkgconfig'] 363 inherits = ['autotools', 'pkgconfig']
54 self._test_recipe_contents(recipefile, checkvars, inherits) 364 self._test_recipe_contents(recipefile, checkvars, inherits)
55
diff --git a/meta/lib/oeqa/utils/commands.py b/meta/lib/oeqa/utils/commands.py
index 663e4e7f41..dc8a9836e7 100644
--- a/meta/lib/oeqa/utils/commands.py
+++ b/meta/lib/oeqa/utils/commands.py
@@ -162,3 +162,14 @@ def get_test_layer():
162 testlayer = l 162 testlayer = l
163 break 163 break
164 return testlayer 164 return testlayer
165
166def create_temp_layer(templayerdir, templayername, priority=999, recipepathspec='recipes-*/*'):
167 os.makedirs(os.path.join(templayerdir, 'conf'))
168 with open(os.path.join(templayerdir, 'conf', 'layer.conf'), 'w') as f:
169 f.write('BBPATH .= ":${LAYERDIR}"\n')
170 f.write('BBFILES += "${LAYERDIR}/%s/*.bb \\' % recipepathspec)
171 f.write(' ${LAYERDIR}/%s/*.bbappend"\n' % recipepathspec)
172 f.write('BBFILE_COLLECTIONS += "%s"\n' % templayername)
173 f.write('BBFILE_PATTERN_%s = "^${LAYERDIR}/"\n' % templayername)
174 f.write('BBFILE_PRIORITY_%s = "%d"\n' % (templayername, priority))
175 f.write('BBFILE_PATTERN_IGNORE_EMPTY_%s = "1"\n' % templayername)
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
new file mode 100644
index 0000000000..39117c1f66
--- /dev/null
+++ b/scripts/lib/recipetool/append.py
@@ -0,0 +1,360 @@
1# Recipe creation tool - append plugin
2#
3# Copyright (C) 2015 Intel Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import sys
19import os
20import argparse
21import glob
22import fnmatch
23import re
24import subprocess
25import logging
26import stat
27import shutil
28import scriptutils
29import errno
30from collections import defaultdict
31
32logger = logging.getLogger('recipetool')
33
34tinfoil = None
35
36def plugin_init(pluginlist):
37 # Don't need to do anything here right now, but plugins must have this function defined
38 pass
39
40def tinfoil_init(instance):
41 global tinfoil
42 tinfoil = instance
43
44
45# FIXME guessing when we don't have pkgdata?
46# FIXME mode to create patch rather than directly substitute
47
48class InvalidTargetFileError(Exception):
49 pass
50
51def find_target_file(targetpath, d, pkglist=None):
52 """Find the recipe installing the specified target path, optionally limited to a select list of packages"""
53 import json
54
55 pkgdata_dir = d.getVar('PKGDATA_DIR', True)
56
57 # The mix between /etc and ${sysconfdir} here may look odd, but it is just
58 # being consistent with usage elsewhere
59 invalidtargets = {'${sysconfdir}/version': '${sysconfdir}/version is written out at image creation time',
60 '/etc/timestamp': '/etc/timestamp is written out at image creation time',
61 '/dev/*': '/dev is handled by udev (or equivalent) and the kernel (devtmpfs)',
62 '/etc/passwd': '/etc/passwd should be managed through the useradd and extrausers classes',
63 '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
64 '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
65 '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
66 '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
67
68 for pthspec, message in invalidtargets.iteritems():
69 if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
70 raise InvalidTargetFileError(d.expand(message))
71
72 targetpath_re = re.compile(r'\s+(\$D)?%s(\s|$)' % targetpath)
73
74 recipes = defaultdict(list)
75 for root, dirs, files in os.walk(os.path.join(pkgdata_dir, 'runtime')):
76 if pkglist:
77 filelist = pkglist
78 else:
79 filelist = files
80 for fn in filelist:
81 pkgdatafile = os.path.join(root, fn)
82 if pkglist and not os.path.exists(pkgdatafile):
83 continue
84 with open(pkgdatafile, 'r') as f:
85 pn = ''
86 # This does assume that PN comes before other values, but that's a fairly safe assumption
87 for line in f:
88 if line.startswith('PN:'):
89 pn = line.split(':', 1)[1].strip()
90 elif line.startswith('FILES_INFO:'):
91 val = line.split(':', 1)[1].strip()
92 dictval = json.loads(val)
93 for fullpth in dictval.keys():
94 if fnmatch.fnmatchcase(fullpth, targetpath):
95 recipes[targetpath].append(pn)
96 elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
97 scriptval = line.split(':', 1)[1].strip().decode('string_escape')
98 if 'update-alternatives --install %s ' % targetpath in scriptval:
99 recipes[targetpath].append('?%s' % pn)
100 elif targetpath_re.search(scriptval):
101 recipes[targetpath].append('!%s' % pn)
102 return recipes
103
104def _get_recipe_file(cooker, pn):
105 import oe.recipeutils
106 recipefile = oe.recipeutils.pn_to_recipe(cooker, pn)
107 if not recipefile:
108 skipreasons = oe.recipeutils.get_unavailable_reasons(cooker, pn)
109 if skipreasons:
110 logger.error('\n'.join(skipreasons))
111 else:
112 logger.error("Unable to find any recipe file matching %s" % pn)
113 return recipefile
114
115def _parse_recipe(pn, tinfoil):
116 import oe.recipeutils
117 recipefile = _get_recipe_file(tinfoil.cooker, pn)
118 if not recipefile:
119 # Error already logged
120 return None
121 append_files = tinfoil.cooker.collection.get_file_appends(recipefile)
122 rd = oe.recipeutils.parse_recipe(recipefile, append_files,
123 tinfoil.config_data)
124 return rd
125
126def determine_file_source(targetpath, rd):
127 """Assuming we know a file came from a specific recipe, figure out exactly where it came from"""
128 import oe.recipeutils
129
130 # See if it's in do_install for the recipe
131 workdir = rd.getVar('WORKDIR', True)
132 src_uri = rd.getVar('SRC_URI', True)
133 srcfile = ''
134 modpatches = []
135 elements = check_do_install(rd, targetpath)
136 if elements:
137 logger.debug('do_install line:\n%s' % ' '.join(elements))
138 srcpath = get_source_path(elements)
139 logger.debug('source path: %s' % srcpath)
140 if not srcpath.startswith('/'):
141 # Handle non-absolute path
142 srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs', True).split()[-1], srcpath))
143 if srcpath.startswith(workdir):
144 # OK, now we have the source file name, look for it in SRC_URI
145 workdirfile = os.path.relpath(srcpath, workdir)
146 # FIXME this is where we ought to have some code in the fetcher, because this is naive
147 for item in src_uri.split():
148 localpath = bb.fetch2.localpath(item, rd)
149 # Source path specified in do_install might be a glob
150 if fnmatch.fnmatch(os.path.basename(localpath), workdirfile):
151 srcfile = 'file://%s' % localpath
152 elif '/' in workdirfile:
153 if item == 'file://%s' % workdirfile:
154 srcfile = 'file://%s' % localpath
155
156 # Check patches
157 srcpatches = []
158 patchedfiles = oe.recipeutils.get_recipe_patched_files(rd)
159 for patch, filelist in patchedfiles.iteritems():
160 for fileitem in filelist:
161 if fileitem[0] == srcpath:
162 srcpatches.append((patch, fileitem[1]))
163 if srcpatches:
164 addpatch = None
165 for patch in srcpatches:
166 if patch[1] == 'A':
167 addpatch = patch[0]
168 else:
169 modpatches.append(patch[0])
170 if addpatch:
171 srcfile = 'patch://%s' % addpatch
172
173 return (srcfile, elements, modpatches)
174
175def get_source_path(cmdelements):
176 """Find the source path specified within a command"""
177 command = cmdelements[0]
178 if command in ['install', 'cp']:
179 helptext = subprocess.check_output('LC_ALL=C %s --help' % command, shell=True)
180 argopts = ''
181 argopt_line_re = re.compile('^-([a-zA-Z0-9]), --[a-z-]+=')
182 for line in helptext.splitlines():
183 line = line.lstrip()
184 res = argopt_line_re.search(line)
185 if res:
186 argopts += res.group(1)
187 if not argopts:
188 # Fallback
189 if command == 'install':
190 argopts = 'gmoSt'
191 elif command == 'cp':
192 argopts = 't'
193 else:
194 raise Exception('No fallback arguments for command %s' % command)
195
196 skipnext = False
197 for elem in cmdelements[1:-1]:
198 if elem.startswith('-'):
199 if len(elem) > 1 and elem[1] in argopts:
200 skipnext = True
201 continue
202 if skipnext:
203 skipnext = False
204 continue
205 return elem
206 else:
207 raise Exception('get_source_path: no handling for command "%s"')
208
209def get_func_deps(func, d):
210 """Find the function dependencies of a shell function"""
211 deps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func, True))
212 deps |= set((d.getVarFlag(func, "vardeps", True) or "").split())
213 funcdeps = []
214 for dep in deps:
215 if d.getVarFlag(dep, 'func', True):
216 funcdeps.append(dep)
217 return funcdeps
218
219def check_do_install(rd, targetpath):
220 """Look at do_install for a command that installs/copies the specified target path"""
221 instpath = os.path.abspath(os.path.join(rd.getVar('D', True), targetpath.lstrip('/')))
222 do_install = rd.getVar('do_install', True)
223 # Handle where do_install calls other functions (somewhat crudely, but good enough for this purpose)
224 deps = get_func_deps('do_install', rd)
225 for dep in deps:
226 do_install = do_install.replace(dep, rd.getVar(dep, True))
227
228 # Look backwards through do_install as we want to catch where a later line (perhaps
229 # from a bbappend) is writing over the top
230 for line in reversed(do_install.splitlines()):
231 line = line.strip()
232 if (line.startswith('install ') and ' -m' in line) or line.startswith('cp '):
233 elements = line.split()
234 destpath = os.path.abspath(elements[-1])
235 if destpath == instpath:
236 return elements
237 elif destpath.rstrip('/') == os.path.dirname(instpath):
238 # FIXME this doesn't take recursive copy into account; unsure if it's practical to do so
239 srcpath = get_source_path(elements)
240 if fnmatch.fnmatchcase(os.path.basename(instpath), os.path.basename(srcpath)):
241 return elements
242 return None
243
244
245def appendfile(args):
246 import oe.recipeutils
247
248 if not args.targetpath.startswith('/'):
249 logger.error('Target path should start with /')
250 return 2
251
252 if os.path.isdir(args.newfile):
253 logger.error('Specified new file "%s" is a directory' % args.newfile)
254 return 2
255
256 if not os.path.exists(args.destlayer):
257 logger.error('Destination layer directory "%s" does not exist' % args.destlayer)
258 return 2
259 if not os.path.exists(os.path.join(args.destlayer, 'conf', 'layer.conf')):
260 logger.error('conf/layer.conf not found in destination layer "%s"' % args.destlayer)
261 return 2
262
263 stdout = ''
264 try:
265 (stdout, _) = bb.process.run('LANG=C file -E -b %s' % args.newfile, shell=True)
266 except bb.process.ExecutionError as err:
267 logger.debug('file command returned error: %s' % err)
268 pass
269 if stdout:
270 logger.debug('file command output: %s' % stdout.rstrip())
271 if ('executable' in stdout and not 'shell script' in stdout) or 'shared object' in stdout:
272 logger.warn('This file looks like it is a binary or otherwise the output of compilation. If it is, you should consider building it properly instead of substituting a binary file directly.')
273
274 if args.recipe:
275 recipes = {args.targetpath: [args.recipe],}
276 else:
277 try:
278 recipes = find_target_file(args.targetpath, tinfoil.config_data)
279 except InvalidTargetFileError as e:
280 logger.error('%s cannot be handled by this tool: %s' % (args.targetpath, e))
281 return 1
282 if not recipes:
283 logger.error('Unable to find any package producing path %s - this may be because the recipe packaging it has not been built yet' % args.targetpath)
284 return 1
285
286 alternative_pns = []
287 postinst_pns = []
288
289 selectpn = None
290 for targetpath, pnlist in recipes.iteritems():
291 for pn in pnlist:
292 if pn.startswith('?'):
293 alternative_pns.append(pn[1:])
294 elif pn.startswith('!'):
295 postinst_pns.append(pn[1:])
296 else:
297 selectpn = pn
298
299 if not selectpn and len(alternative_pns) == 1:
300 selectpn = alternative_pns[0]
301 logger.error('File %s is an alternative possibly provided by recipe %s but seemingly no other, selecting it by default - you should double check other recipes' % (args.targetpath, selectpn))
302
303 if selectpn:
304 logger.debug('Selecting recipe %s for file %s' % (selectpn, args.targetpath))
305 if postinst_pns:
306 logger.warn('%s be modified by postinstall scripts for the following recipes:\n %s\nThis may or may not be an issue depending on what modifications these postinstall scripts make.' % (args.targetpath, '\n '.join(postinst_pns)))
307 rd = _parse_recipe(selectpn, tinfoil)
308 if not rd:
309 # Error message already shown
310 return 1
311 sourcefile, instelements, modpatches = determine_file_source(args.targetpath, rd)
312 sourcepath = None
313 if sourcefile:
314 sourcetype, sourcepath = sourcefile.split('://', 1)
315 logger.debug('Original source file is %s (%s)' % (sourcepath, sourcetype))
316 if sourcetype == 'patch':
317 logger.warn('File %s is added by the patch %s - you may need to remove or replace this patch in order to replace the file.' % (args.targetpath, sourcepath))
318 sourcepath = None
319 else:
320 logger.debug('Unable to determine source file, proceeding anyway')
321 if modpatches:
322 logger.warn('File %s is modified by the following patches:\n %s' % (args.targetpath, '\n '.join(modpatches)))
323
324 if instelements and sourcepath:
325 install = None
326 else:
327 # Auto-determine permissions
328 # Check destination
329 binpaths = '${bindir}:${sbindir}:${base_bindir}:${base_sbindir}:${libexecdir}:${sysconfdir}/init.d'
330 perms = '0644'
331 if os.path.abspath(os.path.dirname(args.targetpath)) in rd.expand(binpaths).split(':'):
332 # File is going into a directory normally reserved for executables, so it should be executable
333 perms = '0755'
334 else:
335 # Check source
336 st = os.stat(args.newfile)
337 if st.st_mode & stat.S_IXUSR:
338 perms = '0755'
339 install = {args.newfile: (args.targetpath, perms)}
340 oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
341 return 0
342 else:
343 if alternative_pns:
344 logger.error('File %s is an alternative possibly provided by the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(alternative_pns)))
345 elif postinst_pns:
346 logger.error('File %s may be written out in a pre/postinstall script of the following recipes:\n %s\nPlease select recipe with -r/--recipe' % (targetpath, '\n '.join(postinst_pns)))
347 return 3
348
349
350def register_command(subparsers):
351 parser_appendfile = subparsers.add_parser('appendfile',
352 help='Create a bbappend to replace a file',
353 description='')
354 parser_appendfile.add_argument('destlayer', help='Destination layer to write the bbappend to')
355 parser_appendfile.add_argument('targetpath', help='Path within the image to the file to be replaced')
356 parser_appendfile.add_argument('newfile', help='Custom file to replace it with')
357 parser_appendfile.add_argument('-r', '--recipe', help='Override recipe to apply to (default is to find which recipe already packages it)')
358 parser_appendfile.add_argument('-m', '--machine', help='Make bbappend changes specific to a machine only', metavar='MACHINE')
359 parser_appendfile.add_argument('-w', '--wildcard-version', help='Use wildcard to make the bbappend apply to any recipe version', action='store_true')
360 parser_appendfile.set_defaults(func=appendfile, parserecipes=True)
diff --git a/scripts/recipetool b/scripts/recipetool
index b7d3ee887c..c68bef4c96 100755
--- a/scripts/recipetool
+++ b/scripts/recipetool
@@ -31,11 +31,11 @@ logger = scriptutils.logger_create('recipetool')
31 31
32plugins = [] 32plugins = []
33 33
34def tinfoil_init(): 34def tinfoil_init(parserecipes):
35 import bb.tinfoil 35 import bb.tinfoil
36 import logging 36 import logging
37 tinfoil = bb.tinfoil.Tinfoil() 37 tinfoil = bb.tinfoil.Tinfoil()
38 tinfoil.prepare(True) 38 tinfoil.prepare(not parserecipes)
39 39
40 for plugin in plugins: 40 for plugin in plugins:
41 if hasattr(plugin, 'tinfoil_init'): 41 if hasattr(plugin, 'tinfoil_init'):
@@ -82,7 +82,7 @@ def main():
82 82
83 scriptutils.logger_setup_color(logger, args.color) 83 scriptutils.logger_setup_color(logger, args.color)
84 84
85 tinfoil_init() 85 tinfoil_init(getattr(args, 'parserecipes', False))
86 86
87 ret = args.func(args) 87 ret = args.func(args)
88 88