summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/recipeutils.py
blob: 0689fb0c714af7474ff3a52eb5258571321b4e6e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
# Utility functions for reading and modifying recipes
#
# Some code borrowed from the OE layer index
#
# Copyright (C) 2013-2014 Intel Corporation
#

import sys
import os
import os.path
import tempfile
import textwrap
import difflib
import utils
import shutil
import re
from collections import OrderedDict, defaultdict


# Help us to find places to insert values
recipe_progression = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION', 'LICENSE', 'LIC_FILES_CHKSUM', 'PROVIDES', 'DEPENDS', 'PR', 'PV', 'SRCREV', 'SRC_URI', 'S', 'do_fetch', 'do_unpack', 'do_patch', 'EXTRA_OECONF', 'do_configure', 'EXTRA_OEMAKE', 'do_compile', 'do_install', 'do_populate_sysroot', 'INITSCRIPT', 'USERADD', 'GROUPADD', 'PACKAGES', 'FILES', 'RDEPENDS', 'RRECOMMENDS', 'RSUGGESTS', 'RPROVIDES', 'RREPLACES', 'RCONFLICTS', 'ALLOW_EMPTY', 'do_package', 'do_deploy']
# Variables that sometimes are a bit long but shouldn't be wrapped
nowrap_vars = ['SUMMARY', 'HOMEPAGE', 'BUGTRACKER']
list_vars = ['SRC_URI', 'LIC_FILES_CHKSUM']
meta_vars = ['SUMMARY', 'DESCRIPTION', 'HOMEPAGE', 'BUGTRACKER', 'SECTION']


def pn_to_recipe(cooker, pn):
    """Convert a recipe name (PN) to the path to the recipe file"""
    import bb.providers

    if pn in cooker.recipecache.pkg_pn:
        filenames = cooker.recipecache.pkg_pn[pn]
        best = bb.providers.findBestProvider(pn, cooker.data, cooker.recipecache, cooker.recipecache.pkg_pn)
        return best[3]
    else:
        return None


def get_unavailable_reasons(cooker, pn):
    """If a recipe could not be found, find out why if possible"""
    import bb.taskdata
    taskdata = bb.taskdata.TaskData(None, skiplist=cooker.skiplist)
    return taskdata.get_reasons(pn)


def parse_recipe(fn, appendfiles, d):
    """
    Parse an individual recipe file, optionally with a list of
    bbappend files.
    """
    import bb.cache
    envdata = bb.cache.Cache.loadDataFull(fn, appendfiles, d)
    return envdata


def parse_recipe_simple(cooker, pn, d, appends=True):
    """
    Parse a recipe and optionally all bbappends that apply to it
    in the current configuration.
    """
    import bb.providers

    recipefile = pn_to_recipe(cooker, pn)
    if not recipefile:
        skipreasons = get_unavailable_reasons(cooker, pn)
        # We may as well re-use bb.providers.NoProvider here
        if skipreasons:
            raise bb.providers.NoProvider(skipreasons)
        else:
            raise bb.providers.NoProvider('Unable to find any recipe file matching %s' % pn)
    if appends:
        appendfiles = cooker.collection.get_file_appends(recipefile)
    return parse_recipe(recipefile, appendfiles, d)


def get_var_files(fn, varlist, d):
    """Find the file in which each of a list of variables is set.
    Note: requires variable history to be enabled when parsing.
    """
    envdata = parse_recipe(fn, [], d)
    varfiles = {}
    for v in varlist:
        history = envdata.varhistory.variable(v)
        files = []
        for event in history:
            if 'file' in event and not 'flag' in event:
                files.append(event['file'])
        if files:
            actualfile = files[-1]
        else:
            actualfile = None
        varfiles[v] = actualfile

    return varfiles


def patch_recipe_file(fn, values, patch=False, relpath=''):
    """Update or insert variable values into a recipe file (assuming you
       have already identified the exact file you want to update.)
       Note that some manual inspection/intervention may be required
       since this cannot handle all situations.
    """
    remainingnames = {}
    for k in values.keys():
        remainingnames[k] = recipe_progression.index(k) if k in recipe_progression else -1
    remainingnames = OrderedDict(sorted(remainingnames.iteritems(), key=lambda x: x[1]))

    with tempfile.NamedTemporaryFile('w', delete=False) as tf:
        def outputvalue(name):
            rawtext = '%s = "%s"\n' % (name, values[name])
            if name in nowrap_vars:
                tf.write(rawtext)
            elif name in list_vars:
                splitvalue = values[name].split()
                if len(splitvalue) > 1:
                    linesplit = ' \\\n' + (' ' * (len(name) + 4))
                    tf.write('%s = "%s%s"\n' % (name, linesplit.join(splitvalue), linesplit))
                else:
                    tf.write(rawtext)
            else:
                wrapped = textwrap.wrap(rawtext)
                for wrapline in wrapped[:-1]:
                    tf.write('%s \\\n' % wrapline)
                tf.write('%s\n' % wrapped[-1])

        tfn = tf.name
        with open(fn, 'r') as f:
            # First runthrough - find existing names (so we know not to insert based on recipe_progression)
            # Second runthrough - make the changes
            existingnames = []
            for runthrough in [1, 2]:
                currname = None
                for line in f:
                    if not currname:
                        insert = False
                        for k in remainingnames.keys():
                            for p in recipe_progression:
                                if re.match('^%s(_prepend|_append)*[ ?:=(]' % p, line):
                                    if remainingnames[k] > -1 and recipe_progression.index(p) > remainingnames[k] and runthrough > 1 and not k in existingnames:
                                        outputvalue(k)
                                        del remainingnames[k]
                                    break
                        for k in remainingnames.keys():
                            if re.match('^%s[ ?:=]' % k, line):
                                currname = k
                                if runthrough == 1:
                                    existingnames.append(k)
                                else:
                                    del remainingnames[k]
                                break
                        if currname and runthrough > 1:
                            outputvalue(currname)

                    if currname:
                        sline = line.rstrip()
                        if not sline.endswith('\\'):
                            currname = None
                        continue
                    if runthrough > 1:
                        tf.write(line)
                f.seek(0)
        if remainingnames:
            tf.write('\n')
            for k in remainingnames.keys():
                outputvalue(k)

    with open(tfn, 'U') as f:
        tolines = f.readlines()
    if patch:
        with open(fn, 'U') as f:
            fromlines = f.readlines()
        relfn = os.path.relpath(fn, relpath)
        diff = difflib.unified_diff(fromlines, tolines, 'a/%s' % relfn, 'b/%s' % relfn)
        os.remove(tfn)
        return diff
    else:
        with open(fn, 'w') as f:
            f.writelines(tolines)
        os.remove(tfn)
        return None

def localise_file_vars(fn, varfiles, varlist):
    """Given a list of variables and variable history (fetched with get_var_files())
    find where each variable should be set/changed. This handles for example where a
    recipe includes an inc file where variables might be changed - in most cases
    we want to update the inc file when changing the variable value rather than adding
    it to the recipe itself.
    """
    fndir = os.path.dirname(fn) + os.sep

    first_meta_file = None
    for v in meta_vars:
        f = varfiles.get(v, None)
        if f:
            actualdir = os.path.dirname(f) + os.sep
            if actualdir.startswith(fndir):
                first_meta_file = f
                break

    filevars = defaultdict(list)
    for v in varlist:
        f = varfiles[v]
        # Only return files that are in the same directory as the recipe or in some directory below there
        # (this excludes bbclass files and common inc files that wouldn't be appropriate to set the variable
        # in if we were going to set a value specific to this recipe)
        if f:
            actualfile = f
        else:
            # Variable isn't in a file, if it's one of the "meta" vars, use the first file with a meta var in it
            if first_meta_file:
                actualfile = first_meta_file
            else:
                actualfile = fn

        actualdir = os.path.dirname(actualfile) + os.sep
        if not actualdir.startswith(fndir):
            actualfile = fn
        filevars[actualfile].append(v)

    return filevars

def patch_recipe(d, fn, varvalues, patch=False, relpath=''):
    """Modify a list of variable values in the specified recipe. Handles inc files if
    used by the recipe.
    """
    varlist = varvalues.keys()
    varfiles = get_var_files(fn, varlist, d)
    locs = localise_file_vars(fn, varfiles, varlist)
    patches = []
    for f,v in locs.iteritems():
        vals = {k: varvalues[k] for k in v}
        patchdata = patch_recipe_file(f, vals, patch, relpath)
        if patch:
            patches.append(patchdata)

    if patch:
        return patches
    else:
        return None



def copy_recipe_files(d, tgt_dir, whole_dir=False, download=True):
    """Copy (local) recipe files, including both files included via include/require,
    and files referred to in the SRC_URI variable."""
    import bb.fetch2
    import oe.path

    # FIXME need a warning if the unexpanded SRC_URI value contains variable references

    uris = (d.getVar('SRC_URI', True) or "").split()
    fetch = bb.fetch2.Fetch(uris, d)
    if download:
        fetch.download()

    # Copy local files to target directory and gather any remote files
    bb_dir = os.path.dirname(d.getVar('FILE', True)) + os.sep
    remotes = []
    includes = [path for path in d.getVar('BBINCLUDED', True).split() if
                path.startswith(bb_dir) and os.path.exists(path)]
    for path in fetch.localpaths() + includes:
        # Only import files that are under the meta directory
        if path.startswith(bb_dir):
            if not whole_dir:
                relpath = os.path.relpath(path, bb_dir)
                subdir = os.path.join(tgt_dir, os.path.dirname(relpath))
                if not os.path.exists(subdir):
                    os.makedirs(subdir)
                shutil.copy2(path, os.path.join(tgt_dir, relpath))
        else:
            remotes.append(path)
    # Simply copy whole meta dir, if requested
    if whole_dir:
        shutil.copytree(bb_dir, tgt_dir)

    return remotes


def get_recipe_patches(d):
    """Get a list of the patches included in SRC_URI within a recipe."""
    patchfiles = []
    # Execute src_patches() defined in patch.bbclass - this works since that class
    # is inherited globally
    patches = bb.utils.exec_flat_python_func('src_patches', d)
    for patch in patches:
        _, _, local, _, _, parm = bb.fetch.decodeurl(patch)
        patchfiles.append(local)
    return patchfiles


def validate_pn(pn):
    """Perform validation on a recipe name (PN) for a new recipe."""
    reserved_names = ['forcevariable', 'append', 'prepend', 'remove']
    if not re.match('[0-9a-z-.]+', pn):
        return 'Recipe name "%s" is invalid: only characters 0-9, a-z, - and . are allowed' % pn
    elif pn in reserved_names:
        return 'Recipe name "%s" is invalid: is a reserved keyword' % pn
    elif pn.startswith('pn-'):
        return 'Recipe name "%s" is invalid: names starting with "pn-" are reserved' % pn
    return ''