diff options
Diffstat (limited to 'meta/classes/sstate.bbclass')
-rw-r--r-- | meta/classes/sstate.bbclass | 737 |
1 files changed, 737 insertions, 0 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass new file mode 100644 index 0000000000..517c1001d2 --- /dev/null +++ b/meta/classes/sstate.bbclass | |||
@@ -0,0 +1,737 @@ | |||
1 | SSTATE_VERSION = "3" | ||
2 | |||
3 | SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control" | ||
4 | SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}" | ||
5 | |||
6 | def generate_sstatefn(spec, hash, d): | ||
7 | if not hash: | ||
8 | hash = "INVALID" | ||
9 | return hash[:2] + "/" + spec + hash | ||
10 | |||
11 | SSTATE_PKGARCH = "${PACKAGE_ARCH}" | ||
12 | SSTATE_PKGSPEC = "sstate-${PN}-${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}-${PV}-${PR}-${SSTATE_PKGARCH}-${SSTATE_VERSION}-" | ||
13 | SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}" | ||
14 | SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" | ||
15 | SSTATE_EXTRAPATH = "" | ||
16 | SSTATE_EXTRAPATHWILDCARD = "" | ||
17 | SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}" | ||
18 | |||
19 | SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/" | ||
20 | # Also need to make cross recipes append to ${PN} and install once for any given PACAGE_ARCH so | ||
21 | # can avoid multiple installs (e.g. routerstationpro+qemumips both using mips32) | ||
22 | SSTATE_DUPWHITELIST += "${STAGING_LIBDIR_NATIVE}/${MULTIMACH_TARGET_SYS} ${STAGING_DIR_NATIVE}/usr/libexec/${MULTIMACH_TARGET_SYS} ${STAGING_BINDIR_NATIVE}/${MULTIMACH_TARGET_SYS} ${STAGING_DIR_NATIVE}${includedir_native}/gcc-build-internal-${MULTIMACH_TARGET_SYS}" | ||
23 | # Avoid docbook/sgml catalog warnings for now | ||
24 | SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml" | ||
25 | |||
26 | SSTATE_SCAN_FILES ?= "*.la *-config *_config" | ||
27 | SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f' | ||
28 | |||
29 | BB_HASHFILENAME = "${SSTATE_EXTRAPATH} ${SSTATE_PKGSPEC}" | ||
30 | |||
31 | SSTATE_MANMACH ?= "${SSTATE_PKGARCH}" | ||
32 | |||
33 | SSTATEPREINSTFUNCS ?= "" | ||
34 | SSTATEPOSTINSTFUNCS ?= "" | ||
35 | EXTRA_STAGING_FIXMES ?= "" | ||
36 | |||
37 | # Specify dirs in which the shell function is executed and don't use ${B} | ||
38 | # as default dirs to avoid possible race about ${B} with other task. | ||
39 | sstate_create_package[dirs] = "${SSTATE_BUILDDIR}" | ||
40 | sstate_unpack_package[dirs] = "${SSTATE_INSTDIR}" | ||
41 | |||
42 | python () { | ||
43 | if bb.data.inherits_class('native', d): | ||
44 | d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH')) | ||
45 | elif bb.data.inherits_class('crosssdk', d): | ||
46 | d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}")) | ||
47 | elif bb.data.inherits_class('cross', d): | ||
48 | d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TUNE_PKGARCH}")) | ||
49 | d.setVar('SSTATE_MANMACH', d.expand("${BUILD_ARCH}_${MACHINE}")) | ||
50 | elif bb.data.inherits_class('nativesdk', d): | ||
51 | d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}")) | ||
52 | elif bb.data.inherits_class('cross-canadian', d): | ||
53 | d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}")) | ||
54 | elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH", True) == "all": | ||
55 | d.setVar('SSTATE_PKGARCH', "allarch") | ||
56 | else: | ||
57 | d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}")) | ||
58 | |||
59 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d): | ||
60 | d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") | ||
61 | d.setVar('SSTATE_EXTRAPATHWILDCARD', "*/") | ||
62 | |||
63 | # These classes encode staging paths into their scripts data so can only be | ||
64 | # reused if we manipulate the paths | ||
65 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d): | ||
66 | scan_cmd = "grep -Irl ${STAGING_DIR} ${SSTATE_BUILDDIR}" | ||
67 | d.setVar('SSTATE_SCAN_CMD', scan_cmd) | ||
68 | |||
69 | unique_tasks = set((d.getVar('SSTATETASKS', True) or "").split()) | ||
70 | d.setVar('SSTATETASKS', " ".join(unique_tasks)) | ||
71 | namemap = [] | ||
72 | for task in unique_tasks: | ||
73 | namemap.append(d.getVarFlag(task, 'sstate-name')) | ||
74 | d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") | ||
75 | d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc") | ||
76 | d.setVar('SSTATETASKNAMES', " ".join(namemap)) | ||
77 | } | ||
78 | |||
79 | def sstate_init(name, task, d): | ||
80 | ss = {} | ||
81 | ss['task'] = task | ||
82 | ss['name'] = name | ||
83 | ss['dirs'] = [] | ||
84 | ss['plaindirs'] = [] | ||
85 | ss['lockfiles'] = [] | ||
86 | ss['lockfiles-shared'] = [] | ||
87 | return ss | ||
88 | |||
89 | def sstate_state_fromvars(d, task = None): | ||
90 | if task is None: | ||
91 | task = d.getVar('BB_CURRENTTASK', True) | ||
92 | if not task: | ||
93 | bb.fatal("sstate code running without task context?!") | ||
94 | task = task.replace("_setscene", "") | ||
95 | |||
96 | name = d.getVarFlag("do_" + task, 'sstate-name', True) | ||
97 | inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs', True) or "").split() | ||
98 | outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs', True) or "").split() | ||
99 | plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs', True) or "").split() | ||
100 | lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile', True) or "").split() | ||
101 | lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared', True) or "").split() | ||
102 | interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs', True) or "").split() | ||
103 | if not name or len(inputs) != len(outputs): | ||
104 | bb.fatal("sstate variables not setup correctly?!") | ||
105 | |||
106 | ss = sstate_init(name, task, d) | ||
107 | for i in range(len(inputs)): | ||
108 | sstate_add(ss, inputs[i], outputs[i], d) | ||
109 | ss['lockfiles'] = lockfiles | ||
110 | ss['lockfiles-shared'] = lockfilesshared | ||
111 | ss['plaindirs'] = plaindirs | ||
112 | ss['interceptfuncs'] = interceptfuncs | ||
113 | return ss | ||
114 | |||
115 | def sstate_add(ss, source, dest, d): | ||
116 | if not source.endswith("/"): | ||
117 | source = source + "/" | ||
118 | if not dest.endswith("/"): | ||
119 | dest = dest + "/" | ||
120 | source = os.path.normpath(source) | ||
121 | dest = os.path.normpath(dest) | ||
122 | srcbase = os.path.basename(source) | ||
123 | ss['dirs'].append([srcbase, source, dest]) | ||
124 | return ss | ||
125 | |||
126 | def sstate_install(ss, d): | ||
127 | import oe.path | ||
128 | import subprocess | ||
129 | |||
130 | sharedfiles = [] | ||
131 | shareddirs = [] | ||
132 | bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) | ||
133 | |||
134 | d2 = d.createCopy() | ||
135 | extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) | ||
136 | if extrainf: | ||
137 | d2.setVar("SSTATE_MANMACH", extrainf) | ||
138 | manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['name']) | ||
139 | |||
140 | if os.access(manifest, os.R_OK): | ||
141 | bb.fatal("Package already staged (%s)?!" % manifest) | ||
142 | |||
143 | locks = [] | ||
144 | for lock in ss['lockfiles-shared']: | ||
145 | locks.append(bb.utils.lockfile(lock, True)) | ||
146 | for lock in ss['lockfiles']: | ||
147 | locks.append(bb.utils.lockfile(lock)) | ||
148 | |||
149 | for state in ss['dirs']: | ||
150 | bb.debug(2, "Staging files from %s to %s" % (state[1], state[2])) | ||
151 | for walkroot, dirs, files in os.walk(state[1]): | ||
152 | for file in files: | ||
153 | srcpath = os.path.join(walkroot, file) | ||
154 | dstpath = srcpath.replace(state[1], state[2]) | ||
155 | #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath)) | ||
156 | sharedfiles.append(dstpath) | ||
157 | for dir in dirs: | ||
158 | srcdir = os.path.join(walkroot, dir) | ||
159 | dstdir = srcdir.replace(state[1], state[2]) | ||
160 | #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir)) | ||
161 | if not dstdir.endswith("/"): | ||
162 | dstdir = dstdir + "/" | ||
163 | shareddirs.append(dstdir) | ||
164 | |||
165 | # Check the file list for conflicts against files which already exist | ||
166 | whitelist = (d.getVar("SSTATE_DUPWHITELIST", True) or "").split() | ||
167 | match = [] | ||
168 | for f in sharedfiles: | ||
169 | if os.path.exists(f): | ||
170 | f = os.path.normpath(f) | ||
171 | realmatch = True | ||
172 | for w in whitelist: | ||
173 | if f.startswith(w): | ||
174 | realmatch = False | ||
175 | break | ||
176 | if realmatch: | ||
177 | match.append(f) | ||
178 | sstate_search_cmd = "grep -rl '%s' %s --exclude=master.list | sed -e 's:^.*/::' -e 's:\.populate-sysroot::'" % (f, d.expand("${SSTATE_MANIFESTS}")) | ||
179 | search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] | ||
180 | if search_output != "": | ||
181 | match.append("Matched in %s" % search_output.rstrip()) | ||
182 | if match: | ||
183 | bb.warn("The recipe %s is trying to install files into a shared area when those files already exist. Those files and their manifest location are:\n %s\nPlease verify which package should provide the above files." % (d.getVar('PN', True), "\n ".join(match))) | ||
184 | |||
185 | # Write out the manifest | ||
186 | f = open(manifest, "w") | ||
187 | for file in sharedfiles: | ||
188 | f.write(file + "\n") | ||
189 | |||
190 | # We want to ensure that directories appear at the end of the manifest | ||
191 | # so that when we test to see if they should be deleted any contents | ||
192 | # added by the task will have been removed first. | ||
193 | dirs = sorted(shareddirs, key=len) | ||
194 | # Must remove children first, which will have a longer path than the parent | ||
195 | for di in reversed(dirs): | ||
196 | f.write(di + "\n") | ||
197 | f.close() | ||
198 | |||
199 | # Run the actual file install | ||
200 | for state in ss['dirs']: | ||
201 | if os.path.exists(state[1]): | ||
202 | oe.path.copyhardlinktree(state[1], state[2]) | ||
203 | |||
204 | for postinst in (d.getVar('SSTATEPOSTINSTFUNCS', True) or '').split(): | ||
205 | bb.build.exec_func(postinst, d) | ||
206 | |||
207 | for lock in locks: | ||
208 | bb.utils.unlockfile(lock) | ||
209 | |||
210 | def sstate_installpkg(ss, d): | ||
211 | import oe.path | ||
212 | import subprocess | ||
213 | |||
214 | def prepdir(dir): | ||
215 | # remove dir if it exists, ensure any parent directories do exist | ||
216 | if os.path.exists(dir): | ||
217 | oe.path.remove(dir) | ||
218 | bb.utils.mkdirhier(dir) | ||
219 | oe.path.remove(dir) | ||
220 | |||
221 | sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['name']) | ||
222 | sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['name'] + ".tgz" | ||
223 | sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['name'] + ".tgz" | ||
224 | |||
225 | if not os.path.exists(sstatepkg): | ||
226 | pstaging_fetch(sstatefetch, sstatepkg, d) | ||
227 | |||
228 | if not os.path.isfile(sstatepkg): | ||
229 | bb.note("Staging package %s does not exist" % sstatepkg) | ||
230 | return False | ||
231 | |||
232 | sstate_clean(ss, d) | ||
233 | |||
234 | d.setVar('SSTATE_INSTDIR', sstateinst) | ||
235 | d.setVar('SSTATE_PKG', sstatepkg) | ||
236 | |||
237 | for preinst in (d.getVar('SSTATEPREINSTFUNCS', True) or '').split(): | ||
238 | bb.build.exec_func(preinst, d) | ||
239 | |||
240 | bb.build.exec_func('sstate_unpack_package', d) | ||
241 | |||
242 | # Fixup hardcoded paths | ||
243 | # | ||
244 | # Note: The logic below must match the reverse logic in | ||
245 | # sstate_hardcode_path(d) | ||
246 | |||
247 | fixmefn = sstateinst + "fixmepath" | ||
248 | if os.path.isfile(fixmefn): | ||
249 | staging = d.getVar('STAGING_DIR', True) | ||
250 | staging_target = d.getVar('STAGING_DIR_TARGET', True) | ||
251 | staging_host = d.getVar('STAGING_DIR_HOST', True) | ||
252 | |||
253 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): | ||
254 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIR:%s:g'" % (staging) | ||
255 | elif bb.data.inherits_class('cross', d): | ||
256 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIR:%s:g'" % (staging_target, staging) | ||
257 | else: | ||
258 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host) | ||
259 | |||
260 | extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' | ||
261 | for fixmevar in extra_staging_fixmes.split(): | ||
262 | fixme_path = d.getVar(fixmevar, True) | ||
263 | sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) | ||
264 | |||
265 | # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed | ||
266 | sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd) | ||
267 | |||
268 | bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd)) | ||
269 | subprocess.call(sstate_hardcode_cmd, shell=True) | ||
270 | |||
271 | # Need to remove this or we'd copy it into the target directory and may | ||
272 | # conflict with another writer | ||
273 | os.remove(fixmefn) | ||
274 | |||
275 | for state in ss['dirs']: | ||
276 | prepdir(state[1]) | ||
277 | os.rename(sstateinst + state[0], state[1]) | ||
278 | sstate_install(ss, d) | ||
279 | |||
280 | for plain in ss['plaindirs']: | ||
281 | workdir = d.getVar('WORKDIR', True) | ||
282 | src = sstateinst + "/" + plain.replace(workdir, '') | ||
283 | dest = plain | ||
284 | bb.utils.mkdirhier(src) | ||
285 | prepdir(dest) | ||
286 | os.rename(src, dest) | ||
287 | |||
288 | return True | ||
289 | |||
290 | def sstate_clean_cachefile(ss, d): | ||
291 | import oe.path | ||
292 | |||
293 | sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['name'] + ".tgz*" | ||
294 | bb.note("Removing %s" % sstatepkgfile) | ||
295 | oe.path.remove(sstatepkgfile) | ||
296 | |||
297 | def sstate_clean_cachefiles(d): | ||
298 | for task in (d.getVar('SSTATETASKS', True) or "").split(): | ||
299 | ss = sstate_state_fromvars(d, task[3:]) | ||
300 | sstate_clean_cachefile(ss, d) | ||
301 | |||
302 | def sstate_clean_manifest(manifest, d): | ||
303 | import oe.path | ||
304 | |||
305 | mfile = open(manifest) | ||
306 | entries = mfile.readlines() | ||
307 | mfile.close() | ||
308 | |||
309 | for entry in entries: | ||
310 | entry = entry.strip() | ||
311 | bb.debug(2, "Removing manifest: %s" % entry) | ||
312 | # We can race against another package populating directories as we're removing them | ||
313 | # so we ignore errors here. | ||
314 | try: | ||
315 | if entry.endswith("/"): | ||
316 | if os.path.islink(entry[:-1]): | ||
317 | os.remove(entry[:-1]) | ||
318 | elif os.path.exists(entry) and len(os.listdir(entry)) == 0: | ||
319 | os.rmdir(entry[:-1]) | ||
320 | else: | ||
321 | oe.path.remove(entry) | ||
322 | except OSError: | ||
323 | pass | ||
324 | |||
325 | oe.path.remove(manifest) | ||
326 | |||
327 | def sstate_clean(ss, d): | ||
328 | import oe.path | ||
329 | |||
330 | d2 = d.createCopy() | ||
331 | extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) | ||
332 | if extrainf: | ||
333 | d2.setVar("SSTATE_MANMACH", extrainf) | ||
334 | manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['name']) | ||
335 | |||
336 | if os.path.exists(manifest): | ||
337 | locks = [] | ||
338 | for lock in ss['lockfiles-shared']: | ||
339 | locks.append(bb.utils.lockfile(lock)) | ||
340 | for lock in ss['lockfiles']: | ||
341 | locks.append(bb.utils.lockfile(lock)) | ||
342 | |||
343 | sstate_clean_manifest(manifest, d) | ||
344 | |||
345 | for lock in locks: | ||
346 | bb.utils.unlockfile(lock) | ||
347 | |||
348 | stfile = d.getVar("STAMP", True) + ".do_" + ss['task'] | ||
349 | oe.path.remove(stfile) | ||
350 | oe.path.remove(stfile + "_setscene") | ||
351 | if extrainf: | ||
352 | oe.path.remove(stfile + ".*" + extrainf) | ||
353 | oe.path.remove(stfile + "_setscene" + ".*" + extrainf) | ||
354 | else: | ||
355 | oe.path.remove(stfile + ".*") | ||
356 | oe.path.remove(stfile + "_setscene" + ".*") | ||
357 | |||
358 | CLEANFUNCS += "sstate_cleanall" | ||
359 | |||
360 | python sstate_cleanall() { | ||
361 | bb.note("Removing shared state for package %s" % d.getVar('PN', True)) | ||
362 | |||
363 | manifest_dir = d.getVar('SSTATE_MANIFESTS', True) | ||
364 | if not os.path.exists(manifest_dir): | ||
365 | return | ||
366 | |||
367 | namemap = d.getVar('SSTATETASKNAMES', True).split() | ||
368 | tasks = d.getVar('SSTATETASKS', True).split() | ||
369 | for name in namemap: | ||
370 | taskname = tasks[namemap.index(name)] | ||
371 | shared_state = sstate_state_fromvars(d, taskname[3:]) | ||
372 | sstate_clean(shared_state, d) | ||
373 | } | ||
374 | |||
375 | def sstate_hardcode_path(d): | ||
376 | import subprocess, platform | ||
377 | |||
378 | # Need to remove hardcoded paths and fix these when we install the | ||
379 | # staging packages. | ||
380 | # | ||
381 | # Note: the logic in this function needs to match the reverse logic | ||
382 | # in sstate_installpkg(ss, d) | ||
383 | |||
384 | staging = d.getVar('STAGING_DIR', True) | ||
385 | staging_target = d.getVar('STAGING_DIR_TARGET', True) | ||
386 | staging_host = d.getVar('STAGING_DIR_HOST', True) | ||
387 | sstate_builddir = d.getVar('SSTATE_BUILDDIR', True) | ||
388 | |||
389 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): | ||
390 | sstate_grep_cmd = "grep -l -e '%s'" % (staging) | ||
391 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIR:g'" % (staging) | ||
392 | elif bb.data.inherits_class('cross', d): | ||
393 | sstate_grep_cmd = "grep -l -e '(%s|%s)'" % (staging_target, staging) | ||
394 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g; s:%s:FIXMESTAGINGDIR:g'" % (staging_target, staging) | ||
395 | else: | ||
396 | sstate_grep_cmd = "grep -l -e '%s'" % (staging_host) | ||
397 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % (staging_host) | ||
398 | |||
399 | extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' | ||
400 | for fixmevar in extra_staging_fixmes.split(): | ||
401 | fixme_path = d.getVar(fixmevar, True) | ||
402 | sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar) | ||
403 | |||
404 | fixmefn = sstate_builddir + "fixmepath" | ||
405 | |||
406 | sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD', True) | ||
407 | sstate_filelist_cmd = "tee %s" % (fixmefn) | ||
408 | |||
409 | # fixmepath file needs relative paths, drop sstate_builddir prefix | ||
410 | sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn) | ||
411 | |||
412 | xargs_no_empty_run_cmd = '--no-run-if-empty' | ||
413 | if platform.system() == 'Darwin': | ||
414 | xargs_no_empty_run_cmd = '' | ||
415 | |||
416 | # Limit the fixpaths and sed operations based on the initial grep search | ||
417 | # This has the side effect of making sure the vfs cache is hot | ||
418 | sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd) | ||
419 | |||
420 | bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd)) | ||
421 | subprocess.call(sstate_hardcode_cmd, shell=True) | ||
422 | |||
423 | # If the fixmefn is empty, remove it.. | ||
424 | if os.stat(fixmefn).st_size == 0: | ||
425 | os.remove(fixmefn) | ||
426 | else: | ||
427 | bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd)) | ||
428 | subprocess.call(sstate_filelist_relative_cmd, shell=True) | ||
429 | |||
430 | def sstate_package(ss, d): | ||
431 | import oe.path | ||
432 | |||
433 | def make_relative_symlink(path, outputpath, d): | ||
434 | # Replace out absolute TMPDIR paths in symlinks with relative ones | ||
435 | if not os.path.islink(path): | ||
436 | return | ||
437 | link = os.readlink(path) | ||
438 | if not os.path.isabs(link): | ||
439 | return | ||
440 | if not link.startswith(tmpdir): | ||
441 | return | ||
442 | |||
443 | depth = outputpath.rpartition(tmpdir)[2].count('/') | ||
444 | base = link.partition(tmpdir)[2].strip() | ||
445 | while depth > 1: | ||
446 | base = "/.." + base | ||
447 | depth -= 1 | ||
448 | base = "." + base | ||
449 | |||
450 | bb.debug(2, "Replacing absolute path %s with relative path %s for %s" % (link, base, outputpath)) | ||
451 | os.remove(path) | ||
452 | os.symlink(base, path) | ||
453 | |||
454 | tmpdir = d.getVar('TMPDIR', True) | ||
455 | |||
456 | sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['name']) | ||
457 | sstatepkg = d.getVar('SSTATE_PKG', True) + '_'+ ss['name'] + ".tgz" | ||
458 | bb.utils.remove(sstatebuild, recurse=True) | ||
459 | bb.utils.mkdirhier(sstatebuild) | ||
460 | bb.utils.mkdirhier(os.path.dirname(sstatepkg)) | ||
461 | for state in ss['dirs']: | ||
462 | if not os.path.exists(state[1]): | ||
463 | continue | ||
464 | srcbase = state[0].rstrip("/").rsplit('/', 1)[0] | ||
465 | for walkroot, dirs, files in os.walk(state[1]): | ||
466 | for file in files: | ||
467 | srcpath = os.path.join(walkroot, file) | ||
468 | dstpath = srcpath.replace(state[1], state[2]) | ||
469 | make_relative_symlink(srcpath, dstpath, d) | ||
470 | for dir in dirs: | ||
471 | srcpath = os.path.join(walkroot, dir) | ||
472 | dstpath = srcpath.replace(state[1], state[2]) | ||
473 | make_relative_symlink(srcpath, dstpath, d) | ||
474 | bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0])) | ||
475 | oe.path.copyhardlinktree(state[1], sstatebuild + state[0]) | ||
476 | |||
477 | workdir = d.getVar('WORKDIR', True) | ||
478 | for plain in ss['plaindirs']: | ||
479 | pdir = plain.replace(workdir, sstatebuild) | ||
480 | bb.utils.mkdirhier(plain) | ||
481 | bb.utils.mkdirhier(pdir) | ||
482 | oe.path.copyhardlinktree(plain, pdir) | ||
483 | |||
484 | d.setVar('SSTATE_BUILDDIR', sstatebuild) | ||
485 | d.setVar('SSTATE_PKG', sstatepkg) | ||
486 | sstate_hardcode_path(d) | ||
487 | bb.build.exec_func('sstate_create_package', d) | ||
488 | |||
489 | bb.siggen.dump_this_task(sstatepkg + ".siginfo", d) | ||
490 | |||
491 | return | ||
492 | |||
493 | def pstaging_fetch(sstatefetch, sstatepkg, d): | ||
494 | import bb.fetch2 | ||
495 | |||
496 | # Only try and fetch if the user has configured a mirror | ||
497 | mirrors = d.getVar('SSTATE_MIRRORS', True) | ||
498 | if not mirrors: | ||
499 | return | ||
500 | |||
501 | # Copy the data object and override DL_DIR and SRC_URI | ||
502 | localdata = bb.data.createCopy(d) | ||
503 | bb.data.update_data(localdata) | ||
504 | |||
505 | dldir = localdata.expand("${SSTATE_DIR}") | ||
506 | bb.utils.mkdirhier(dldir) | ||
507 | |||
508 | localdata.delVar('MIRRORS') | ||
509 | localdata.delVar('FILESPATH') | ||
510 | localdata.setVar('DL_DIR', dldir) | ||
511 | localdata.setVar('PREMIRRORS', mirrors) | ||
512 | |||
513 | # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, | ||
514 | # we'll want to allow network access for the current set of fetches. | ||
515 | if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": | ||
516 | localdata.delVar('BB_NO_NETWORK') | ||
517 | |||
518 | # Try a fetch from the sstate mirror, if it fails just return and | ||
519 | # we will build the package | ||
520 | for srcuri in ['file://{0}'.format(sstatefetch), | ||
521 | 'file://{0}.siginfo'.format(sstatefetch)]: | ||
522 | localdata.setVar('SRC_URI', srcuri) | ||
523 | try: | ||
524 | fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) | ||
525 | fetcher.download() | ||
526 | |||
527 | # Need to optimise this, if using file:// urls, the fetcher just changes the local path | ||
528 | # For now work around by symlinking | ||
529 | localpath = bb.data.expand(fetcher.localpath(srcuri), localdata) | ||
530 | if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg): | ||
531 | os.symlink(localpath, sstatepkg) | ||
532 | |||
533 | except bb.fetch2.BBFetchException: | ||
534 | break | ||
535 | |||
536 | def sstate_setscene(d): | ||
537 | shared_state = sstate_state_fromvars(d) | ||
538 | accelerate = sstate_installpkg(shared_state, d) | ||
539 | if not accelerate: | ||
540 | raise bb.build.FuncFailed("No suitable staging package found") | ||
541 | |||
542 | python sstate_task_prefunc () { | ||
543 | shared_state = sstate_state_fromvars(d) | ||
544 | sstate_clean(shared_state, d) | ||
545 | } | ||
546 | |||
547 | python sstate_task_postfunc () { | ||
548 | shared_state = sstate_state_fromvars(d) | ||
549 | sstate_install(shared_state, d) | ||
550 | for intercept in shared_state['interceptfuncs']: | ||
551 | bb.build.exec_func(intercept, d) | ||
552 | omask = os.umask(002) | ||
553 | if omask != 002: | ||
554 | bb.note("Using umask 002 (not %0o) for sstate packaging" % omask) | ||
555 | sstate_package(shared_state, d) | ||
556 | os.umask(omask) | ||
557 | } | ||
558 | |||
559 | |||
560 | # | ||
561 | # Shell function to generate a sstate package from a directory | ||
562 | # set as SSTATE_BUILDDIR | ||
563 | # | ||
564 | sstate_create_package () { | ||
565 | cd ${SSTATE_BUILDDIR} | ||
566 | TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX` | ||
567 | # Need to handle empty directories | ||
568 | if [ "$(ls -A)" ]; then | ||
569 | set +e | ||
570 | tar -czf $TFILE * | ||
571 | if [ $? -ne 0 ] && [ $? -ne 1 ]; then | ||
572 | exit 1 | ||
573 | fi | ||
574 | set -e | ||
575 | else | ||
576 | tar -cz --file=$TFILE --files-from=/dev/null | ||
577 | fi | ||
578 | chmod 0664 $TFILE | ||
579 | mv -f $TFILE ${SSTATE_PKG} | ||
580 | |||
581 | cd ${WORKDIR} | ||
582 | rm -rf ${SSTATE_BUILDDIR} | ||
583 | } | ||
584 | |||
585 | # | ||
586 | # Shell function to decompress and prepare a package for installation | ||
587 | # | ||
588 | sstate_unpack_package () { | ||
589 | mkdir -p ${SSTATE_INSTDIR} | ||
590 | cd ${SSTATE_INSTDIR} | ||
591 | tar -xmvzf ${SSTATE_PKG} | ||
592 | } | ||
593 | |||
594 | # Need to inject information about classes not in the global configuration scope | ||
595 | EXTRASSTATEMAPS += "do_deploy:deploy" | ||
596 | |||
597 | BB_HASHCHECK_FUNCTION = "sstate_checkhashes" | ||
598 | |||
599 | def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d): | ||
600 | |||
601 | ret = [] | ||
602 | mapping = {} | ||
603 | for t in d.getVar("SSTATETASKS", True).split(): | ||
604 | mapping[t] = d.getVarFlag(t, "sstate-name", True) | ||
605 | for extra in d.getVar("EXTRASSTATEMAPS", True).split(): | ||
606 | e = extra.split(":") | ||
607 | mapping[e[0]] = e[1] | ||
608 | |||
609 | for task in range(len(sq_fn)): | ||
610 | spec = sq_hashfn[task].split(" ")[1] | ||
611 | extrapath = sq_hashfn[task].split(" ")[0] | ||
612 | |||
613 | sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz") | ||
614 | if os.path.exists(sstatefile): | ||
615 | bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) | ||
616 | ret.append(task) | ||
617 | continue | ||
618 | else: | ||
619 | bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) | ||
620 | |||
621 | mirrors = d.getVar("SSTATE_MIRRORS", True) | ||
622 | if mirrors: | ||
623 | # Copy the data object and override DL_DIR and SRC_URI | ||
624 | localdata = bb.data.createCopy(d) | ||
625 | bb.data.update_data(localdata) | ||
626 | |||
627 | dldir = localdata.expand("${SSTATE_DIR}") | ||
628 | localdata.setVar('DL_DIR', dldir) | ||
629 | localdata.setVar('PREMIRRORS', mirrors) | ||
630 | |||
631 | bb.debug(2, "SState using premirror of: %s" % mirrors) | ||
632 | |||
633 | # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, | ||
634 | # we'll want to allow network access for the current set of fetches. | ||
635 | if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": | ||
636 | localdata.delVar('BB_NO_NETWORK') | ||
637 | |||
638 | for task in range(len(sq_fn)): | ||
639 | if task in ret: | ||
640 | continue | ||
641 | |||
642 | spec = sq_hashfn[task].split(" ")[1] | ||
643 | extrapath = sq_hashfn[task].split(" ")[0] | ||
644 | sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + mapping[sq_task[task]] + ".tgz") | ||
645 | |||
646 | srcuri = "file://" + sstatefile | ||
647 | localdata.setVar('SRC_URI', srcuri) | ||
648 | bb.debug(2, "SState: Attempting to fetch %s" % srcuri) | ||
649 | |||
650 | try: | ||
651 | fetcher = bb.fetch2.Fetch(srcuri.split(), localdata) | ||
652 | fetcher.checkstatus() | ||
653 | bb.debug(2, "SState: Successful fetch test for %s" % srcuri) | ||
654 | ret.append(task) | ||
655 | except: | ||
656 | bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri) | ||
657 | pass | ||
658 | |||
659 | return ret | ||
660 | |||
661 | BB_SETSCENE_DEPVALID = "setscene_depvalid" | ||
662 | |||
663 | def setscene_depvalid(task, taskdependees, notneeded, d): | ||
664 | # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME] | ||
665 | # task is included in taskdependees too | ||
666 | |||
667 | bb.debug(2, "Considering setscene task: %s" % (str(taskdependees[task]))) | ||
668 | |||
669 | def isNative(x): | ||
670 | return x.endswith("-native") | ||
671 | def isNativeCross(x): | ||
672 | return x.endswith("-native") or x.endswith("-cross") or x.endswith("-cross-initial") | ||
673 | def isSafeDep(x): | ||
674 | if x in ["quilt-native", "autoconf-native", "automake-native", "gnu-config-native", "libtool-native", "pkgconfig-native", "gcc-cross", "binutils-cross", "gcc-cross-initial"]: | ||
675 | return True | ||
676 | return False | ||
677 | def isPostInstDep(x): | ||
678 | if x in ["qemu-native", "gdk-pixbuf-native", "qemuwrapper-cross", "depmodwrapper-cross", "systemd-systemctl-native", "gtk-update-icon-cache-native"]: | ||
679 | return True | ||
680 | return False | ||
681 | |||
682 | # We can skip these "safe" dependencies since the aren't runtime dependencies, just build time | ||
683 | if isSafeDep(taskdependees[task][0]) and taskdependees[task][1] == "do_populate_sysroot": | ||
684 | return True | ||
685 | |||
686 | # We only need to trigger populate_lic through direct dependencies | ||
687 | if taskdependees[task][1] == "do_populate_lic": | ||
688 | return True | ||
689 | |||
690 | for dep in taskdependees: | ||
691 | bb.debug(2, " considering dependency: %s" % (str(taskdependees[dep]))) | ||
692 | if task == dep: | ||
693 | continue | ||
694 | if dep in notneeded: | ||
695 | continue | ||
696 | # do_package_write_* and do_package doesn't need do_package | ||
697 | if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata']: | ||
698 | continue | ||
699 | # do_package_write_* and do_package doesn't need do_populate_sysroot, unless is a postinstall dependency | ||
700 | if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata']: | ||
701 | if isPostInstDep(taskdependees[task][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']: | ||
702 | return False | ||
703 | continue | ||
704 | # Native/Cross packages don't exist and are noexec anyway | ||
705 | if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata']: | ||
706 | continue | ||
707 | |||
708 | # Consider sysroot depending on sysroot tasks | ||
709 | if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': | ||
710 | # base-passwd/shadow-sysroot don't need their dependencies | ||
711 | if taskdependees[dep][0].endswith(("base-passwd", "shadow-sysroot")): | ||
712 | continue | ||
713 | # Nothing need depend on libc-initial/gcc-cross-initial | ||
714 | if taskdependees[task][0].endswith("-initial"): | ||
715 | continue | ||
716 | # Native/Cross populate_sysroot need their dependencies | ||
717 | if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]): | ||
718 | return False | ||
719 | # Target populate_sysroot depended on by cross tools need to be installed | ||
720 | if isNativeCross(taskdependees[dep][0]): | ||
721 | return False | ||
722 | # Native/cross tools depended upon by target sysroot are not needed | ||
723 | if isNativeCross(taskdependees[task][0]): | ||
724 | continue | ||
725 | # Target populate_sysroot need their dependencies | ||
726 | return False | ||
727 | |||
728 | # This is due to the [depends] in useradd.bbclass complicating matters | ||
729 | # The logic *is* reversed here due to the way hard setscene dependencies are injected | ||
730 | if taskdependees[task][1] == 'do_package' and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot': | ||
731 | continue | ||
732 | |||
733 | # Safe fallthrough default | ||
734 | bb.debug(2, " Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep]))) | ||
735 | return False | ||
736 | return True | ||
737 | |||