diff options
Diffstat (limited to 'meta/classes/sstate.bbclass')
-rw-r--r-- | meta/classes/sstate.bbclass | 798 |
1 files changed, 798 insertions, 0 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass new file mode 100644 index 0000000000..7b16bc04d5 --- /dev/null +++ b/meta/classes/sstate.bbclass | |||
@@ -0,0 +1,798 @@ | |||
1 | SSTATE_VERSION = "3" | ||
2 | |||
3 | SSTATE_MANIFESTS ?= "${TMPDIR}/sstate-control" | ||
4 | SSTATE_MANFILEPREFIX = "${SSTATE_MANIFESTS}/manifest-${SSTATE_MANMACH}-${PN}" | ||
5 | |||
6 | def generate_sstatefn(spec, hash, d): | ||
7 | if not hash: | ||
8 | hash = "INVALID" | ||
9 | return hash[:2] + "/" + spec + hash | ||
10 | |||
11 | SSTATE_PKGARCH = "${PACKAGE_ARCH}" | ||
12 | SSTATE_PKGSPEC = "sstate:${PN}:${PACKAGE_ARCH}${TARGET_VENDOR}-${TARGET_OS}:${PV}:${PR}:${SSTATE_PKGARCH}:${SSTATE_VERSION}:" | ||
13 | SSTATE_SWSPEC = "sstate:${BPN}::${PV}:${PR}::${SSTATE_VERSION}:" | ||
14 | SSTATE_PKGNAME = "${SSTATE_EXTRAPATH}${@generate_sstatefn(d.getVar('SSTATE_PKGSPEC', True), d.getVar('BB_TASKHASH', True), d)}" | ||
15 | SSTATE_PKG = "${SSTATE_DIR}/${SSTATE_PKGNAME}" | ||
16 | SSTATE_EXTRAPATH = "" | ||
17 | SSTATE_EXTRAPATHWILDCARD = "" | ||
18 | SSTATE_PATHSPEC = "${SSTATE_DIR}/${SSTATE_EXTRAPATHWILDCARD}*/${SSTATE_PKGSPEC}" | ||
19 | |||
20 | # We don't want the sstate to depend on things like the distro string | ||
21 | # of the system, we let the sstate paths take care of this. | ||
22 | SSTATE_EXTRAPATH[vardepvalue] = "" | ||
23 | |||
24 | SSTATE_DUPWHITELIST = "${DEPLOY_DIR_IMAGE}/ ${DEPLOY_DIR}/licenses/" | ||
25 | # Also need to make cross recipes append to ${PN} and install once for any given PACAGE_ARCH so | ||
26 | # can avoid multiple installs (e.g. routerstationpro+qemumips both using mips32) | ||
27 | SSTATE_DUPWHITELIST += "${STAGING_LIBDIR_NATIVE}/${MULTIMACH_TARGET_SYS} ${STAGING_DIR_NATIVE}/usr/libexec/${MULTIMACH_TARGET_SYS} ${STAGING_BINDIR_NATIVE}/${MULTIMACH_TARGET_SYS} ${STAGING_DIR_NATIVE}${includedir_native}/gcc-build-internal-${MULTIMACH_TARGET_SYS}" | ||
28 | SSTATE_DUPWHITELIST += "${STAGING_DIR_NATIVE}/sysroot-providers/virtual_${TARGET_PREFIX} ${STAGING_DIR_NATIVE}/sysroot-providers/binutils-cross ${STAGING_DIR_NATIVE}/sysroot-providers/gcc-cross" | ||
29 | # Avoid docbook/sgml catalog warnings for now | ||
30 | SSTATE_DUPWHITELIST += "${STAGING_ETCDIR_NATIVE}/sgml ${STAGING_DATADIR_NATIVE}/sgml" | ||
31 | |||
32 | SSTATE_SCAN_FILES ?= "*.la *-config *_config" | ||
33 | SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d.getVar("SSTATE_SCAN_FILES", True).split())}" \) -type f' | ||
34 | |||
35 | BB_HASHFILENAME = "${SSTATE_EXTRAPATH} ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}" | ||
36 | |||
37 | SSTATE_MANMACH ?= "${SSTATE_PKGARCH}" | ||
38 | |||
39 | SSTATEPREINSTFUNCS ?= "" | ||
40 | SSTATEPOSTINSTFUNCS ?= "" | ||
41 | EXTRA_STAGING_FIXMES ?= "" | ||
42 | |||
43 | # Specify dirs in which the shell function is executed and don't use ${B} | ||
44 | # as default dirs to avoid possible race about ${B} with other task. | ||
45 | sstate_create_package[dirs] = "${SSTATE_BUILDDIR}" | ||
46 | sstate_unpack_package[dirs] = "${SSTATE_INSTDIR}" | ||
47 | |||
48 | python () { | ||
49 | if bb.data.inherits_class('native', d): | ||
50 | d.setVar('SSTATE_PKGARCH', d.getVar('BUILD_ARCH')) | ||
51 | elif bb.data.inherits_class('crosssdk', d): | ||
52 | d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${SDK_ARCH}")) | ||
53 | elif bb.data.inherits_class('cross', d): | ||
54 | d.setVar('SSTATE_PKGARCH', d.expand("${BUILD_ARCH}_${TUNE_PKGARCH}")) | ||
55 | d.setVar('SSTATE_MANMACH', d.expand("${BUILD_ARCH}_${MACHINE}")) | ||
56 | elif bb.data.inherits_class('nativesdk', d): | ||
57 | d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}")) | ||
58 | elif bb.data.inherits_class('cross-canadian', d): | ||
59 | d.setVar('SSTATE_PKGARCH', d.expand("${SDK_ARCH}_${PACKAGE_ARCH}")) | ||
60 | elif bb.data.inherits_class('allarch', d) and d.getVar("PACKAGE_ARCH", True) == "all": | ||
61 | d.setVar('SSTATE_PKGARCH', "allarch") | ||
62 | else: | ||
63 | d.setVar('SSTATE_MANMACH', d.expand("${PACKAGE_ARCH}")) | ||
64 | |||
65 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross', d): | ||
66 | d.setVar('SSTATE_EXTRAPATH', "${NATIVELSBSTRING}/") | ||
67 | d.setVar('SSTATE_EXTRAPATHWILDCARD', "*/") | ||
68 | |||
69 | # These classes encode staging paths into their scripts data so can only be | ||
70 | # reused if we manipulate the paths | ||
71 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d): | ||
72 | scan_cmd = "grep -Irl ${STAGING_DIR} ${SSTATE_BUILDDIR}" | ||
73 | d.setVar('SSTATE_SCAN_CMD', scan_cmd) | ||
74 | |||
75 | unique_tasks = set((d.getVar('SSTATETASKS', True) or "").split()) | ||
76 | d.setVar('SSTATETASKS', " ".join(unique_tasks)) | ||
77 | for task in unique_tasks: | ||
78 | d.prependVarFlag(task, 'prefuncs', "sstate_task_prefunc ") | ||
79 | d.appendVarFlag(task, 'postfuncs', " sstate_task_postfunc") | ||
80 | } | ||
81 | |||
82 | def sstate_init(task, d): | ||
83 | ss = {} | ||
84 | ss['task'] = task | ||
85 | ss['dirs'] = [] | ||
86 | ss['plaindirs'] = [] | ||
87 | ss['lockfiles'] = [] | ||
88 | ss['lockfiles-shared'] = [] | ||
89 | return ss | ||
90 | |||
91 | def sstate_state_fromvars(d, task = None): | ||
92 | if task is None: | ||
93 | task = d.getVar('BB_CURRENTTASK', True) | ||
94 | if not task: | ||
95 | bb.fatal("sstate code running without task context?!") | ||
96 | task = task.replace("_setscene", "") | ||
97 | |||
98 | if task.startswith("do_"): | ||
99 | task = task[3:] | ||
100 | inputs = (d.getVarFlag("do_" + task, 'sstate-inputdirs', True) or "").split() | ||
101 | outputs = (d.getVarFlag("do_" + task, 'sstate-outputdirs', True) or "").split() | ||
102 | plaindirs = (d.getVarFlag("do_" + task, 'sstate-plaindirs', True) or "").split() | ||
103 | lockfiles = (d.getVarFlag("do_" + task, 'sstate-lockfile', True) or "").split() | ||
104 | lockfilesshared = (d.getVarFlag("do_" + task, 'sstate-lockfile-shared', True) or "").split() | ||
105 | interceptfuncs = (d.getVarFlag("do_" + task, 'sstate-interceptfuncs', True) or "").split() | ||
106 | if not task or len(inputs) != len(outputs): | ||
107 | bb.fatal("sstate variables not setup correctly?!") | ||
108 | |||
109 | if task == "populate_lic": | ||
110 | d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") | ||
111 | d.setVar("SSTATE_EXTRAPATH", "") | ||
112 | |||
113 | ss = sstate_init(task, d) | ||
114 | for i in range(len(inputs)): | ||
115 | sstate_add(ss, inputs[i], outputs[i], d) | ||
116 | ss['lockfiles'] = lockfiles | ||
117 | ss['lockfiles-shared'] = lockfilesshared | ||
118 | ss['plaindirs'] = plaindirs | ||
119 | ss['interceptfuncs'] = interceptfuncs | ||
120 | return ss | ||
121 | |||
122 | def sstate_add(ss, source, dest, d): | ||
123 | if not source.endswith("/"): | ||
124 | source = source + "/" | ||
125 | if not dest.endswith("/"): | ||
126 | dest = dest + "/" | ||
127 | source = os.path.normpath(source) | ||
128 | dest = os.path.normpath(dest) | ||
129 | srcbase = os.path.basename(source) | ||
130 | ss['dirs'].append([srcbase, source, dest]) | ||
131 | return ss | ||
132 | |||
133 | def sstate_install(ss, d): | ||
134 | import oe.path | ||
135 | import subprocess | ||
136 | |||
137 | sharedfiles = [] | ||
138 | shareddirs = [] | ||
139 | bb.utils.mkdirhier(d.expand("${SSTATE_MANIFESTS}")) | ||
140 | |||
141 | d2 = d.createCopy() | ||
142 | extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) | ||
143 | if extrainf: | ||
144 | d2.setVar("SSTATE_MANMACH", extrainf) | ||
145 | manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task']) | ||
146 | |||
147 | if os.access(manifest, os.R_OK): | ||
148 | bb.fatal("Package already staged (%s)?!" % manifest) | ||
149 | |||
150 | locks = [] | ||
151 | for lock in ss['lockfiles-shared']: | ||
152 | locks.append(bb.utils.lockfile(lock, True)) | ||
153 | for lock in ss['lockfiles']: | ||
154 | locks.append(bb.utils.lockfile(lock)) | ||
155 | |||
156 | for state in ss['dirs']: | ||
157 | bb.debug(2, "Staging files from %s to %s" % (state[1], state[2])) | ||
158 | for walkroot, dirs, files in os.walk(state[1]): | ||
159 | for file in files: | ||
160 | srcpath = os.path.join(walkroot, file) | ||
161 | dstpath = srcpath.replace(state[1], state[2]) | ||
162 | #bb.debug(2, "Staging %s to %s" % (srcpath, dstpath)) | ||
163 | sharedfiles.append(dstpath) | ||
164 | for dir in dirs: | ||
165 | srcdir = os.path.join(walkroot, dir) | ||
166 | dstdir = srcdir.replace(state[1], state[2]) | ||
167 | #bb.debug(2, "Staging %s to %s" % (srcdir, dstdir)) | ||
168 | if not dstdir.endswith("/"): | ||
169 | dstdir = dstdir + "/" | ||
170 | shareddirs.append(dstdir) | ||
171 | |||
172 | # Check the file list for conflicts against files which already exist | ||
173 | whitelist = (d.getVar("SSTATE_DUPWHITELIST", True) or "").split() | ||
174 | match = [] | ||
175 | for f in sharedfiles: | ||
176 | if os.path.exists(f): | ||
177 | f = os.path.normpath(f) | ||
178 | realmatch = True | ||
179 | for w in whitelist: | ||
180 | if f.startswith(w): | ||
181 | realmatch = False | ||
182 | break | ||
183 | if realmatch: | ||
184 | match.append(f) | ||
185 | sstate_search_cmd = "grep -rl '%s' %s --exclude=master.list | sed -e 's:^.*/::' -e 's:\.populate-sysroot::'" % (f, d.expand("${SSTATE_MANIFESTS}")) | ||
186 | search_output = subprocess.Popen(sstate_search_cmd, shell=True, stdout=subprocess.PIPE).communicate()[0] | ||
187 | if search_output != "": | ||
188 | match.append("Matched in %s" % search_output.rstrip()) | ||
189 | if match: | ||
190 | bb.warn("The recipe %s is trying to install files into a shared area when those files already exist. Those files and their manifest location are:\n %s\nPlease verify which package should provide the above files." % (d.getVar('PN', True), "\n ".join(match))) | ||
191 | |||
192 | # Write out the manifest | ||
193 | f = open(manifest, "w") | ||
194 | for file in sharedfiles: | ||
195 | f.write(file + "\n") | ||
196 | |||
197 | # We want to ensure that directories appear at the end of the manifest | ||
198 | # so that when we test to see if they should be deleted any contents | ||
199 | # added by the task will have been removed first. | ||
200 | dirs = sorted(shareddirs, key=len) | ||
201 | # Must remove children first, which will have a longer path than the parent | ||
202 | for di in reversed(dirs): | ||
203 | f.write(di + "\n") | ||
204 | f.close() | ||
205 | |||
206 | # Run the actual file install | ||
207 | for state in ss['dirs']: | ||
208 | if os.path.exists(state[1]): | ||
209 | oe.path.copyhardlinktree(state[1], state[2]) | ||
210 | |||
211 | for postinst in (d.getVar('SSTATEPOSTINSTFUNCS', True) or '').split(): | ||
212 | bb.build.exec_func(postinst, d) | ||
213 | |||
214 | for lock in locks: | ||
215 | bb.utils.unlockfile(lock) | ||
216 | |||
217 | sstate_install[vardepsexclude] += "SSTATE_DUPWHITELIST STATE_MANMACH SSTATE_MANFILEPREFIX" | ||
218 | sstate_install[vardeps] += "${SSTATEPOSTINSTFUNCS}" | ||
219 | |||
220 | def sstate_installpkg(ss, d): | ||
221 | import oe.path | ||
222 | import subprocess | ||
223 | |||
224 | def prepdir(dir): | ||
225 | # remove dir if it exists, ensure any parent directories do exist | ||
226 | if os.path.exists(dir): | ||
227 | oe.path.remove(dir) | ||
228 | bb.utils.mkdirhier(dir) | ||
229 | oe.path.remove(dir) | ||
230 | |||
231 | sstateinst = d.expand("${WORKDIR}/sstate-install-%s/" % ss['task']) | ||
232 | sstatefetch = d.getVar('SSTATE_PKGNAME', True) + '_' + ss['task'] + ".tgz" | ||
233 | sstatepkg = d.getVar('SSTATE_PKG', True) + '_' + ss['task'] + ".tgz" | ||
234 | |||
235 | if not os.path.exists(sstatepkg): | ||
236 | pstaging_fetch(sstatefetch, sstatepkg, d) | ||
237 | |||
238 | if not os.path.isfile(sstatepkg): | ||
239 | bb.note("Staging package %s does not exist" % sstatepkg) | ||
240 | return False | ||
241 | |||
242 | sstate_clean(ss, d) | ||
243 | |||
244 | d.setVar('SSTATE_INSTDIR', sstateinst) | ||
245 | d.setVar('SSTATE_PKG', sstatepkg) | ||
246 | |||
247 | for preinst in (d.getVar('SSTATEPREINSTFUNCS', True) or '').split(): | ||
248 | bb.build.exec_func(preinst, d) | ||
249 | |||
250 | bb.build.exec_func('sstate_unpack_package', d) | ||
251 | |||
252 | # Fixup hardcoded paths | ||
253 | # | ||
254 | # Note: The logic below must match the reverse logic in | ||
255 | # sstate_hardcode_path(d) | ||
256 | |||
257 | fixmefn = sstateinst + "fixmepath" | ||
258 | if os.path.isfile(fixmefn): | ||
259 | staging = d.getVar('STAGING_DIR', True) | ||
260 | staging_target = d.getVar('STAGING_DIR_TARGET', True) | ||
261 | staging_host = d.getVar('STAGING_DIR_HOST', True) | ||
262 | |||
263 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): | ||
264 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIR:%s:g'" % (staging) | ||
265 | elif bb.data.inherits_class('cross', d): | ||
266 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRTARGET:%s:g; s:FIXMESTAGINGDIR:%s:g'" % (staging_target, staging) | ||
267 | else: | ||
268 | sstate_sed_cmd = "sed -i -e 's:FIXMESTAGINGDIRHOST:%s:g'" % (staging_host) | ||
269 | |||
270 | extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' | ||
271 | for fixmevar in extra_staging_fixmes.split(): | ||
272 | fixme_path = d.getVar(fixmevar, True) | ||
273 | sstate_sed_cmd += " -e 's:FIXME_%s:%s:g'" % (fixmevar, fixme_path) | ||
274 | |||
275 | # Add sstateinst to each filename in fixmepath, use xargs to efficiently call sed | ||
276 | sstate_hardcode_cmd = "sed -e 's:^:%s:g' %s | xargs %s" % (sstateinst, fixmefn, sstate_sed_cmd) | ||
277 | |||
278 | bb.note("Replacing fixme paths in sstate package: %s" % (sstate_hardcode_cmd)) | ||
279 | subprocess.call(sstate_hardcode_cmd, shell=True) | ||
280 | |||
281 | # Need to remove this or we'd copy it into the target directory and may | ||
282 | # conflict with another writer | ||
283 | os.remove(fixmefn) | ||
284 | |||
285 | for state in ss['dirs']: | ||
286 | prepdir(state[1]) | ||
287 | os.rename(sstateinst + state[0], state[1]) | ||
288 | sstate_install(ss, d) | ||
289 | |||
290 | for plain in ss['plaindirs']: | ||
291 | workdir = d.getVar('WORKDIR', True) | ||
292 | src = sstateinst + "/" + plain.replace(workdir, '') | ||
293 | dest = plain | ||
294 | bb.utils.mkdirhier(src) | ||
295 | prepdir(dest) | ||
296 | os.rename(src, dest) | ||
297 | |||
298 | return True | ||
299 | |||
300 | def sstate_clean_cachefile(ss, d): | ||
301 | import oe.path | ||
302 | |||
303 | sstatepkgfile = d.getVar('SSTATE_PATHSPEC', True) + "*_" + ss['task'] + ".tgz*" | ||
304 | bb.note("Removing %s" % sstatepkgfile) | ||
305 | oe.path.remove(sstatepkgfile) | ||
306 | |||
307 | def sstate_clean_cachefiles(d): | ||
308 | for task in (d.getVar('SSTATETASKS', True) or "").split(): | ||
309 | ld = d.createCopy() | ||
310 | ss = sstate_state_fromvars(ld, task) | ||
311 | sstate_clean_cachefile(ss, ld) | ||
312 | |||
313 | def sstate_clean_manifest(manifest, d): | ||
314 | import oe.path | ||
315 | |||
316 | mfile = open(manifest) | ||
317 | entries = mfile.readlines() | ||
318 | mfile.close() | ||
319 | |||
320 | for entry in entries: | ||
321 | entry = entry.strip() | ||
322 | bb.debug(2, "Removing manifest: %s" % entry) | ||
323 | # We can race against another package populating directories as we're removing them | ||
324 | # so we ignore errors here. | ||
325 | try: | ||
326 | if entry.endswith("/"): | ||
327 | if os.path.islink(entry[:-1]): | ||
328 | os.remove(entry[:-1]) | ||
329 | elif os.path.exists(entry) and len(os.listdir(entry)) == 0: | ||
330 | os.rmdir(entry[:-1]) | ||
331 | else: | ||
332 | oe.path.remove(entry) | ||
333 | except OSError: | ||
334 | pass | ||
335 | |||
336 | oe.path.remove(manifest) | ||
337 | |||
338 | def sstate_clean(ss, d): | ||
339 | import oe.path | ||
340 | import glob | ||
341 | |||
342 | d2 = d.createCopy() | ||
343 | stamp_clean = d.getVar("STAMPCLEAN", True) | ||
344 | extrainf = d.getVarFlag("do_" + ss['task'], 'stamp-extra-info', True) | ||
345 | if extrainf: | ||
346 | d2.setVar("SSTATE_MANMACH", extrainf) | ||
347 | wildcard_stfile = "%s.do_%s*.%s" % (stamp_clean, ss['task'], extrainf) | ||
348 | else: | ||
349 | wildcard_stfile = "%s.do_%s*" % (stamp_clean, ss['task']) | ||
350 | |||
351 | manifest = d2.expand("${SSTATE_MANFILEPREFIX}.%s" % ss['task']) | ||
352 | |||
353 | if os.path.exists(manifest): | ||
354 | locks = [] | ||
355 | for lock in ss['lockfiles-shared']: | ||
356 | locks.append(bb.utils.lockfile(lock)) | ||
357 | for lock in ss['lockfiles']: | ||
358 | locks.append(bb.utils.lockfile(lock)) | ||
359 | |||
360 | sstate_clean_manifest(manifest, d) | ||
361 | |||
362 | for lock in locks: | ||
363 | bb.utils.unlockfile(lock) | ||
364 | |||
365 | # Remove the current and previous stamps, but keep the sigdata. | ||
366 | # | ||
367 | # The glob() matches do_task* which may match multiple tasks, for | ||
368 | # example: do_package and do_package_write_ipk, so we need to | ||
369 | # exactly match *.do_task.* and *.do_task_setscene.* | ||
370 | rm_stamp = '.do_%s.' % ss['task'] | ||
371 | rm_setscene = '.do_%s_setscene.' % ss['task'] | ||
372 | # For BB_SIGNATURE_HANDLER = "noop" | ||
373 | rm_nohash = ".do_%s" % ss['task'] | ||
374 | for stfile in glob.glob(wildcard_stfile): | ||
375 | # Keep the sigdata | ||
376 | if ".sigdata." in stfile: | ||
377 | continue | ||
378 | # Preserve taint files in the stamps directory | ||
379 | if stfile.endswith('.taint'): | ||
380 | continue | ||
381 | if rm_stamp in stfile or rm_setscene in stfile or \ | ||
382 | stfile.endswith(rm_nohash): | ||
383 | oe.path.remove(stfile) | ||
384 | |||
385 | sstate_clean[vardepsexclude] = "SSTATE_MANFILEPREFIX" | ||
386 | |||
387 | CLEANFUNCS += "sstate_cleanall" | ||
388 | |||
389 | python sstate_cleanall() { | ||
390 | bb.note("Removing shared state for package %s" % d.getVar('PN', True)) | ||
391 | |||
392 | manifest_dir = d.getVar('SSTATE_MANIFESTS', True) | ||
393 | if not os.path.exists(manifest_dir): | ||
394 | return | ||
395 | |||
396 | tasks = d.getVar('SSTATETASKS', True).split() | ||
397 | for name in tasks: | ||
398 | ld = d.createCopy() | ||
399 | shared_state = sstate_state_fromvars(ld, name) | ||
400 | sstate_clean(shared_state, ld) | ||
401 | } | ||
402 | |||
403 | def sstate_hardcode_path(d): | ||
404 | import subprocess, platform | ||
405 | |||
406 | # Need to remove hardcoded paths and fix these when we install the | ||
407 | # staging packages. | ||
408 | # | ||
409 | # Note: the logic in this function needs to match the reverse logic | ||
410 | # in sstate_installpkg(ss, d) | ||
411 | |||
412 | staging = d.getVar('STAGING_DIR', True) | ||
413 | staging_target = d.getVar('STAGING_DIR_TARGET', True) | ||
414 | staging_host = d.getVar('STAGING_DIR_HOST', True) | ||
415 | sstate_builddir = d.getVar('SSTATE_BUILDDIR', True) | ||
416 | |||
417 | if bb.data.inherits_class('native', d) or bb.data.inherits_class('nativesdk', d) or bb.data.inherits_class('crosssdk', d) or bb.data.inherits_class('cross-canadian', d): | ||
418 | sstate_grep_cmd = "grep -l -e '%s'" % (staging) | ||
419 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIR:g'" % (staging) | ||
420 | elif bb.data.inherits_class('cross', d): | ||
421 | sstate_grep_cmd = "grep -l -e '(%s|%s)'" % (staging_target, staging) | ||
422 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRTARGET:g; s:%s:FIXMESTAGINGDIR:g'" % (staging_target, staging) | ||
423 | else: | ||
424 | sstate_grep_cmd = "grep -l -e '%s'" % (staging_host) | ||
425 | sstate_sed_cmd = "sed -i -e 's:%s:FIXMESTAGINGDIRHOST:g'" % (staging_host) | ||
426 | |||
427 | extra_staging_fixmes = d.getVar('EXTRA_STAGING_FIXMES', True) or '' | ||
428 | for fixmevar in extra_staging_fixmes.split(): | ||
429 | fixme_path = d.getVar(fixmevar, True) | ||
430 | sstate_sed_cmd += " -e 's:%s:FIXME_%s:g'" % (fixme_path, fixmevar) | ||
431 | |||
432 | fixmefn = sstate_builddir + "fixmepath" | ||
433 | |||
434 | sstate_scan_cmd = d.getVar('SSTATE_SCAN_CMD', True) | ||
435 | sstate_filelist_cmd = "tee %s" % (fixmefn) | ||
436 | |||
437 | # fixmepath file needs relative paths, drop sstate_builddir prefix | ||
438 | sstate_filelist_relative_cmd = "sed -i -e 's:^%s::g' %s" % (sstate_builddir, fixmefn) | ||
439 | |||
440 | xargs_no_empty_run_cmd = '--no-run-if-empty' | ||
441 | if platform.system() == 'Darwin': | ||
442 | xargs_no_empty_run_cmd = '' | ||
443 | |||
444 | # Limit the fixpaths and sed operations based on the initial grep search | ||
445 | # This has the side effect of making sure the vfs cache is hot | ||
446 | sstate_hardcode_cmd = "%s | xargs %s | %s | xargs %s %s" % (sstate_scan_cmd, sstate_grep_cmd, sstate_filelist_cmd, xargs_no_empty_run_cmd, sstate_sed_cmd) | ||
447 | |||
448 | bb.note("Removing hardcoded paths from sstate package: '%s'" % (sstate_hardcode_cmd)) | ||
449 | subprocess.call(sstate_hardcode_cmd, shell=True) | ||
450 | |||
451 | # If the fixmefn is empty, remove it.. | ||
452 | if os.stat(fixmefn).st_size == 0: | ||
453 | os.remove(fixmefn) | ||
454 | else: | ||
455 | bb.note("Replacing absolute paths in fixmepath file: '%s'" % (sstate_filelist_relative_cmd)) | ||
456 | subprocess.call(sstate_filelist_relative_cmd, shell=True) | ||
457 | |||
458 | def sstate_package(ss, d): | ||
459 | import oe.path | ||
460 | |||
461 | def make_relative_symlink(path, outputpath, d): | ||
462 | # Replace out absolute TMPDIR paths in symlinks with relative ones | ||
463 | if not os.path.islink(path): | ||
464 | return | ||
465 | link = os.readlink(path) | ||
466 | if not os.path.isabs(link): | ||
467 | return | ||
468 | if not link.startswith(tmpdir): | ||
469 | return | ||
470 | |||
471 | depth = outputpath.rpartition(tmpdir)[2].count('/') | ||
472 | base = link.partition(tmpdir)[2].strip() | ||
473 | while depth > 1: | ||
474 | base = "/.." + base | ||
475 | depth -= 1 | ||
476 | base = "." + base | ||
477 | |||
478 | bb.debug(2, "Replacing absolute path %s with relative path %s for %s" % (link, base, outputpath)) | ||
479 | os.remove(path) | ||
480 | os.symlink(base, path) | ||
481 | |||
482 | tmpdir = d.getVar('TMPDIR', True) | ||
483 | |||
484 | sstatebuild = d.expand("${WORKDIR}/sstate-build-%s/" % ss['task']) | ||
485 | sstatepkg = d.getVar('SSTATE_PKG', True) + '_'+ ss['task'] + ".tgz" | ||
486 | bb.utils.remove(sstatebuild, recurse=True) | ||
487 | bb.utils.mkdirhier(sstatebuild) | ||
488 | bb.utils.mkdirhier(os.path.dirname(sstatepkg)) | ||
489 | for state in ss['dirs']: | ||
490 | if not os.path.exists(state[1]): | ||
491 | continue | ||
492 | srcbase = state[0].rstrip("/").rsplit('/', 1)[0] | ||
493 | for walkroot, dirs, files in os.walk(state[1]): | ||
494 | for file in files: | ||
495 | srcpath = os.path.join(walkroot, file) | ||
496 | dstpath = srcpath.replace(state[1], state[2]) | ||
497 | make_relative_symlink(srcpath, dstpath, d) | ||
498 | for dir in dirs: | ||
499 | srcpath = os.path.join(walkroot, dir) | ||
500 | dstpath = srcpath.replace(state[1], state[2]) | ||
501 | make_relative_symlink(srcpath, dstpath, d) | ||
502 | bb.debug(2, "Preparing tree %s for packaging at %s" % (state[1], sstatebuild + state[0])) | ||
503 | oe.path.copyhardlinktree(state[1], sstatebuild + state[0]) | ||
504 | |||
505 | workdir = d.getVar('WORKDIR', True) | ||
506 | for plain in ss['plaindirs']: | ||
507 | pdir = plain.replace(workdir, sstatebuild) | ||
508 | bb.utils.mkdirhier(plain) | ||
509 | bb.utils.mkdirhier(pdir) | ||
510 | oe.path.copyhardlinktree(plain, pdir) | ||
511 | |||
512 | d.setVar('SSTATE_BUILDDIR', sstatebuild) | ||
513 | d.setVar('SSTATE_PKG', sstatepkg) | ||
514 | sstate_hardcode_path(d) | ||
515 | bb.build.exec_func('sstate_create_package', d) | ||
516 | |||
517 | bb.siggen.dump_this_task(sstatepkg + ".siginfo", d) | ||
518 | |||
519 | return | ||
520 | |||
521 | def pstaging_fetch(sstatefetch, sstatepkg, d): | ||
522 | import bb.fetch2 | ||
523 | |||
524 | # Only try and fetch if the user has configured a mirror | ||
525 | mirrors = d.getVar('SSTATE_MIRRORS', True) | ||
526 | if not mirrors: | ||
527 | return | ||
528 | |||
529 | # Copy the data object and override DL_DIR and SRC_URI | ||
530 | localdata = bb.data.createCopy(d) | ||
531 | bb.data.update_data(localdata) | ||
532 | |||
533 | dldir = localdata.expand("${SSTATE_DIR}") | ||
534 | bb.utils.mkdirhier(dldir) | ||
535 | |||
536 | localdata.delVar('MIRRORS') | ||
537 | localdata.delVar('FILESPATH') | ||
538 | localdata.setVar('DL_DIR', dldir) | ||
539 | localdata.setVar('PREMIRRORS', mirrors) | ||
540 | |||
541 | # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, | ||
542 | # we'll want to allow network access for the current set of fetches. | ||
543 | if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": | ||
544 | localdata.delVar('BB_NO_NETWORK') | ||
545 | |||
546 | # Try a fetch from the sstate mirror, if it fails just return and | ||
547 | # we will build the package | ||
548 | for srcuri in ['file://{0}'.format(sstatefetch), | ||
549 | 'file://{0}.siginfo'.format(sstatefetch)]: | ||
550 | localdata.setVar('SRC_URI', srcuri) | ||
551 | try: | ||
552 | fetcher = bb.fetch2.Fetch([srcuri], localdata, cache=False) | ||
553 | fetcher.download() | ||
554 | |||
555 | # Need to optimise this, if using file:// urls, the fetcher just changes the local path | ||
556 | # For now work around by symlinking | ||
557 | localpath = bb.data.expand(fetcher.localpath(srcuri), localdata) | ||
558 | if localpath != sstatepkg and os.path.exists(localpath) and not os.path.exists(sstatepkg): | ||
559 | os.symlink(localpath, sstatepkg) | ||
560 | |||
561 | except bb.fetch2.BBFetchException: | ||
562 | break | ||
563 | |||
564 | def sstate_setscene(d): | ||
565 | shared_state = sstate_state_fromvars(d) | ||
566 | accelerate = sstate_installpkg(shared_state, d) | ||
567 | if not accelerate: | ||
568 | raise bb.build.FuncFailed("No suitable staging package found") | ||
569 | |||
570 | python sstate_task_prefunc () { | ||
571 | shared_state = sstate_state_fromvars(d) | ||
572 | sstate_clean(shared_state, d) | ||
573 | } | ||
574 | |||
575 | python sstate_task_postfunc () { | ||
576 | shared_state = sstate_state_fromvars(d) | ||
577 | sstate_install(shared_state, d) | ||
578 | for intercept in shared_state['interceptfuncs']: | ||
579 | bb.build.exec_func(intercept, d) | ||
580 | omask = os.umask(002) | ||
581 | if omask != 002: | ||
582 | bb.note("Using umask 002 (not %0o) for sstate packaging" % omask) | ||
583 | sstate_package(shared_state, d) | ||
584 | os.umask(omask) | ||
585 | } | ||
586 | |||
587 | |||
588 | # | ||
589 | # Shell function to generate a sstate package from a directory | ||
590 | # set as SSTATE_BUILDDIR | ||
591 | # | ||
592 | sstate_create_package () { | ||
593 | cd ${SSTATE_BUILDDIR} | ||
594 | TFILE=`mktemp ${SSTATE_PKG}.XXXXXXXX` | ||
595 | # Need to handle empty directories | ||
596 | if [ "$(ls -A)" ]; then | ||
597 | set +e | ||
598 | tar -czf $TFILE * | ||
599 | if [ $? -ne 0 ] && [ $? -ne 1 ]; then | ||
600 | exit 1 | ||
601 | fi | ||
602 | set -e | ||
603 | else | ||
604 | tar -cz --file=$TFILE --files-from=/dev/null | ||
605 | fi | ||
606 | chmod 0664 $TFILE | ||
607 | mv -f $TFILE ${SSTATE_PKG} | ||
608 | |||
609 | cd ${WORKDIR} | ||
610 | rm -rf ${SSTATE_BUILDDIR} | ||
611 | } | ||
612 | |||
613 | # | ||
614 | # Shell function to decompress and prepare a package for installation | ||
615 | # | ||
616 | sstate_unpack_package () { | ||
617 | mkdir -p ${SSTATE_INSTDIR} | ||
618 | cd ${SSTATE_INSTDIR} | ||
619 | tar -xmvzf ${SSTATE_PKG} | ||
620 | } | ||
621 | |||
622 | BB_HASHCHECK_FUNCTION = "sstate_checkhashes" | ||
623 | |||
624 | def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d): | ||
625 | |||
626 | ret = [] | ||
627 | missed = [] | ||
628 | |||
629 | def getpathcomponents(task, d): | ||
630 | # Magic data from BB_HASHFILENAME | ||
631 | splithashfn = sq_hashfn[task].split(" ") | ||
632 | spec = splithashfn[1] | ||
633 | extrapath = splithashfn[0] | ||
634 | |||
635 | tname = sq_task[task][3:] | ||
636 | |||
637 | if tname in ["fetch", "unpack", "patch", "populate_lic"] and splithashfn[2]: | ||
638 | spec = splithashfn[2] | ||
639 | extrapath = "" | ||
640 | |||
641 | return spec, extrapath, tname | ||
642 | |||
643 | |||
644 | for task in range(len(sq_fn)): | ||
645 | |||
646 | spec, extrapath, tname = getpathcomponents(task, d) | ||
647 | |||
648 | sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz.siginfo") | ||
649 | |||
650 | if os.path.exists(sstatefile): | ||
651 | bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) | ||
652 | ret.append(task) | ||
653 | continue | ||
654 | else: | ||
655 | missed.append(task) | ||
656 | bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) | ||
657 | |||
658 | mirrors = d.getVar("SSTATE_MIRRORS", True) | ||
659 | if mirrors: | ||
660 | # Copy the data object and override DL_DIR and SRC_URI | ||
661 | localdata = bb.data.createCopy(d) | ||
662 | bb.data.update_data(localdata) | ||
663 | |||
664 | dldir = localdata.expand("${SSTATE_DIR}") | ||
665 | localdata.setVar('DL_DIR', dldir) | ||
666 | localdata.setVar('PREMIRRORS', mirrors) | ||
667 | |||
668 | bb.debug(2, "SState using premirror of: %s" % mirrors) | ||
669 | |||
670 | # if BB_NO_NETWORK is set but we also have SSTATE_MIRROR_ALLOW_NETWORK, | ||
671 | # we'll want to allow network access for the current set of fetches. | ||
672 | if localdata.getVar('BB_NO_NETWORK', True) == "1" and localdata.getVar('SSTATE_MIRROR_ALLOW_NETWORK', True) == "1": | ||
673 | localdata.delVar('BB_NO_NETWORK') | ||
674 | |||
675 | for task in range(len(sq_fn)): | ||
676 | if task in ret: | ||
677 | continue | ||
678 | |||
679 | spec, extrapath, tname = getpathcomponents(task, d) | ||
680 | |||
681 | sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz.siginfo") | ||
682 | |||
683 | srcuri = "file://" + sstatefile | ||
684 | localdata.setVar('SRC_URI', srcuri) | ||
685 | bb.debug(2, "SState: Attempting to fetch %s" % srcuri) | ||
686 | |||
687 | try: | ||
688 | fetcher = bb.fetch2.Fetch(srcuri.split(), localdata) | ||
689 | fetcher.checkstatus() | ||
690 | bb.debug(2, "SState: Successful fetch test for %s" % srcuri) | ||
691 | ret.append(task) | ||
692 | if task in missed: | ||
693 | missed.remove(task) | ||
694 | except: | ||
695 | missed.append(task) | ||
696 | bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri) | ||
697 | pass | ||
698 | |||
699 | inheritlist = d.getVar("INHERIT", True) | ||
700 | if "toaster" in inheritlist: | ||
701 | evdata = {'missed': [], 'found': []}; | ||
702 | for task in missed: | ||
703 | spec, extrapath, tname = getpathcomponents(task, d) | ||
704 | sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") | ||
705 | evdata['missed'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) | ||
706 | for task in ret: | ||
707 | spec, extrapath, tname = getpathcomponents(task, d) | ||
708 | sstatefile = d.expand(extrapath + generate_sstatefn(spec, sq_hash[task], d) + "_" + tname + ".tgz") | ||
709 | evdata['found'].append( (sq_fn[task], sq_task[task], sq_hash[task], sstatefile ) ) | ||
710 | bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d) | ||
711 | |||
712 | return ret | ||
713 | |||
714 | BB_SETSCENE_DEPVALID = "setscene_depvalid" | ||
715 | |||
716 | def setscene_depvalid(task, taskdependees, notneeded, d): | ||
717 | # taskdependees is a dict of tasks which depend on task, each being a 3 item list of [PN, TASKNAME, FILENAME] | ||
718 | # task is included in taskdependees too | ||
719 | |||
720 | bb.debug(2, "Considering setscene task: %s" % (str(taskdependees[task]))) | ||
721 | |||
722 | def isNativeCross(x): | ||
723 | return x.endswith("-native") or x.endswith("-cross") or x.endswith("-cross-initial") or x.endswith("-crosssdk") or x.endswith("-crosssdk-initial") | ||
724 | |||
725 | def isPostInstDep(x): | ||
726 | if x in ["qemu-native", "gdk-pixbuf-native", "qemuwrapper-cross", "depmodwrapper-cross", "systemd-systemctl-native", "gtk-update-icon-cache-native"]: | ||
727 | return True | ||
728 | return False | ||
729 | |||
730 | # We only need to trigger populate_lic through direct dependencies | ||
731 | if taskdependees[task][1] == "do_populate_lic": | ||
732 | return True | ||
733 | |||
734 | for dep in taskdependees: | ||
735 | bb.debug(2, " considering dependency: %s" % (str(taskdependees[dep]))) | ||
736 | if task == dep: | ||
737 | continue | ||
738 | if dep in notneeded: | ||
739 | continue | ||
740 | # do_package_write_* and do_package doesn't need do_package | ||
741 | if taskdependees[task][1] == "do_package" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata']: | ||
742 | continue | ||
743 | # do_package_write_* and do_package doesn't need do_populate_sysroot, unless is a postinstall dependency | ||
744 | if taskdependees[task][1] == "do_populate_sysroot" and taskdependees[dep][1] in ['do_package', 'do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata']: | ||
745 | if isPostInstDep(taskdependees[task][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm']: | ||
746 | return False | ||
747 | continue | ||
748 | # Native/Cross packages don't exist and are noexec anyway | ||
749 | if isNativeCross(taskdependees[dep][0]) and taskdependees[dep][1] in ['do_package_write_deb', 'do_package_write_ipk', 'do_package_write_rpm', 'do_packagedata', 'do_package']: | ||
750 | continue | ||
751 | |||
752 | # Consider sysroot depending on sysroot tasks | ||
753 | if taskdependees[task][1] == 'do_populate_sysroot' and taskdependees[dep][1] == 'do_populate_sysroot': | ||
754 | # base-passwd/shadow-sysroot don't need their dependencies | ||
755 | if taskdependees[dep][0].endswith(("base-passwd", "shadow-sysroot")): | ||
756 | continue | ||
757 | # Nothing need depend on libc-initial/gcc-cross-initial | ||
758 | if taskdependees[task][0].endswith("-initial"): | ||
759 | continue | ||
760 | # Native/Cross populate_sysroot need their dependencies | ||
761 | if isNativeCross(taskdependees[task][0]) and isNativeCross(taskdependees[dep][0]): | ||
762 | return False | ||
763 | # Target populate_sysroot depended on by cross tools need to be installed | ||
764 | if isNativeCross(taskdependees[dep][0]): | ||
765 | return False | ||
766 | # Native/cross tools depended upon by target sysroot are not needed | ||
767 | if isNativeCross(taskdependees[task][0]): | ||
768 | continue | ||
769 | # Target populate_sysroot need their dependencies | ||
770 | return False | ||
771 | |||
772 | # This is due to the [depends] in useradd.bbclass complicating matters | ||
773 | # The logic *is* reversed here due to the way hard setscene dependencies are injected | ||
774 | if taskdependees[task][1] == 'do_package' and taskdependees[dep][0].endswith(('shadow-native', 'shadow-sysroot', 'base-passwd', 'pseudo-native')) and taskdependees[dep][1] == 'do_populate_sysroot': | ||
775 | continue | ||
776 | |||
777 | # Safe fallthrough default | ||
778 | bb.debug(2, " Default setscene dependency fall through due to dependency: %s" % (str(taskdependees[dep]))) | ||
779 | return False | ||
780 | return True | ||
781 | |||
782 | addhandler sstate_eventhandler | ||
783 | sstate_eventhandler[eventmask] = "bb.build.TaskSucceeded" | ||
784 | python sstate_eventhandler() { | ||
785 | d = e.data | ||
786 | # When we write an sstate package we rewrite the SSTATE_PKG | ||
787 | spkg = d.getVar('SSTATE_PKG', True) | ||
788 | if not spkg.endswith(".tgz"): | ||
789 | taskname = d.getVar("BB_RUNTASK", True)[3:] | ||
790 | spec = d.getVar('SSTATE_PKGSPEC', True) | ||
791 | swspec = d.getVar('SSTATE_SWSPEC', True) | ||
792 | if taskname in ["fetch", "unpack", "patch", "populate_lic"] and swspec: | ||
793 | d.setVar("SSTATE_PKGSPEC", "${SSTATE_SWSPEC}") | ||
794 | d.setVar("SSTATE_EXTRAPATH", "") | ||
795 | sstatepkg = d.getVar('SSTATE_PKG', True) | ||
796 | bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d) | ||
797 | } | ||
798 | |||