summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2010-08-04 11:34:18 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2010-08-19 20:06:25 +0100
commitc7a80295702a65c4bd528accbb4bb6f9998505f5 (patch)
treee6afb04f1cb850460e10b47fb15ce007a1ba5d5b
parent787c1cf81195e97dda5fdab3a0a5d8bda97f6770 (diff)
downloadpoky-c7a80295702a65c4bd528accbb4bb6f9998505f5.tar.gz
Start sstate
Add pre clean hook Add sstate_installpkg sstate: Tie all pieces together with a setscene function Use pythonic functions rather than os.system() Implement sstate_clean function package_ipk: clean shared state in prestate packaged-staging2: implement fetching packages from mirror Copy add staging_fetch() from packaged-staging.bbclass packaged-staging2: make the output slightly friendlier packaged-staging2: handle directories in the shared state Add directories to the manifest when installing and remove them if empty when cleaning. Move most of the sstate code into the class itself removing the need for heavy boilderplate packaged-staging2: Programatically ensure directories are last in the manifest packaged-staging2: add cleanall packaged-staging2.bbclass: Connect in cleanall function to do_clean We want to ensure that directories appear at the end of the manifest so that when we test to see if they should be deleted any contents added by the task will have been removed first. pstage2: Use oe.path.copytree packaged-staging2: support lockfiles and plaindirs Signed-off-by: Richard Purdie <rpurdie@linux.intel.com> Signed-off-by: Joshua Lock <josh@linux.intel.com>
-rw-r--r--meta/classes/packaged-staging2.bbclass297
-rw-r--r--meta/conf/distro/poky.conf2
2 files changed, 298 insertions, 1 deletions
diff --git a/meta/classes/packaged-staging2.bbclass b/meta/classes/packaged-staging2.bbclass
new file mode 100644
index 0000000000..6476257fc8
--- /dev/null
+++ b/meta/classes/packaged-staging2.bbclass
@@ -0,0 +1,297 @@
1PSTAGE2_MANIFESTS = "${TMPDIR}/pstagelogs"
2PSTAGE2_MANFILEPREFIX = "${PSTAGE2_MANIFESTS}/manifest-${PSTAGE2_PKGARCH}-${PN}"
3
4
5PSTAGE2_PKGARCH = "${TARGET_ARCH}"
6PSTAGE2_PKGVERSION = "${PV}-${PR}"
7PSTAGE2_PKGPN = "${@bb.data.expand('staging-${PN}-${MULTIMACH_ARCH}${TARGET_VENDOR}-${TARGET_OS}', d).replace('_', '-')}"
8
9PSTAGE2_PKGNAME = "${PSTAGE2_PKGPN}_${PSTAGE2_PKGVERSION}_${PSTAGE2_PKGARCH}"
10PSTAGE2_EXTRAPATH ?= ""
11PSTAGE2_PKGPATH = "${DISTRO}/${OELAYOUT_ABI}${PSTAGE2_EXTRAPATH}"
12PSTAGE2_PKG = "${PSTAGE_DIR}2/${PSTAGE2_PKGPATH}/${PSTAGE2_PKGNAME}"
13
14PSTAGE2_SCAN_CMD ?= "find ${PSTAGE2_BUILDDIR} \( -name "*.la" -o -name "*-config" \) -type f"
15
16python () {
17 if bb.data.inherits_class('native', d):
18 bb.data.setVar('PSTAGE2_PKGARCH', bb.data.getVar('BUILD_ARCH', d), d)
19 elif bb.data.inherits_class('cross', d) or bb.data.inherits_class('crosssdk', d):
20 bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${BUILD_ARCH}_${TARGET_ARCH}", d), d)
21 elif bb.data.inherits_class('nativesdk', d):
22 bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${SDK_ARCH}", d), d)
23 elif bb.data.inherits_class('cross-canadian', d):
24 bb.data.setVar('PSTAGE2_PKGARCH', bb.data.expand("${SDK_ARCH}_${TARGET_ARCH}", d), d)
25
26 # These classes encode staging paths into their scripts data so can only be
27 # reused if we manipulate the paths
28 if bb.data.inherits_class('native', d) or bb.data.inherits_class('cross', d) or bb.data.inherits_class('sdk', d) or bb.data.inherits_class('crosssdk', d):
29 scan_cmd = "grep -Irl ${STAGING_DIR} ${PSTAGE2_BUILDDIR}"
30 bb.data.setVar('PSTAGE2_SCAN_CMD', scan_cmd, d)
31
32 for task in (bb.data.getVar('SSTATETASKS', d, True) or "").split():
33 funcs = bb.data.getVarFlag(task, 'prefuncs', d) or ""
34 funcs = "sstate_task_prefunc " + funcs
35 bb.data.setVarFlag(task, 'prefuncs', funcs, d)
36 funcs = bb.data.getVarFlag(task, 'postfuncs', d) or ""
37 funcs = "sstate_task_postfunc " + funcs
38 bb.data.setVarFlag(task, 'postfuncs', funcs, d)
39}
40
41def sstate_init(name, d):
42 ss = {}
43 ss['name'] = name
44 ss['dirs'] = []
45 ss['plaindirs'] = []
46 ss['lockfiles'] = []
47 return ss
48
49def sstate_state_fromvars(d):
50 task = bb.data.getVar('BB_CURRENTTASK', d, True)
51 if not task:
52 bb.fatal("sstate code running without task context?!")
53 task = task.replace("_setscene", "")
54
55 name = bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-name', d), d)
56 inputs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-inputdirs', d) or "", d)).split()
57 outputs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-outputdirs', d) or "", d)).split()
58 plaindirs = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-plaindirs', d) or "", d)).split()
59 lockfiles = (bb.data.expand(bb.data.getVarFlag("do_" + task, 'sstate-lockfile', d) or "", d)).split()
60 if not name or len(inputs) != len(outputs):
61 bb.fatal("sstate variables not setup correctly?!")
62
63 ss = sstate_init(name, d)
64 for i in range(len(inputs)):
65 sstate_add(ss, inputs[i], outputs[i], d)
66 ss['lockfiles'] = lockfiles
67 ss['plaindirs'] = plaindirs
68 return ss
69
70def sstate_add(ss, source, dest, d):
71 srcbase = os.path.basename(source)
72 ss['dirs'].append([srcbase, source, dest])
73 return ss
74
75def sstate_install(ss, d):
76 import oe.path
77
78 sharedfiles = []
79 shareddirs = []
80 bb.mkdirhier(bb.data.expand("${PSTAGE2_MANIFESTS}", d))
81 manifest = bb.data.expand("${PSTAGE2_MANFILEPREFIX}.%s" % ss['name'], d)
82
83 if os.access(manifest, os.R_OK):
84 bb.fatal("Package already staged (%s)?!" % manifest)
85
86 locks = []
87 for lock in ss['lockfiles']:
88 locks.append(bb.utils.lockfile(lock))
89
90 for state in ss['dirs']:
91 oe.path.copytree(state[1], state[2])
92 for walkroot, dirs, files in os.walk(state[1]):
93 for file in files:
94 srcpath = os.path.join(walkroot, file)
95 dstpath = srcpath.replace(state[1], state[2])
96 bb.debug(2, "Staging %s to %s" % (srcpath, dstpath))
97 sharedfiles.append(dstpath)
98 for dir in dirs:
99 dir = os.path.join(state[2], dir)
100 if not dir.endswith("/"):
101 dir = dir + "/"
102 shareddirs.append(dir)
103 f = open(manifest, "w")
104 for file in sharedfiles:
105 f.write(file + "\n")
106 # We want to ensure that directories appear at the end of the manifest
107 # so that when we test to see if they should be deleted any contents
108 # added by the task will have been removed first.
109 for dir in shareddirs:
110 f.write(dir + "\n")
111 f.close()
112
113 for lock in locks:
114 bb.utils.unlockfile(lock)
115
116def sstate_installpkg(ss, d):
117 import oe.path
118
119 pstageinst = bb.data.expand("${WORKDIR}/pstage-install-%s/" % ss['name'], d)
120 pstagepkg = bb.data.getVar('PSTAGE2_PKG', d, True) + '_' + ss['name'] + ".tgz"
121
122 if not os.path.exists(pstagepkg):
123 pstaging_fetch(pstagepkg, d)
124
125 if not os.path.isfile(pstagepkg):
126 bb.note("Staging package %s does not exist" % pstagepkg)
127 return False
128
129 sstate_clean(ss, d)
130
131 bb.data.setVar('PSTAGE2_INSTDIR', pstageinst, d)
132 bb.data.setVar('PSTAGE2_PKG', pstagepkg, d)
133 bb.build.exec_func('sstate_unpack_package', d)
134
135 # Fixup hardcoded paths
136 fixmefn = pstageinst + "fixmepath"
137 if os.path.isfile(fixmefn):
138 staging = bb.data.getVar('STAGING_DIR', d, True)
139 fixmefd = open(fixmefn, "r")
140 fixmefiles = fixmefd.readlines()
141 fixmefd.close()
142 for file in fixmefiles:
143 os.system("sed -i -e s:FIXMESTAGINGDIR:%s:g %s" % (staging, pstageinst + file))
144
145 for state in ss['dirs']:
146 if os.path.exists(state[1]):
147 oe.path.remove(state[1])
148 oe.path.copytree(pstageinst + state[0], state[1])
149 sstate_install(ss, d)
150
151 for plain in ss['plaindirs']:
152 bb.mkdirhier(pstageinst + plain)
153 oe.path.copytree(pstageinst + plain, bb.data.getVar('WORKDIR', d, True) + plain)
154
155 return True
156
157def sstate_clean_manifest(manifest, d):
158 import oe.path
159
160 if not os.path.exists(manifest):
161 return
162
163 mfile = open(manifest)
164 entries = mfile.readlines()
165 mfile.close()
166
167 for entry in entries:
168 entry = entry.strip()
169 if entry.endswith("/"):
170 if os.path.exists(entry) and len(os.listdir(entry)) == 0:
171 os.rmdir(entry)
172 else:
173 oe.path.remove(entry)
174
175 oe.path.remove(manifest)
176
177def sstate_clean(ss, d):
178
179 manifest = bb.data.expand("${PSTAGE2_MANFILEPREFIX}.%s" % ss['name'], d)
180
181 locks = []
182 for lock in ss['lockfiles']:
183 locks.append(bb.utils.lockfile(lock))
184
185 sstate_clean_manifest(manifest, d)
186
187 for lock in locks:
188 bb.utils.unlockfile(lock)
189
190python sstate_cleanall() {
191 import fnmatch
192
193 bb.note("Removing %s from staging" % bb.data.getVar('PN', d, True))
194
195 manifest_dir = bb.data.getVar('PSTAGE2_MANIFESTS', d, True)
196 manifest_pattern = bb.data.expand("manifest-${PN}.*", d)
197
198 for manifest in (os.listdir(manifest_dir)):
199 if fnmatch.fnmatch(manifest, manifest_pattern):
200 sstate_clean_manifest(manifest_dir + "/" + manifest, d)
201}
202
203do_clean[postfuncs] += "sstate_cleanall"
204do_clean[dirs] += "${PSTAGE2_MANIFESTS}"
205
206def sstate_package(ss, d):
207 import oe.path
208
209 pstagebuild = bb.data.expand("${WORKDIR}/pstage-build-%s/" % ss['name'], d)
210 pstagepkg = bb.data.getVar('PSTAGE2_PKG', d, True) + '_'+ ss['name'] + ".tgz"
211 bb.mkdirhier(pstagebuild)
212 bb.mkdirhier(os.path.dirname(pstagepkg))
213 for state in ss['dirs']:
214 srcbase = state[0].rstrip("/").rsplit('/', 1)[0]
215 oe.path.copytree(state[1], pstagebuild + state[0])
216 for walkroot, dirs, files in os.walk(state[1]):
217 for file in files:
218 srcpath = os.path.join(walkroot, file)
219 dstpath = srcpath.replace(state[1], pstagebuild + state[0])
220 bb.debug(2, "Preparing %s for packaging at %s" % (srcpath, dstpath))
221
222 workdir = bb.data.getVar('WORKDIR', d, True)
223 for plain in ss['plaindirs']:
224 pdir = plain.replace(workdir, pstagebuild)
225 bb.mkdirhier(plain)
226 bb.mkdirhier(pdir)
227 oe.path.copytree(plain, pdir)
228
229 bb.data.setVar('PSTAGE2_BUILDDIR', pstagebuild, d)
230 bb.data.setVar('PSTAGE2_PKG', pstagepkg, d)
231 bb.build.exec_func('sstate_create_package', d)
232
233 return
234
235def pstaging_fetch(pstagepkg, d):
236 import bb.fetch
237
238 # only try and fetch if the user has configured a mirror
239 if bb.data.getVar('PSTAGE_MIRROR', d) != "":
240 # Copy the data object and override DL_DIR and SRC_URI
241 pd = d.createCopy()
242 dldir = bb.data.expand("${PSTAGE_DIR}/${PSTAGE_PKGPATH}", pd)
243 mirror = bb.data.expand("${PSTAGE_MIRROR}/${PSTAGE_PKGPATH}/", pd)
244 srcuri = mirror + os.path.basename(pstagepkg)
245 bb.data.setVar('DL_DIR', dldir, pd)
246 bb.data.setVar('SRC_URI', srcuri, pd)
247
248 # Try a fetch from the pstage mirror, if it fails just return and
249 # we will build the package
250 try:
251 bb.fetch.init([srcuri], pd)
252 bb.fetch.go(pd, [srcuri])
253 except:
254 return
255
256def sstate_setscene(d):
257 shared_state = sstate_state_fromvars(d)
258 accelerate = sstate_installpkg(shared_state, d)
259 if not accelerate:
260 raise bb.build.FuncFailed("No suitable staging package found")
261
262python sstate_task_prefunc () {
263 shared_state = sstate_state_fromvars(d)
264 sstate_clean(shared_state, d)
265}
266
267python sstate_task_postfunc () {
268 shared_state = sstate_state_fromvars(d)
269 sstate_install(shared_state, d)
270 sstate_package(shared_state, d)
271}
272
273
274#
275# Shell function to generate a pstage package from a directory
276# set as PSTAGE2_BUILDDIR
277#
278sstate_create_package () {
279 # Need to remove hardcoded paths and fix these when we install the
280 # staging packages.
281 for i in `${PSTAGE2_SCAN_CMD}` ; do \
282 sed -i -e s:${STAGING_DIR}:FIXMESTAGINGDIR:g $i
283 echo $i | sed -e 's:${PSTAGE2_BUILDDIR}::' >> ${PSTAGE2_BUILDDIR}fixmepath
284 done
285
286 cd ${PSTAGE2_BUILDDIR}
287 tar -cvzf ${PSTAGE2_PKG} *
288}
289
290#
291# Shell function to decompress and prepare a package for installation
292#
293sstate_unpack_package () {
294 mkdir -p ${PSTAGE2_INSTDIR}
295 cd ${PSTAGE2_INSTDIR}
296 tar -xvzf ${PSTAGE2_PKG}
297}
diff --git a/meta/conf/distro/poky.conf b/meta/conf/distro/poky.conf
index df07d04a90..d6d335e5a6 100644
--- a/meta/conf/distro/poky.conf
+++ b/meta/conf/distro/poky.conf
@@ -11,7 +11,7 @@ USER_CLASSES ?= ""
11 11
12PACKAGE_CLASSES ?= "package_ipk" 12PACKAGE_CLASSES ?= "package_ipk"
13INHERIT_INSANE ?= "insane" 13INHERIT_INSANE ?= "insane"
14INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} debian poky devshell ${INHERIT_INSANE} packaged-staging" 14INHERIT += "${PACKAGE_CLASSES} ${USER_CLASSES} debian poky devshell ${INHERIT_INSANE} packaged-staging packaged-staging2"
15# For some reason, this doesn't work 15# For some reason, this doesn't work
16# TARGET_OS ?= "linux" 16# TARGET_OS ?= "linux"
17# TARGET_VENDOR ?= "-poky" 17# TARGET_VENDOR ?= "-poky"