summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2012-01-19 10:32:09 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-01-19 14:34:32 +0000
commit2f4f8ef0fc5f2ff024e21903520a2680b0524721 (patch)
treee41336ca9b1398f1b4963017114d9650564b8fbc
parentffa93f0e25565475d9d8c410a79c59e13759182b (diff)
downloadpoky-2f4f8ef0fc5f2ff024e21903520a2680b0524721.tar.gz
buildhistory: record additional image info
Record some additional information about images - the uncompressed size of the final image as well as the values of various variables that may have influenced its contents. This is recorded in a machine-readable "image-info.txt" file similar in structure to the package history files. Also add some code to analyse changes to these values. (Most of the variable values aren't monitored directly but will be used as contextual information when they change at the same time as the content of the image changing.) (From OE-Core rev: 459ed6759a307b389f6ec1874136ec9aa0749120) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/buildhistory.bbclass18
-rw-r--r--meta/lib/oe/buildhistory_analysis.py51
2 files changed, 47 insertions, 22 deletions
diff --git a/meta/classes/buildhistory.bbclass b/meta/classes/buildhistory.bbclass
index 34cc297909..69a9d02a5a 100644
--- a/meta/classes/buildhistory.bbclass
+++ b/meta/classes/buildhistory.bbclass
@@ -312,6 +312,14 @@ buildhistory_get_imageinfo() {
312 # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo 312 # This awk script is somewhat messy, but handles where the size is not printed for device files under pseudo
313 ( cd ${IMAGE_ROOTFS} && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt ) 313 ( cd ${IMAGE_ROOTFS} && find . -ls | awk '{ if ( $7 ~ /[0-9]/ ) printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, $7, $11, $12, $13 ; else printf "%s %10-s %10-s %10s %s %s %s\n", $3, $5, $6, 0, $10, $11, $12 }' > ${BUILDHISTORY_DIR_IMAGE}/files-in-image.txt )
314 314
315 # Record some machine-readable meta-information about the image
316 echo -n > ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
317 cat >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt <<END
318${@buildhistory_get_imagevars(d)}
319END
320 imagesize=`du -ks ${IMAGE_ROOTFS} | awk '{ print $1 }'`
321 echo "IMAGESIZE = $imagesize" >> ${BUILDHISTORY_DIR_IMAGE}/image-info.txt
322
315 # Add some configuration information 323 # Add some configuration information
316 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id 324 echo "${MACHINE}: ${IMAGE_BASENAME} configured for ${DISTRO} ${DISTRO_VERSION}" > ${BUILDHISTORY_DIR_IMAGE}/build-id
317 325
@@ -330,6 +338,16 @@ def buildhistory_get_layers(d):
330 return layertext 338 return layertext
331 339
332 340
341def buildhistory_get_imagevars(d):
342 imagevars = "DISTRO DISTRO_VERSION USER_CLASSES IMAGE_CLASSES IMAGE_FEATURES IMAGE_LINGUAS IMAGE_INSTALL BAD_RECOMMENDATIONS ROOTFS_POSTPROCESS_COMMAND IMAGE_POSTPROCESS_COMMAND"
343
344 ret = ""
345 for var in imagevars.split():
346 value = d.getVar(var, True) or ""
347 ret += "%s = %s\n" % (var, value)
348 return ret.rstrip('\n')
349
350
333buildhistory_commit() { 351buildhistory_commit() {
334 if [ ! -d ${BUILDHISTORY_DIR} ] ; then 352 if [ ! -d ${BUILDHISTORY_DIR} ] ; then
335 # Code above that creates this dir never executed, so there can't be anything to commit 353 # Code above that creates this dir never executed, so there can't be anything to commit
diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py
index 9f42fe38ac..d3c0448fd1 100644
--- a/meta/lib/oe/buildhistory_analysis.py
+++ b/meta/lib/oe/buildhistory_analysis.py
@@ -15,16 +15,15 @@ import git
15 15
16 16
17# How to display fields 17# How to display fields
18pkg_list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST'] 18list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST', 'USER_CLASSES', 'IMAGE_CLASSES', 'IMAGE_FEATURES', 'IMAGE_LINGUAS', 'IMAGE_INSTALL', 'BAD_RECOMMENDATIONS']
19pkg_numeric_fields = ['PKGSIZE'] 19numeric_fields = ['PKGSIZE', 'IMAGESIZE']
20# Fields to monitor 20# Fields to monitor
21pkg_monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE'] 21monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE', 'IMAGESIZE']
22# Percentage change to alert for numeric fields 22# Percentage change to alert for numeric fields
23pkg_monitor_numeric_threshold = 20 23monitor_numeric_threshold = 20
24# Image files to monitor 24# Image files to monitor (note that image-info.txt is handled separately)
25img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt'] 25img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
26 26
27
28class ChangeRecord: 27class ChangeRecord:
29 def __init__(self, path, fieldname, oldvalue, newvalue): 28 def __init__(self, path, fieldname, oldvalue, newvalue):
30 self.path = path 29 self.path = path
@@ -34,13 +33,13 @@ class ChangeRecord:
34 self.filechanges = None 33 self.filechanges = None
35 34
36 def __str__(self): 35 def __str__(self):
37 if self.fieldname in pkg_list_fields: 36 if self.fieldname in list_fields:
38 aitems = self.oldvalue.split(' ') 37 aitems = self.oldvalue.split(' ')
39 bitems = self.newvalue.split(' ') 38 bitems = self.newvalue.split(' ')
40 removed = list(set(aitems) - set(bitems)) 39 removed = list(set(aitems) - set(bitems))
41 added = list(set(bitems) - set(aitems)) 40 added = list(set(bitems) - set(aitems))
42 return '%s: %s:%s%s' % (self.path, self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '') 41 return '%s: %s:%s%s' % (self.path, self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '')
43 elif self.fieldname in pkg_numeric_fields: 42 elif self.fieldname in numeric_fields:
44 aval = int(self.oldvalue) 43 aval = int(self.oldvalue)
45 bval = int(self.newvalue) 44 bval = int(self.newvalue)
46 percentchg = ((bval - aval) / float(aval)) * 100 45 percentchg = ((bval - aval) / float(aval)) * 100
@@ -190,6 +189,25 @@ def compare_lists(alines, blines):
190 return filechanges 189 return filechanges
191 190
192 191
192def compare_dict_blobs(path, ablob, bblob, report_all):
193 adict = blob_to_dict(ablob)
194 bdict = blob_to_dict(bblob)
195
196 changes = []
197 for key in adict:
198 if report_all or key in monitor_fields:
199 if adict[key] != bdict[key]:
200 if (not report_all) and key in numeric_fields:
201 aval = int(adict[key])
202 bval = int(bdict[key])
203 percentchg = ((bval - aval) / float(aval)) * 100
204 if percentchg < monitor_numeric_threshold:
205 continue
206 chg = ChangeRecord(path, key, adict[key], bdict[key])
207 changes.append(chg)
208 return changes
209
210
193def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False): 211def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False):
194 repo = git.Repo(repopath) 212 repo = git.Repo(repopath)
195 assert repo.bare == False 213 assert repo.bare == False
@@ -200,20 +218,7 @@ def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False)
200 for d in diff.iter_change_type('M'): 218 for d in diff.iter_change_type('M'):
201 path = os.path.dirname(d.a_blob.path) 219 path = os.path.dirname(d.a_blob.path)
202 if path.startswith('packages/'): 220 if path.startswith('packages/'):
203 adict = blob_to_dict(d.a_blob) 221 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
204 bdict = blob_to_dict(d.b_blob)
205
206 for key in adict:
207 if report_all or key in pkg_monitor_fields:
208 if adict[key] != bdict[key]:
209 if (not report_all) and key in pkg_numeric_fields:
210 aval = int(adict[key])
211 bval = int(bdict[key])
212 percentchg = ((bval - aval) / float(aval)) * 100
213 if percentchg < pkg_monitor_numeric_threshold:
214 continue
215 chg = ChangeRecord(path, key, adict[key], bdict[key])
216 changes.append(chg)
217 elif path.startswith('images/'): 222 elif path.startswith('images/'):
218 filename = os.path.basename(d.a_blob.path) 223 filename = os.path.basename(d.a_blob.path)
219 if filename in img_monitor_files: 224 if filename in img_monitor_files:
@@ -236,5 +241,7 @@ def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False)
236 else: 241 else:
237 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read()) 242 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read())
238 changes.append(chg) 243 changes.append(chg)
244 elif filename == 'image-info.txt':
245 changes.extend(compare_dict_blobs(path, d.a_blob, d.b_blob, report_all))
239 246
240 return changes 247 return changes