summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/buildhistory_analysis.py
diff options
context:
space:
mode:
Diffstat (limited to 'meta/lib/oe/buildhistory_analysis.py')
-rw-r--r--meta/lib/oe/buildhistory_analysis.py240
1 files changed, 240 insertions, 0 deletions
diff --git a/meta/lib/oe/buildhistory_analysis.py b/meta/lib/oe/buildhistory_analysis.py
new file mode 100644
index 0000000000..9f42fe38ac
--- /dev/null
+++ b/meta/lib/oe/buildhistory_analysis.py
@@ -0,0 +1,240 @@
1# Report significant differences in the buildhistory repository since a specific revision
2#
3# Copyright (C) 2012 Intel Corporation
4# Author: Paul Eggleton <paul.eggleton@linux.intel.com>
5#
6# Note: requires GitPython 0.3.1+
7#
8# You can use this from the command line by running scripts/buildhistory-diff
9#
10
11import sys
12import os.path
13import difflib
14import git
15
16
17# How to display fields
18pkg_list_fields = ['DEPENDS', 'RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILES', 'FILELIST']
19pkg_numeric_fields = ['PKGSIZE']
20# Fields to monitor
21pkg_monitor_fields = ['RDEPENDS', 'RRECOMMENDS', 'PACKAGES', 'FILELIST', 'PKGSIZE']
22# Percentage change to alert for numeric fields
23pkg_monitor_numeric_threshold = 20
24# Image files to monitor
25img_monitor_files = ['installed-package-names.txt', 'files-in-image.txt']
26
27
28class ChangeRecord:
29 def __init__(self, path, fieldname, oldvalue, newvalue):
30 self.path = path
31 self.fieldname = fieldname
32 self.oldvalue = oldvalue
33 self.newvalue = newvalue
34 self.filechanges = None
35
36 def __str__(self):
37 if self.fieldname in pkg_list_fields:
38 aitems = self.oldvalue.split(' ')
39 bitems = self.newvalue.split(' ')
40 removed = list(set(aitems) - set(bitems))
41 added = list(set(bitems) - set(aitems))
42 return '%s: %s:%s%s' % (self.path, self.fieldname, ' removed "%s"' % ' '.join(removed) if removed else '', ' added "%s"' % ' '.join(added) if added else '')
43 elif self.fieldname in pkg_numeric_fields:
44 aval = int(self.oldvalue)
45 bval = int(self.newvalue)
46 percentchg = ((bval - aval) / float(aval)) * 100
47 return '%s: %s changed from %d to %d (%s%d%%)' % (self.path, self.fieldname, aval, bval, '+' if percentchg > 0 else '', percentchg)
48 elif self.fieldname in img_monitor_files:
49 out = 'Changes to %s (%s):\n ' % (self.path, self.fieldname)
50 if self.filechanges:
51 out += '\n '.join(['%s' % i for i in self.filechanges])
52 else:
53 alines = self.oldvalue.splitlines()
54 blines = self.newvalue.splitlines()
55 diff = difflib.unified_diff(alines, blines, self.fieldname, self.fieldname, lineterm='')
56 out += '\n '.join(list(diff))
57 out += '\n --'
58 return out
59 else:
60 return '%s: %s changed from "%s" to "%s"' % (self.path, self.self.fieldname, self.oldvalue, self.newvalue)
61
62
63class FileChange:
64 changetype_add = 'A'
65 changetype_remove = 'R'
66 changetype_type = 'T'
67 changetype_perms = 'P'
68 changetype_ownergroup = 'O'
69 changetype_link = 'L'
70
71 def __init__(self, path, changetype, oldvalue = None, newvalue = None):
72 self.path = path
73 self.changetype = changetype
74 self.oldvalue = oldvalue
75 self.newvalue = newvalue
76
77 def _ftype_str(self, ftype):
78 if ftype == '-':
79 return 'file'
80 elif ftype == 'd':
81 return 'directory'
82 elif ftype == 'l':
83 return 'symlink'
84 elif ftype == 'c':
85 return 'char device'
86 elif ftype == 'b':
87 return 'block device'
88 elif ftype == 'p':
89 return 'fifo'
90 elif ftype == 's':
91 return 'socket'
92 else:
93 return 'unknown (%s)' % ftype
94
95 def __str__(self):
96 if self.changetype == self.changetype_add:
97 return '%s was added' % self.path
98 elif self.changetype == self.changetype_remove:
99 return '%s was removed' % self.path
100 elif self.changetype == self.changetype_type:
101 return '%s changed type from %s to %s' % (self.path, self._ftype_str(self.oldvalue), self._ftype_str(self.newvalue))
102 elif self.changetype == self.changetype_perms:
103 return '%s changed permissions from %s to %s' % (self.path, self.oldvalue, self.newvalue)
104 elif self.changetype == self.changetype_ownergroup:
105 return '%s changed owner/group from %s to %s' % (self.path, self.oldvalue, self.newvalue)
106 elif self.changetype == self.changetype_link:
107 return '%s changed symlink target from %s to %s' % (self.path, self.oldvalue, self.newvalue)
108 else:
109 return '%s changed (unknown)' % self.path
110
111
112def blob_to_dict(blob):
113 alines = blob.data_stream.read().splitlines()
114 adict = {}
115 for line in alines:
116 splitv = [i.strip() for i in line.split('=',1)]
117 if splitv.count > 1:
118 adict[splitv[0]] = splitv[1]
119 return adict
120
121
122def file_list_to_dict(lines):
123 adict = {}
124 for line in lines:
125 # Leave the last few fields intact so we handle file names containing spaces
126 splitv = line.split(None,4)
127 # Grab the path and remove the leading .
128 path = splitv[4][1:].strip()
129 # Handle symlinks
130 if(' -> ' in path):
131 target = path.split(' -> ')[1]
132 path = path.split(' -> ')[0]
133 adict[path] = splitv[0:3] + [target]
134 else:
135 adict[path] = splitv[0:3]
136 return adict
137
138
139def compare_file_lists(alines, blines):
140 adict = file_list_to_dict(alines)
141 bdict = file_list_to_dict(blines)
142 filechanges = []
143 for path, splitv in adict.iteritems():
144 newsplitv = bdict.pop(path, None)
145 if newsplitv:
146 # Check type
147 oldvalue = splitv[0][0]
148 newvalue = newsplitv[0][0]
149 if oldvalue != newvalue:
150 filechanges.append(FileChange(path, FileChange.changetype_type, oldvalue, newvalue))
151 # Check permissions
152 oldvalue = splitv[0][1:]
153 newvalue = newsplitv[0][1:]
154 if oldvalue != newvalue:
155 filechanges.append(FileChange(path, FileChange.changetype_perms, oldvalue, newvalue))
156 # Check owner/group
157 oldvalue = '%s/%s' % (splitv[1], splitv[2])
158 newvalue = '%s/%s' % (newsplitv[1], newsplitv[2])
159 if oldvalue != newvalue:
160 filechanges.append(FileChange(path, FileChange.changetype_ownergroup, oldvalue, newvalue))
161 # Check symlink target
162 if newsplitv[0][0] == 'l':
163 if splitv.count > 3:
164 oldvalue = splitv[3]
165 else:
166 oldvalue = None
167 newvalue = newsplitv[3]
168 if oldvalue != newvalue:
169 filechanges.append(FileChange(path, FileChange.changetype_link, oldvalue, newvalue))
170 else:
171 filechanges.append(FileChange(path, FileChange.changetype_remove))
172
173 # Whatever is left over has been added
174 for path in bdict:
175 filechanges.append(FileChange(path, FileChange.changetype_add))
176
177 return filechanges
178
179
180def compare_lists(alines, blines):
181 removed = list(set(alines) - set(blines))
182 added = list(set(blines) - set(alines))
183
184 filechanges = []
185 for pkg in removed:
186 filechanges.append(FileChange(pkg, FileChange.changetype_remove))
187 for pkg in added:
188 filechanges.append(FileChange(pkg, FileChange.changetype_add))
189
190 return filechanges
191
192
193def process_changes(repopath, revision1, revision2 = 'HEAD', report_all = False):
194 repo = git.Repo(repopath)
195 assert repo.bare == False
196 commit = repo.commit(revision1)
197 diff = commit.diff(revision2)
198
199 changes = []
200 for d in diff.iter_change_type('M'):
201 path = os.path.dirname(d.a_blob.path)
202 if path.startswith('packages/'):
203 adict = blob_to_dict(d.a_blob)
204 bdict = blob_to_dict(d.b_blob)
205
206 for key in adict:
207 if report_all or key in pkg_monitor_fields:
208 if adict[key] != bdict[key]:
209 if (not report_all) and key in pkg_numeric_fields:
210 aval = int(adict[key])
211 bval = int(bdict[key])
212 percentchg = ((bval - aval) / float(aval)) * 100
213 if percentchg < pkg_monitor_numeric_threshold:
214 continue
215 chg = ChangeRecord(path, key, adict[key], bdict[key])
216 changes.append(chg)
217 elif path.startswith('images/'):
218 filename = os.path.basename(d.a_blob.path)
219 if filename in img_monitor_files:
220 if filename == 'files-in-image.txt':
221 alines = d.a_blob.data_stream.read().splitlines()
222 blines = d.b_blob.data_stream.read().splitlines()
223 filechanges = compare_file_lists(alines,blines)
224 if filechanges:
225 chg = ChangeRecord(path, filename, None, None)
226 chg.filechanges = filechanges
227 changes.append(chg)
228 elif filename == 'installed-package-names.txt':
229 alines = d.a_blob.data_stream.read().splitlines()
230 blines = d.b_blob.data_stream.read().splitlines()
231 filechanges = compare_lists(alines,blines)
232 if filechanges:
233 chg = ChangeRecord(path, filename, None, None)
234 chg.filechanges = filechanges
235 changes.append(chg)
236 else:
237 chg = ChangeRecord(path, filename, d.a_blob.data_stream.read(), d.b_blob.data_stream.read())
238 changes.append(chg)
239
240 return changes