diff options
author | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2014-10-16 03:05:19 +0200 |
commit | c527fd1f14c27855a37f2e8ac5346ce8d940ced2 (patch) | |
tree | bb002c1fdf011c41dbd2f0927bed23ecb5f83c97 /scripts/lib/mic/utils/misc.py | |
download | poky-c527fd1f14c27855a37f2e8ac5346ce8d940ced2.tar.gz |
initial commit for Enea Linux 4.0-140929daisy-140929
Migrated from the internal git server on the daisy-enea-point-release branch
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'scripts/lib/mic/utils/misc.py')
-rw-r--r-- | scripts/lib/mic/utils/misc.py | 1065 |
1 files changed, 1065 insertions, 0 deletions
diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py new file mode 100644 index 0000000000..95241d7f15 --- /dev/null +++ b/scripts/lib/mic/utils/misc.py | |||
@@ -0,0 +1,1065 @@ | |||
1 | #!/usr/bin/python -tt | ||
2 | # | ||
3 | # Copyright (c) 2010, 2011 Intel Inc. | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify it | ||
6 | # under the terms of the GNU General Public License as published by the Free | ||
7 | # Software Foundation; version 2 of the License | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, but | ||
10 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY | ||
11 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||
12 | # for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU General Public License along | ||
15 | # with this program; if not, write to the Free Software Foundation, Inc., 59 | ||
16 | # Temple Place - Suite 330, Boston, MA 02111-1307, USA. | ||
17 | |||
18 | import os | ||
19 | import sys | ||
20 | import time | ||
21 | import tempfile | ||
22 | import re | ||
23 | import shutil | ||
24 | import glob | ||
25 | import hashlib | ||
26 | import subprocess | ||
27 | import platform | ||
28 | import traceback | ||
29 | |||
30 | |||
31 | try: | ||
32 | import sqlite3 as sqlite | ||
33 | except ImportError: | ||
34 | import sqlite | ||
35 | |||
36 | try: | ||
37 | from xml.etree import cElementTree | ||
38 | except ImportError: | ||
39 | import cElementTree | ||
40 | xmlparse = cElementTree.parse | ||
41 | |||
42 | from mic import msger | ||
43 | from mic.utils.errors import CreatorError, SquashfsError | ||
44 | from mic.utils.fs_related import find_binary_path, makedirs | ||
45 | from mic.utils.proxy import get_proxy_for | ||
46 | from mic.utils import runner | ||
47 | |||
48 | |||
49 | RPM_RE = re.compile("(.*)\.(.*) (.*)-(.*)") | ||
50 | RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s" | ||
51 | SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm") | ||
52 | |||
53 | |||
54 | def build_name(kscfg, release=None, prefix = None, suffix = None): | ||
55 | """Construct and return an image name string. | ||
56 | |||
57 | This is a utility function to help create sensible name and fslabel | ||
58 | strings. The name is constructed using the sans-prefix-and-extension | ||
59 | kickstart filename and the supplied prefix and suffix. | ||
60 | |||
61 | kscfg -- a path to a kickstart file | ||
62 | release -- a replacement to suffix for image release | ||
63 | prefix -- a prefix to prepend to the name; defaults to None, which causes | ||
64 | no prefix to be used | ||
65 | suffix -- a suffix to append to the name; defaults to None, which causes | ||
66 | a YYYYMMDDHHMM suffix to be used | ||
67 | |||
68 | Note, if maxlen is less then the len(suffix), you get to keep both pieces. | ||
69 | |||
70 | """ | ||
71 | name = os.path.basename(kscfg) | ||
72 | idx = name.rfind('.') | ||
73 | if idx >= 0: | ||
74 | name = name[:idx] | ||
75 | |||
76 | if release is not None: | ||
77 | suffix = "" | ||
78 | if prefix is None: | ||
79 | prefix = "" | ||
80 | if suffix is None: | ||
81 | suffix = time.strftime("%Y%m%d%H%M") | ||
82 | |||
83 | if name.startswith(prefix): | ||
84 | name = name[len(prefix):] | ||
85 | |||
86 | prefix = "%s-" % prefix if prefix else "" | ||
87 | suffix = "-%s" % suffix if suffix else "" | ||
88 | |||
89 | ret = prefix + name + suffix | ||
90 | return ret | ||
91 | |||
92 | def get_distro(): | ||
93 | """Detect linux distribution, support "meego" | ||
94 | """ | ||
95 | |||
96 | support_dists = ('SuSE', | ||
97 | 'debian', | ||
98 | 'fedora', | ||
99 | 'redhat', | ||
100 | 'centos', | ||
101 | 'meego', | ||
102 | 'moblin', | ||
103 | 'tizen') | ||
104 | try: | ||
105 | (dist, ver, id) = platform.linux_distribution( \ | ||
106 | supported_dists = support_dists) | ||
107 | except: | ||
108 | (dist, ver, id) = platform.dist( \ | ||
109 | supported_dists = support_dists) | ||
110 | |||
111 | return (dist, ver, id) | ||
112 | |||
113 | def get_distro_str(): | ||
114 | """Get composited string for current linux distribution | ||
115 | """ | ||
116 | (dist, ver, id) = get_distro() | ||
117 | |||
118 | if not dist: | ||
119 | return 'Unknown Linux Distro' | ||
120 | else: | ||
121 | distro_str = ' '.join(map(str.strip, (dist, ver, id))) | ||
122 | return distro_str.strip() | ||
123 | |||
124 | _LOOP_RULE_PTH = None | ||
125 | |||
126 | def hide_loopdev_presentation(): | ||
127 | udev_rules = "80-prevent-loop-present.rules" | ||
128 | udev_rules_dir = [ | ||
129 | '/usr/lib/udev/rules.d/', | ||
130 | '/lib/udev/rules.d/', | ||
131 | '/etc/udev/rules.d/' | ||
132 | ] | ||
133 | |||
134 | global _LOOP_RULE_PTH | ||
135 | |||
136 | for rdir in udev_rules_dir: | ||
137 | if os.path.exists(rdir): | ||
138 | _LOOP_RULE_PTH = os.path.join(rdir, udev_rules) | ||
139 | |||
140 | if not _LOOP_RULE_PTH: | ||
141 | return | ||
142 | |||
143 | try: | ||
144 | with open(_LOOP_RULE_PTH, 'w') as wf: | ||
145 | wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"') | ||
146 | |||
147 | runner.quiet('udevadm trigger') | ||
148 | except: | ||
149 | pass | ||
150 | |||
151 | def unhide_loopdev_presentation(): | ||
152 | global _LOOP_RULE_PTH | ||
153 | |||
154 | if not _LOOP_RULE_PTH: | ||
155 | return | ||
156 | |||
157 | try: | ||
158 | os.unlink(_LOOP_RULE_PTH) | ||
159 | runner.quiet('udevadm trigger') | ||
160 | except: | ||
161 | pass | ||
162 | |||
163 | def extract_rpm(rpmfile, targetdir): | ||
164 | rpm2cpio = find_binary_path("rpm2cpio") | ||
165 | cpio = find_binary_path("cpio") | ||
166 | |||
167 | olddir = os.getcwd() | ||
168 | os.chdir(targetdir) | ||
169 | |||
170 | msger.verbose("Extract rpm file with cpio: %s" % rpmfile) | ||
171 | p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE) | ||
172 | p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout, | ||
173 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) | ||
174 | (sout, serr) = p2.communicate() | ||
175 | msger.verbose(sout or serr) | ||
176 | |||
177 | os.chdir(olddir) | ||
178 | |||
179 | def compressing(fpath, method): | ||
180 | comp_map = { | ||
181 | "gz": "gzip", | ||
182 | "bz2": "bzip2" | ||
183 | } | ||
184 | if method not in comp_map: | ||
185 | raise CreatorError("Unsupport compress format: %s, valid values: %s" | ||
186 | % (method, ','.join(comp_map.keys()))) | ||
187 | cmd = find_binary_path(comp_map[method]) | ||
188 | rc = runner.show([cmd, "-f", fpath]) | ||
189 | if rc: | ||
190 | raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath)) | ||
191 | |||
192 | def taring(dstfile, target): | ||
193 | import tarfile | ||
194 | basen, ext = os.path.splitext(dstfile) | ||
195 | comp = {".tar": None, | ||
196 | ".gz": "gz", # for .tar.gz | ||
197 | ".bz2": "bz2", # for .tar.bz2 | ||
198 | ".tgz": "gz", | ||
199 | ".tbz": "bz2"}[ext] | ||
200 | |||
201 | # specify tarball file path | ||
202 | if not comp: | ||
203 | tarpath = dstfile | ||
204 | elif basen.endswith(".tar"): | ||
205 | tarpath = basen | ||
206 | else: | ||
207 | tarpath = basen + ".tar" | ||
208 | wf = tarfile.open(tarpath, 'w') | ||
209 | |||
210 | if os.path.isdir(target): | ||
211 | for item in os.listdir(target): | ||
212 | wf.add(os.path.join(target, item), item) | ||
213 | else: | ||
214 | wf.add(target, os.path.basename(target)) | ||
215 | wf.close() | ||
216 | |||
217 | if comp: | ||
218 | compressing(tarpath, comp) | ||
219 | # when dstfile ext is ".tgz" and ".tbz", should rename | ||
220 | if not basen.endswith(".tar"): | ||
221 | shutil.move("%s.%s" % (tarpath, comp), dstfile) | ||
222 | |||
223 | def ziping(dstfile, target): | ||
224 | import zipfile | ||
225 | wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED) | ||
226 | if os.path.isdir(target): | ||
227 | for item in os.listdir(target): | ||
228 | fpath = os.path.join(target, item) | ||
229 | if not os.path.isfile(fpath): | ||
230 | continue | ||
231 | wf.write(fpath, item, zipfile.ZIP_DEFLATED) | ||
232 | else: | ||
233 | wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED) | ||
234 | wf.close() | ||
235 | |||
236 | pack_formats = { | ||
237 | ".tar": taring, | ||
238 | ".tar.gz": taring, | ||
239 | ".tar.bz2": taring, | ||
240 | ".tgz": taring, | ||
241 | ".tbz": taring, | ||
242 | ".zip": ziping, | ||
243 | } | ||
244 | |||
245 | def packing(dstfile, target): | ||
246 | (base, ext) = os.path.splitext(dstfile) | ||
247 | if ext in (".gz", ".bz2") and base.endswith(".tar"): | ||
248 | ext = ".tar" + ext | ||
249 | if ext not in pack_formats: | ||
250 | raise CreatorError("Unsupport pack format: %s, valid values: %s" | ||
251 | % (ext, ','.join(pack_formats.keys()))) | ||
252 | func = pack_formats[ext] | ||
253 | # func should be callable | ||
254 | func(dstfile, target) | ||
255 | |||
256 | def human_size(size): | ||
257 | """Return human readable string for Bytes size | ||
258 | """ | ||
259 | |||
260 | if size <= 0: | ||
261 | return "0M" | ||
262 | import math | ||
263 | measure = ['B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'] | ||
264 | expo = int(math.log(size, 1024)) | ||
265 | mant = float(size/math.pow(1024, expo)) | ||
266 | return "{0:.1f}{1:s}".format(mant, measure[expo]) | ||
267 | |||
268 | def get_block_size(file_obj): | ||
269 | """ Returns block size for file object 'file_obj'. Errors are indicated by | ||
270 | the 'IOError' exception. """ | ||
271 | |||
272 | from fcntl import ioctl | ||
273 | import struct | ||
274 | |||
275 | # Get the block size of the host file-system for the image file by calling | ||
276 | # the FIGETBSZ ioctl (number 2). | ||
277 | binary_data = ioctl(file_obj, 2, struct.pack('I', 0)) | ||
278 | return struct.unpack('I', binary_data)[0] | ||
279 | |||
280 | def check_space_pre_cp(src, dst): | ||
281 | """Check whether disk space is enough before 'cp' like | ||
282 | operations, else exception will be raised. | ||
283 | """ | ||
284 | |||
285 | srcsize = get_file_size(src) * 1024 * 1024 | ||
286 | freesize = get_filesystem_avail(dst) | ||
287 | if srcsize > freesize: | ||
288 | raise CreatorError("space on %s(%s) is not enough for about %s files" | ||
289 | % (dst, human_size(freesize), human_size(srcsize))) | ||
290 | |||
291 | def calc_hashes(file_path, hash_names, start = 0, end = None): | ||
292 | """ Calculate hashes for a file. The 'file_path' argument is the file | ||
293 | to calculate hash functions for, 'start' and 'end' are the starting and | ||
294 | ending file offset to calculate the has functions for. The 'hash_names' | ||
295 | argument is a list of hash names to calculate. Returns the the list | ||
296 | of calculated hash values in the hexadecimal form in the same order | ||
297 | as 'hash_names'. | ||
298 | """ | ||
299 | if end == None: | ||
300 | end = os.path.getsize(file_path) | ||
301 | |||
302 | chunk_size = 65536 | ||
303 | to_read = end - start | ||
304 | read = 0 | ||
305 | |||
306 | hashes = [] | ||
307 | for hash_name in hash_names: | ||
308 | hashes.append(hashlib.new(hash_name)) | ||
309 | |||
310 | with open(file_path, "rb") as f: | ||
311 | f.seek(start) | ||
312 | |||
313 | while read < to_read: | ||
314 | if read + chunk_size > to_read: | ||
315 | chunk_size = to_read - read | ||
316 | chunk = f.read(chunk_size) | ||
317 | for hash_obj in hashes: | ||
318 | hash_obj.update(chunk) | ||
319 | read += chunk_size | ||
320 | |||
321 | result = [] | ||
322 | for hash_obj in hashes: | ||
323 | result.append(hash_obj.hexdigest()) | ||
324 | |||
325 | return result | ||
326 | |||
327 | def get_md5sum(fpath): | ||
328 | return calc_hashes(fpath, ('md5', ))[0] | ||
329 | |||
330 | |||
331 | def normalize_ksfile(ksconf, release, arch): | ||
332 | ''' | ||
333 | Return the name of a normalized ks file in which macro variables | ||
334 | @BUILD_ID@ and @ARCH@ are replace with real values. | ||
335 | |||
336 | The original ks file is returned if no special macro is used, otherwise | ||
337 | a temp file is created and returned, which will be deleted when program | ||
338 | exits normally. | ||
339 | ''' | ||
340 | |||
341 | if not release: | ||
342 | release = "latest" | ||
343 | if not arch or re.match(r'i.86', arch): | ||
344 | arch = "ia32" | ||
345 | |||
346 | with open(ksconf) as f: | ||
347 | ksc = f.read() | ||
348 | |||
349 | if "@ARCH@" not in ksc and "@BUILD_ID@" not in ksc: | ||
350 | return ksconf | ||
351 | |||
352 | msger.info("Substitute macro variable @BUILD_ID@/@ARCH@ in ks: %s" % ksconf) | ||
353 | ksc = ksc.replace("@ARCH@", arch) | ||
354 | ksc = ksc.replace("@BUILD_ID@", release) | ||
355 | |||
356 | fd, ksconf = tempfile.mkstemp(prefix=os.path.basename(ksconf)) | ||
357 | os.write(fd, ksc) | ||
358 | os.close(fd) | ||
359 | |||
360 | msger.debug('normalized ks file:%s' % ksconf) | ||
361 | |||
362 | def remove_temp_ks(): | ||
363 | try: | ||
364 | os.unlink(ksconf) | ||
365 | except OSError, err: | ||
366 | msger.warning('Failed to remove temp ks file:%s:%s' % (ksconf, err)) | ||
367 | |||
368 | import atexit | ||
369 | atexit.register(remove_temp_ks) | ||
370 | |||
371 | return ksconf | ||
372 | |||
373 | |||
374 | def _check_mic_chroot(rootdir): | ||
375 | def _path(path): | ||
376 | return rootdir.rstrip('/') + path | ||
377 | |||
378 | release_files = map(_path, [ "/etc/moblin-release", | ||
379 | "/etc/meego-release", | ||
380 | "/etc/tizen-release"]) | ||
381 | |||
382 | if not any(map(os.path.exists, release_files)): | ||
383 | msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir) | ||
384 | |||
385 | if not glob.glob(rootdir + "/boot/vmlinuz-*"): | ||
386 | msger.warning("Failed to find kernel module under %s" % rootdir) | ||
387 | |||
388 | return | ||
389 | |||
390 | def selinux_check(arch, fstypes): | ||
391 | try: | ||
392 | getenforce = find_binary_path('getenforce') | ||
393 | except CreatorError: | ||
394 | return | ||
395 | |||
396 | selinux_status = runner.outs([getenforce]) | ||
397 | if arch and arch.startswith("arm") and selinux_status == "Enforcing": | ||
398 | raise CreatorError("Can't create arm image if selinux is enabled, " | ||
399 | "please run 'setenforce 0' to disable selinux") | ||
400 | |||
401 | use_btrfs = filter(lambda typ: typ == 'btrfs', fstypes) | ||
402 | if use_btrfs and selinux_status == "Enforcing": | ||
403 | raise CreatorError("Can't create btrfs image if selinux is enabled," | ||
404 | " please run 'setenforce 0' to disable selinux") | ||
405 | |||
406 | def get_image_type(path): | ||
407 | def _get_extension_name(path): | ||
408 | match = re.search("(?<=\.)\w+$", path) | ||
409 | if match: | ||
410 | return match.group(0) | ||
411 | else: | ||
412 | return None | ||
413 | |||
414 | if os.path.isdir(path): | ||
415 | _check_mic_chroot(path) | ||
416 | return "fs" | ||
417 | |||
418 | maptab = { | ||
419 | "tar": "loop", | ||
420 | "raw":"raw", | ||
421 | "vmdk":"vmdk", | ||
422 | "vdi":"vdi", | ||
423 | "iso":"livecd", | ||
424 | "usbimg":"liveusb", | ||
425 | } | ||
426 | |||
427 | extension = _get_extension_name(path) | ||
428 | if extension in maptab: | ||
429 | return maptab[extension] | ||
430 | |||
431 | fd = open(path, "rb") | ||
432 | file_header = fd.read(1024) | ||
433 | fd.close() | ||
434 | vdi_flag = "<<< Sun VirtualBox Disk Image >>>" | ||
435 | if file_header[0:len(vdi_flag)] == vdi_flag: | ||
436 | return maptab["vdi"] | ||
437 | |||
438 | output = runner.outs(['file', path]) | ||
439 | isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*") | ||
440 | usbimgptn = re.compile(r".*x86 boot sector.*active.*") | ||
441 | rawptn = re.compile(r".*x86 boot sector.*") | ||
442 | vmdkptn = re.compile(r".*VMware. disk image.*") | ||
443 | ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*") | ||
444 | ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*") | ||
445 | btrfsimgptn = re.compile(r".*BTRFS.*") | ||
446 | if isoptn.match(output): | ||
447 | return maptab["iso"] | ||
448 | elif usbimgptn.match(output): | ||
449 | return maptab["usbimg"] | ||
450 | elif rawptn.match(output): | ||
451 | return maptab["raw"] | ||
452 | elif vmdkptn.match(output): | ||
453 | return maptab["vmdk"] | ||
454 | elif ext3fsimgptn.match(output): | ||
455 | return "ext3fsimg" | ||
456 | elif ext4fsimgptn.match(output): | ||
457 | return "ext4fsimg" | ||
458 | elif btrfsimgptn.match(output): | ||
459 | return "btrfsimg" | ||
460 | else: | ||
461 | raise CreatorError("Cannot detect the type of image: %s" % path) | ||
462 | |||
463 | |||
464 | def get_file_size(filename): | ||
465 | """ Return size in MB unit """ | ||
466 | cmd = ['du', "-s", "-b", "-B", "1M", filename] | ||
467 | rc, duOutput = runner.runtool(cmd) | ||
468 | if rc != 0: | ||
469 | raise CreatorError("Failed to run: %s" % ' '.join(cmd)) | ||
470 | size1 = int(duOutput.split()[0]) | ||
471 | |||
472 | cmd = ['du', "-s", "-B", "1M", filename] | ||
473 | rc, duOutput = runner.runtool(cmd) | ||
474 | if rc != 0: | ||
475 | raise CreatorError("Failed to run: %s" % ' '.join(cmd)) | ||
476 | |||
477 | size2 = int(duOutput.split()[0]) | ||
478 | return max(size1, size2) | ||
479 | |||
480 | |||
481 | def get_filesystem_avail(fs): | ||
482 | vfstat = os.statvfs(fs) | ||
483 | return vfstat.f_bavail * vfstat.f_bsize | ||
484 | |||
485 | def convert_image(srcimg, srcfmt, dstimg, dstfmt): | ||
486 | #convert disk format | ||
487 | if dstfmt != "raw": | ||
488 | raise CreatorError("Invalid destination image format: %s" % dstfmt) | ||
489 | msger.debug("converting %s image to %s" % (srcimg, dstimg)) | ||
490 | if srcfmt == "vmdk": | ||
491 | path = find_binary_path("qemu-img") | ||
492 | argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt, dstimg] | ||
493 | elif srcfmt == "vdi": | ||
494 | path = find_binary_path("VBoxManage") | ||
495 | argv = [path, "internalcommands", "converttoraw", srcimg, dstimg] | ||
496 | else: | ||
497 | raise CreatorError("Invalid soure image format: %s" % srcfmt) | ||
498 | |||
499 | rc = runner.show(argv) | ||
500 | if rc == 0: | ||
501 | msger.debug("convert successful") | ||
502 | if rc != 0: | ||
503 | raise CreatorError("Unable to convert disk to %s" % dstfmt) | ||
504 | |||
505 | def uncompress_squashfs(squashfsimg, outdir): | ||
506 | """Uncompress file system from squshfs image""" | ||
507 | unsquashfs = find_binary_path("unsquashfs") | ||
508 | args = [ unsquashfs, "-d", outdir, squashfsimg ] | ||
509 | rc = runner.show(args) | ||
510 | if (rc != 0): | ||
511 | raise SquashfsError("Failed to uncompress %s." % squashfsimg) | ||
512 | |||
513 | def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"): | ||
514 | """ FIXME: use the dir in wic.conf instead """ | ||
515 | |||
516 | makedirs(dir) | ||
517 | return tempfile.mkdtemp(dir = dir, prefix = prefix) | ||
518 | |||
519 | def get_repostrs_from_ks(ks): | ||
520 | def _get_temp_reponame(baseurl): | ||
521 | md5obj = hashlib.md5(baseurl) | ||
522 | tmpreponame = "%s" % md5obj.hexdigest() | ||
523 | return tmpreponame | ||
524 | |||
525 | kickstart_repos = [] | ||
526 | |||
527 | for repodata in ks.handler.repo.repoList: | ||
528 | repo = {} | ||
529 | for attr in ('name', | ||
530 | 'baseurl', | ||
531 | 'mirrorlist', | ||
532 | 'includepkgs', # val is list | ||
533 | 'excludepkgs', # val is list | ||
534 | 'cost', # int | ||
535 | 'priority',# int | ||
536 | 'save', | ||
537 | 'proxy', | ||
538 | 'proxyuser', | ||
539 | 'proxypasswd', | ||
540 | 'proxypasswd', | ||
541 | 'debuginfo', | ||
542 | 'source', | ||
543 | 'gpgkey', | ||
544 | 'ssl_verify'): | ||
545 | if hasattr(repodata, attr) and getattr(repodata, attr): | ||
546 | repo[attr] = getattr(repodata, attr) | ||
547 | |||
548 | if 'name' not in repo: | ||
549 | repo['name'] = _get_temp_reponame(repodata.baseurl) | ||
550 | |||
551 | kickstart_repos.append(repo) | ||
552 | |||
553 | return kickstart_repos | ||
554 | |||
555 | def _get_uncompressed_data_from_url(url, filename, proxies): | ||
556 | filename = myurlgrab(url, filename, proxies) | ||
557 | suffix = None | ||
558 | if filename.endswith(".gz"): | ||
559 | suffix = ".gz" | ||
560 | runner.quiet(['gunzip', "-f", filename]) | ||
561 | elif filename.endswith(".bz2"): | ||
562 | suffix = ".bz2" | ||
563 | runner.quiet(['bunzip2', "-f", filename]) | ||
564 | if suffix: | ||
565 | filename = filename.replace(suffix, "") | ||
566 | return filename | ||
567 | |||
568 | def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename, | ||
569 | sumtype=None, checksum=None): | ||
570 | url = os.path.join(baseurl, filename) | ||
571 | filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename))) | ||
572 | if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"): | ||
573 | filename = os.path.splitext(filename_tmp)[0] | ||
574 | else: | ||
575 | filename = filename_tmp | ||
576 | if sumtype and checksum and os.path.exists(filename): | ||
577 | try: | ||
578 | sumcmd = find_binary_path("%ssum" % sumtype) | ||
579 | except: | ||
580 | file_checksum = None | ||
581 | else: | ||
582 | file_checksum = runner.outs([sumcmd, filename]).split()[0] | ||
583 | |||
584 | if file_checksum and file_checksum == checksum: | ||
585 | return filename | ||
586 | |||
587 | return _get_uncompressed_data_from_url(url,filename_tmp,proxies) | ||
588 | |||
589 | def get_metadata_from_repos(repos, cachedir): | ||
590 | my_repo_metadata = [] | ||
591 | for repo in repos: | ||
592 | reponame = repo['name'] | ||
593 | baseurl = repo['baseurl'] | ||
594 | |||
595 | |||
596 | if 'proxy' in repo: | ||
597 | proxy = repo['proxy'] | ||
598 | else: | ||
599 | proxy = get_proxy_for(baseurl) | ||
600 | |||
601 | proxies = None | ||
602 | if proxy: | ||
603 | proxies = {str(baseurl.split(":")[0]):str(proxy)} | ||
604 | |||
605 | makedirs(os.path.join(cachedir, reponame)) | ||
606 | url = os.path.join(baseurl, "repodata/repomd.xml") | ||
607 | filename = os.path.join(cachedir, reponame, 'repomd.xml') | ||
608 | repomd = myurlgrab(url, filename, proxies) | ||
609 | try: | ||
610 | root = xmlparse(repomd) | ||
611 | except SyntaxError: | ||
612 | raise CreatorError("repomd.xml syntax error.") | ||
613 | |||
614 | ns = root.getroot().tag | ||
615 | ns = ns[0:ns.rindex("}")+1] | ||
616 | |||
617 | filepaths = {} | ||
618 | checksums = {} | ||
619 | sumtypes = {} | ||
620 | |||
621 | for elm in root.getiterator("%sdata" % ns): | ||
622 | if elm.attrib["type"] == "patterns": | ||
623 | filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href'] | ||
624 | checksums['patterns'] = elm.find("%sopen-checksum" % ns).text | ||
625 | sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type'] | ||
626 | break | ||
627 | |||
628 | for elm in root.getiterator("%sdata" % ns): | ||
629 | if elm.attrib["type"] in ("group_gz", "group"): | ||
630 | filepaths['comps'] = elm.find("%slocation" % ns).attrib['href'] | ||
631 | checksums['comps'] = elm.find("%sopen-checksum" % ns).text | ||
632 | sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type'] | ||
633 | break | ||
634 | |||
635 | primary_type = None | ||
636 | for elm in root.getiterator("%sdata" % ns): | ||
637 | if elm.attrib["type"] in ("primary_db", "primary"): | ||
638 | primary_type = elm.attrib["type"] | ||
639 | filepaths['primary'] = elm.find("%slocation" % ns).attrib['href'] | ||
640 | checksums['primary'] = elm.find("%sopen-checksum" % ns).text | ||
641 | sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type'] | ||
642 | break | ||
643 | |||
644 | if not primary_type: | ||
645 | continue | ||
646 | |||
647 | for item in ("primary", "patterns", "comps"): | ||
648 | if item not in filepaths: | ||
649 | filepaths[item] = None | ||
650 | continue | ||
651 | if not filepaths[item]: | ||
652 | continue | ||
653 | filepaths[item] = _get_metadata_from_repo(baseurl, | ||
654 | proxies, | ||
655 | cachedir, | ||
656 | reponame, | ||
657 | filepaths[item], | ||
658 | sumtypes[item], | ||
659 | checksums[item]) | ||
660 | |||
661 | """ Get repo key """ | ||
662 | try: | ||
663 | repokey = _get_metadata_from_repo(baseurl, | ||
664 | proxies, | ||
665 | cachedir, | ||
666 | reponame, | ||
667 | "repodata/repomd.xml.key") | ||
668 | except CreatorError: | ||
669 | repokey = None | ||
670 | msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key")) | ||
671 | |||
672 | my_repo_metadata.append({"name":reponame, | ||
673 | "baseurl":baseurl, | ||
674 | "repomd":repomd, | ||
675 | "primary":filepaths['primary'], | ||
676 | "cachedir":cachedir, | ||
677 | "proxies":proxies, | ||
678 | "patterns":filepaths['patterns'], | ||
679 | "comps":filepaths['comps'], | ||
680 | "repokey":repokey}) | ||
681 | |||
682 | return my_repo_metadata | ||
683 | |||
684 | def get_rpmver_in_repo(repometadata): | ||
685 | for repo in repometadata: | ||
686 | if repo["primary"].endswith(".xml"): | ||
687 | root = xmlparse(repo["primary"]) | ||
688 | ns = root.getroot().tag | ||
689 | ns = ns[0:ns.rindex("}")+1] | ||
690 | |||
691 | versionlist = [] | ||
692 | for elm in root.getiterator("%spackage" % ns): | ||
693 | if elm.find("%sname" % ns).text == 'rpm': | ||
694 | for node in elm.getchildren(): | ||
695 | if node.tag == "%sversion" % ns: | ||
696 | versionlist.append(node.attrib['ver']) | ||
697 | |||
698 | if versionlist: | ||
699 | return reversed( | ||
700 | sorted( | ||
701 | versionlist, | ||
702 | key = lambda ver: map(int, ver.split('.')))).next() | ||
703 | |||
704 | elif repo["primary"].endswith(".sqlite"): | ||
705 | con = sqlite.connect(repo["primary"]) | ||
706 | for row in con.execute("select version from packages where " | ||
707 | "name=\"rpm\" ORDER by version DESC"): | ||
708 | con.close() | ||
709 | return row[0] | ||
710 | |||
711 | return None | ||
712 | |||
713 | def get_arch(repometadata): | ||
714 | archlist = [] | ||
715 | for repo in repometadata: | ||
716 | if repo["primary"].endswith(".xml"): | ||
717 | root = xmlparse(repo["primary"]) | ||
718 | ns = root.getroot().tag | ||
719 | ns = ns[0:ns.rindex("}")+1] | ||
720 | for elm in root.getiterator("%spackage" % ns): | ||
721 | if elm.find("%sarch" % ns).text not in ("noarch", "src"): | ||
722 | arch = elm.find("%sarch" % ns).text | ||
723 | if arch not in archlist: | ||
724 | archlist.append(arch) | ||
725 | elif repo["primary"].endswith(".sqlite"): | ||
726 | con = sqlite.connect(repo["primary"]) | ||
727 | for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"): | ||
728 | if row[0] not in archlist: | ||
729 | archlist.append(row[0]) | ||
730 | |||
731 | con.close() | ||
732 | |||
733 | uniq_arch = [] | ||
734 | for i in range(len(archlist)): | ||
735 | if archlist[i] not in rpmmisc.archPolicies.keys(): | ||
736 | continue | ||
737 | need_append = True | ||
738 | j = 0 | ||
739 | while j < len(uniq_arch): | ||
740 | if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'): | ||
741 | need_append = False | ||
742 | break | ||
743 | if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'): | ||
744 | if need_append: | ||
745 | uniq_arch[j] = archlist[i] | ||
746 | need_append = False | ||
747 | else: | ||
748 | uniq_arch.remove(uniq_arch[j]) | ||
749 | continue | ||
750 | j += 1 | ||
751 | if need_append: | ||
752 | uniq_arch.append(archlist[i]) | ||
753 | |||
754 | return uniq_arch, archlist | ||
755 | |||
756 | def get_package(pkg, repometadata, arch = None): | ||
757 | ver = "" | ||
758 | target_repo = None | ||
759 | if not arch: | ||
760 | arches = [] | ||
761 | elif arch not in rpmmisc.archPolicies: | ||
762 | arches = [arch] | ||
763 | else: | ||
764 | arches = rpmmisc.archPolicies[arch].split(':') | ||
765 | arches.append('noarch') | ||
766 | |||
767 | for repo in repometadata: | ||
768 | if repo["primary"].endswith(".xml"): | ||
769 | root = xmlparse(repo["primary"]) | ||
770 | ns = root.getroot().tag | ||
771 | ns = ns[0:ns.rindex("}")+1] | ||
772 | for elm in root.getiterator("%spackage" % ns): | ||
773 | if elm.find("%sname" % ns).text == pkg: | ||
774 | if elm.find("%sarch" % ns).text in arches: | ||
775 | version = elm.find("%sversion" % ns) | ||
776 | tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel']) | ||
777 | if tmpver > ver: | ||
778 | ver = tmpver | ||
779 | location = elm.find("%slocation" % ns) | ||
780 | pkgpath = "%s" % location.attrib['href'] | ||
781 | target_repo = repo | ||
782 | break | ||
783 | if repo["primary"].endswith(".sqlite"): | ||
784 | con = sqlite.connect(repo["primary"]) | ||
785 | if arch: | ||
786 | sql = 'select version, release, location_href from packages ' \ | ||
787 | 'where name = "%s" and arch IN ("%s")' % \ | ||
788 | (pkg, '","'.join(arches)) | ||
789 | for row in con.execute(sql): | ||
790 | tmpver = "%s-%s" % (row[0], row[1]) | ||
791 | if tmpver > ver: | ||
792 | ver = tmpver | ||
793 | pkgpath = "%s" % row[2] | ||
794 | target_repo = repo | ||
795 | break | ||
796 | else: | ||
797 | sql = 'select version, release, location_href from packages ' \ | ||
798 | 'where name = "%s"' % pkg | ||
799 | for row in con.execute(sql): | ||
800 | tmpver = "%s-%s" % (row[0], row[1]) | ||
801 | if tmpver > ver: | ||
802 | ver = tmpver | ||
803 | pkgpath = "%s" % row[2] | ||
804 | target_repo = repo | ||
805 | break | ||
806 | con.close() | ||
807 | if target_repo: | ||
808 | makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"])) | ||
809 | url = os.path.join(target_repo["baseurl"], pkgpath) | ||
810 | filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath))) | ||
811 | if os.path.exists(filename): | ||
812 | ret = rpmmisc.checkRpmIntegrity('rpm', filename) | ||
813 | if ret == 0: | ||
814 | return filename | ||
815 | |||
816 | msger.warning("package %s is damaged: %s" % | ||
817 | (os.path.basename(filename), filename)) | ||
818 | os.unlink(filename) | ||
819 | |||
820 | pkg = myurlgrab(str(url), filename, target_repo["proxies"]) | ||
821 | return pkg | ||
822 | else: | ||
823 | return None | ||
824 | |||
825 | def get_source_name(pkg, repometadata): | ||
826 | |||
827 | def get_bin_name(pkg): | ||
828 | m = RPM_RE.match(pkg) | ||
829 | if m: | ||
830 | return m.group(1) | ||
831 | return None | ||
832 | |||
833 | def get_src_name(srpm): | ||
834 | m = SRPM_RE.match(srpm) | ||
835 | if m: | ||
836 | return m.group(1) | ||
837 | return None | ||
838 | |||
839 | ver = "" | ||
840 | target_repo = None | ||
841 | |||
842 | pkg_name = get_bin_name(pkg) | ||
843 | if not pkg_name: | ||
844 | return None | ||
845 | |||
846 | for repo in repometadata: | ||
847 | if repo["primary"].endswith(".xml"): | ||
848 | root = xmlparse(repo["primary"]) | ||
849 | ns = root.getroot().tag | ||
850 | ns = ns[0:ns.rindex("}")+1] | ||
851 | for elm in root.getiterator("%spackage" % ns): | ||
852 | if elm.find("%sname" % ns).text == pkg_name: | ||
853 | if elm.find("%sarch" % ns).text != "src": | ||
854 | version = elm.find("%sversion" % ns) | ||
855 | tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel']) | ||
856 | if tmpver > ver: | ||
857 | ver = tmpver | ||
858 | fmt = elm.find("%sformat" % ns) | ||
859 | if fmt: | ||
860 | fns = fmt.getchildren()[0].tag | ||
861 | fns = fns[0:fns.rindex("}")+1] | ||
862 | pkgpath = fmt.find("%ssourcerpm" % fns).text | ||
863 | target_repo = repo | ||
864 | break | ||
865 | |||
866 | if repo["primary"].endswith(".sqlite"): | ||
867 | con = sqlite.connect(repo["primary"]) | ||
868 | for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name): | ||
869 | tmpver = "%s-%s" % (row[0], row[1]) | ||
870 | if tmpver > ver: | ||
871 | pkgpath = "%s" % row[2] | ||
872 | target_repo = repo | ||
873 | break | ||
874 | con.close() | ||
875 | if target_repo: | ||
876 | return get_src_name(pkgpath) | ||
877 | else: | ||
878 | return None | ||
879 | |||
880 | def get_pkglist_in_patterns(group, patterns): | ||
881 | found = False | ||
882 | pkglist = [] | ||
883 | try: | ||
884 | root = xmlparse(patterns) | ||
885 | except SyntaxError: | ||
886 | raise SyntaxError("%s syntax error." % patterns) | ||
887 | |||
888 | for elm in list(root.getroot()): | ||
889 | ns = elm.tag | ||
890 | ns = ns[0:ns.rindex("}")+1] | ||
891 | name = elm.find("%sname" % ns) | ||
892 | summary = elm.find("%ssummary" % ns) | ||
893 | if name.text == group or summary.text == group: | ||
894 | found = True | ||
895 | break | ||
896 | |||
897 | if not found: | ||
898 | return pkglist | ||
899 | |||
900 | found = False | ||
901 | for requires in list(elm): | ||
902 | if requires.tag.endswith("requires"): | ||
903 | found = True | ||
904 | break | ||
905 | |||
906 | if not found: | ||
907 | return pkglist | ||
908 | |||
909 | for pkg in list(requires): | ||
910 | pkgname = pkg.attrib["name"] | ||
911 | if pkgname not in pkglist: | ||
912 | pkglist.append(pkgname) | ||
913 | |||
914 | return pkglist | ||
915 | |||
916 | def get_pkglist_in_comps(group, comps): | ||
917 | found = False | ||
918 | pkglist = [] | ||
919 | try: | ||
920 | root = xmlparse(comps) | ||
921 | except SyntaxError: | ||
922 | raise SyntaxError("%s syntax error." % comps) | ||
923 | |||
924 | for elm in root.getiterator("group"): | ||
925 | id = elm.find("id") | ||
926 | name = elm.find("name") | ||
927 | if id.text == group or name.text == group: | ||
928 | packagelist = elm.find("packagelist") | ||
929 | found = True | ||
930 | break | ||
931 | |||
932 | if not found: | ||
933 | return pkglist | ||
934 | |||
935 | for require in elm.getiterator("packagereq"): | ||
936 | if require.tag.endswith("packagereq"): | ||
937 | pkgname = require.text | ||
938 | if pkgname not in pkglist: | ||
939 | pkglist.append(pkgname) | ||
940 | |||
941 | return pkglist | ||
942 | |||
943 | def is_statically_linked(binary): | ||
944 | return ", statically linked, " in runner.outs(['file', binary]) | ||
945 | |||
946 | def setup_qemu_emulator(rootdir, arch): | ||
947 | # mount binfmt_misc if it doesn't exist | ||
948 | if not os.path.exists("/proc/sys/fs/binfmt_misc"): | ||
949 | modprobecmd = find_binary_path("modprobe") | ||
950 | runner.show([modprobecmd, "binfmt_misc"]) | ||
951 | if not os.path.exists("/proc/sys/fs/binfmt_misc/register"): | ||
952 | mountcmd = find_binary_path("mount") | ||
953 | runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"]) | ||
954 | |||
955 | # qemu_emulator is a special case, we can't use find_binary_path | ||
956 | # qemu emulator should be a statically-linked executable file | ||
957 | qemu_emulator = "/usr/bin/qemu-arm" | ||
958 | if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator): | ||
959 | qemu_emulator = "/usr/bin/qemu-arm-static" | ||
960 | if not os.path.exists(qemu_emulator): | ||
961 | raise CreatorError("Please install a statically-linked qemu-arm") | ||
962 | |||
963 | # qemu emulator version check | ||
964 | armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')] | ||
965 | if arch in armv7_list: # need qemu (>=0.13.0) | ||
966 | qemuout = runner.outs([qemu_emulator, "-h"]) | ||
967 | m = re.search("version\s*([.\d]+)", qemuout) | ||
968 | if m: | ||
969 | qemu_version = m.group(1) | ||
970 | if qemu_version < "0.13": | ||
971 | raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch)) | ||
972 | else: | ||
973 | msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator) | ||
974 | |||
975 | if not os.path.exists(rootdir + "/usr/bin"): | ||
976 | makedirs(rootdir + "/usr/bin") | ||
977 | shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static") | ||
978 | qemu_emulator = "/usr/bin/qemu-arm-static" | ||
979 | |||
980 | # disable selinux, selinux will block qemu emulator to run | ||
981 | if os.path.exists("/usr/sbin/setenforce"): | ||
982 | msger.info('Try to disable selinux') | ||
983 | runner.show(["/usr/sbin/setenforce", "0"]) | ||
984 | |||
985 | # unregister it if it has been registered and is a dynamically-linked executable | ||
986 | node = "/proc/sys/fs/binfmt_misc/arm" | ||
987 | if os.path.exists(node): | ||
988 | qemu_unregister_string = "-1\n" | ||
989 | fd = open("/proc/sys/fs/binfmt_misc/arm", "w") | ||
990 | fd.write(qemu_unregister_string) | ||
991 | fd.close() | ||
992 | |||
993 | # register qemu emulator for interpreting other arch executable file | ||
994 | if not os.path.exists(node): | ||
995 | qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator | ||
996 | fd = open("/proc/sys/fs/binfmt_misc/register", "w") | ||
997 | fd.write(qemu_arm_string) | ||
998 | fd.close() | ||
999 | |||
1000 | return qemu_emulator | ||
1001 | |||
1002 | def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir): | ||
1003 | def get_source_repometadata(repometadata): | ||
1004 | src_repometadata=[] | ||
1005 | for repo in repometadata: | ||
1006 | if repo["name"].endswith("-source"): | ||
1007 | src_repometadata.append(repo) | ||
1008 | if src_repometadata: | ||
1009 | return src_repometadata | ||
1010 | return None | ||
1011 | |||
1012 | def get_src_name(srpm): | ||
1013 | m = SRPM_RE.match(srpm) | ||
1014 | if m: | ||
1015 | return m.group(1) | ||
1016 | return None | ||
1017 | |||
1018 | src_repometadata = get_source_repometadata(repometadata) | ||
1019 | |||
1020 | if not src_repometadata: | ||
1021 | msger.warning("No source repo found") | ||
1022 | return None | ||
1023 | |||
1024 | src_pkgs = [] | ||
1025 | lpkgs_dict = {} | ||
1026 | lpkgs_path = [] | ||
1027 | for repo in src_repometadata: | ||
1028 | cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"]) | ||
1029 | lpkgs_path += glob.glob(cachepath) | ||
1030 | |||
1031 | for lpkg in lpkgs_path: | ||
1032 | lpkg_name = get_src_name(os.path.basename(lpkg)) | ||
1033 | lpkgs_dict[lpkg_name] = lpkg | ||
1034 | localpkgs = lpkgs_dict.keys() | ||
1035 | |||
1036 | cached_count = 0 | ||
1037 | destdir = instroot+'/usr/src/SRPMS' | ||
1038 | if not os.path.exists(destdir): | ||
1039 | os.makedirs(destdir) | ||
1040 | |||
1041 | srcpkgset = set() | ||
1042 | for _pkg in pkgs: | ||
1043 | srcpkg_name = get_source_name(_pkg, repometadata) | ||
1044 | if not srcpkg_name: | ||
1045 | continue | ||
1046 | srcpkgset.add(srcpkg_name) | ||
1047 | |||
1048 | for pkg in list(srcpkgset): | ||
1049 | if pkg in localpkgs: | ||
1050 | cached_count += 1 | ||
1051 | shutil.copy(lpkgs_dict[pkg], destdir) | ||
1052 | src_pkgs.append(os.path.basename(lpkgs_dict[pkg])) | ||
1053 | else: | ||
1054 | src_pkg = get_package(pkg, src_repometadata, 'src') | ||
1055 | if src_pkg: | ||
1056 | shutil.copy(src_pkg, destdir) | ||
1057 | src_pkgs.append(src_pkg) | ||
1058 | msger.info("%d source packages gotten from cache" % cached_count) | ||
1059 | |||
1060 | return src_pkgs | ||
1061 | |||
1062 | def strip_end(text, suffix): | ||
1063 | if not text.endswith(suffix): | ||
1064 | return text | ||
1065 | return text[:-len(suffix)] | ||