summaryrefslogtreecommitdiffstats
path: root/scripts/lib/mic/utils
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/lib/mic/utils')
-rw-r--r--scripts/lib/mic/utils/misc.py782
-rw-r--r--scripts/lib/mic/utils/proxy.py183
2 files changed, 0 insertions, 965 deletions
diff --git a/scripts/lib/mic/utils/misc.py b/scripts/lib/mic/utils/misc.py
index 95241d7f15..8c1f0160b1 100644
--- a/scripts/lib/mic/utils/misc.py
+++ b/scripts/lib/mic/utils/misc.py
@@ -42,15 +42,8 @@ xmlparse = cElementTree.parse
42from mic import msger 42from mic import msger
43from mic.utils.errors import CreatorError, SquashfsError 43from mic.utils.errors import CreatorError, SquashfsError
44from mic.utils.fs_related import find_binary_path, makedirs 44from mic.utils.fs_related import find_binary_path, makedirs
45from mic.utils.proxy import get_proxy_for
46from mic.utils import runner 45from mic.utils import runner
47 46
48
49RPM_RE = re.compile("(.*)\.(.*) (.*)-(.*)")
50RPM_FMT = "%(name)s.%(arch)s %(version)s-%(release)s"
51SRPM_RE = re.compile("(.*)-(\d+.*)-(\d+\.\d+).src.rpm")
52
53
54def build_name(kscfg, release=None, prefix = None, suffix = None): 47def build_name(kscfg, release=None, prefix = None, suffix = None):
55 """Construct and return an image name string. 48 """Construct and return an image name string.
56 49
@@ -123,136 +116,6 @@ def get_distro_str():
123 116
124_LOOP_RULE_PTH = None 117_LOOP_RULE_PTH = None
125 118
126def hide_loopdev_presentation():
127 udev_rules = "80-prevent-loop-present.rules"
128 udev_rules_dir = [
129 '/usr/lib/udev/rules.d/',
130 '/lib/udev/rules.d/',
131 '/etc/udev/rules.d/'
132 ]
133
134 global _LOOP_RULE_PTH
135
136 for rdir in udev_rules_dir:
137 if os.path.exists(rdir):
138 _LOOP_RULE_PTH = os.path.join(rdir, udev_rules)
139
140 if not _LOOP_RULE_PTH:
141 return
142
143 try:
144 with open(_LOOP_RULE_PTH, 'w') as wf:
145 wf.write('KERNEL=="loop*", ENV{UDISKS_PRESENTATION_HIDE}="1"')
146
147 runner.quiet('udevadm trigger')
148 except:
149 pass
150
151def unhide_loopdev_presentation():
152 global _LOOP_RULE_PTH
153
154 if not _LOOP_RULE_PTH:
155 return
156
157 try:
158 os.unlink(_LOOP_RULE_PTH)
159 runner.quiet('udevadm trigger')
160 except:
161 pass
162
163def extract_rpm(rpmfile, targetdir):
164 rpm2cpio = find_binary_path("rpm2cpio")
165 cpio = find_binary_path("cpio")
166
167 olddir = os.getcwd()
168 os.chdir(targetdir)
169
170 msger.verbose("Extract rpm file with cpio: %s" % rpmfile)
171 p1 = subprocess.Popen([rpm2cpio, rpmfile], stdout=subprocess.PIPE)
172 p2 = subprocess.Popen([cpio, "-idv"], stdin=p1.stdout,
173 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
174 (sout, serr) = p2.communicate()
175 msger.verbose(sout or serr)
176
177 os.chdir(olddir)
178
179def compressing(fpath, method):
180 comp_map = {
181 "gz": "gzip",
182 "bz2": "bzip2"
183 }
184 if method not in comp_map:
185 raise CreatorError("Unsupport compress format: %s, valid values: %s"
186 % (method, ','.join(comp_map.keys())))
187 cmd = find_binary_path(comp_map[method])
188 rc = runner.show([cmd, "-f", fpath])
189 if rc:
190 raise CreatorError("Failed to %s file: %s" % (comp_map[method], fpath))
191
192def taring(dstfile, target):
193 import tarfile
194 basen, ext = os.path.splitext(dstfile)
195 comp = {".tar": None,
196 ".gz": "gz", # for .tar.gz
197 ".bz2": "bz2", # for .tar.bz2
198 ".tgz": "gz",
199 ".tbz": "bz2"}[ext]
200
201 # specify tarball file path
202 if not comp:
203 tarpath = dstfile
204 elif basen.endswith(".tar"):
205 tarpath = basen
206 else:
207 tarpath = basen + ".tar"
208 wf = tarfile.open(tarpath, 'w')
209
210 if os.path.isdir(target):
211 for item in os.listdir(target):
212 wf.add(os.path.join(target, item), item)
213 else:
214 wf.add(target, os.path.basename(target))
215 wf.close()
216
217 if comp:
218 compressing(tarpath, comp)
219 # when dstfile ext is ".tgz" and ".tbz", should rename
220 if not basen.endswith(".tar"):
221 shutil.move("%s.%s" % (tarpath, comp), dstfile)
222
223def ziping(dstfile, target):
224 import zipfile
225 wf = zipfile.ZipFile(dstfile, 'w', compression=zipfile.ZIP_DEFLATED)
226 if os.path.isdir(target):
227 for item in os.listdir(target):
228 fpath = os.path.join(target, item)
229 if not os.path.isfile(fpath):
230 continue
231 wf.write(fpath, item, zipfile.ZIP_DEFLATED)
232 else:
233 wf.write(target, os.path.basename(target), zipfile.ZIP_DEFLATED)
234 wf.close()
235
236pack_formats = {
237 ".tar": taring,
238 ".tar.gz": taring,
239 ".tar.bz2": taring,
240 ".tgz": taring,
241 ".tbz": taring,
242 ".zip": ziping,
243}
244
245def packing(dstfile, target):
246 (base, ext) = os.path.splitext(dstfile)
247 if ext in (".gz", ".bz2") and base.endswith(".tar"):
248 ext = ".tar" + ext
249 if ext not in pack_formats:
250 raise CreatorError("Unsupport pack format: %s, valid values: %s"
251 % (ext, ','.join(pack_formats.keys())))
252 func = pack_formats[ext]
253 # func should be callable
254 func(dstfile, target)
255
256def human_size(size): 119def human_size(size):
257 """Return human readable string for Bytes size 120 """Return human readable string for Bytes size
258 """ 121 """
@@ -371,22 +234,6 @@ def normalize_ksfile(ksconf, release, arch):
371 return ksconf 234 return ksconf
372 235
373 236
374def _check_mic_chroot(rootdir):
375 def _path(path):
376 return rootdir.rstrip('/') + path
377
378 release_files = map(_path, [ "/etc/moblin-release",
379 "/etc/meego-release",
380 "/etc/tizen-release"])
381
382 if not any(map(os.path.exists, release_files)):
383 msger.warning("Dir %s is not a MeeGo/Tizen chroot env" % rootdir)
384
385 if not glob.glob(rootdir + "/boot/vmlinuz-*"):
386 msger.warning("Failed to find kernel module under %s" % rootdir)
387
388 return
389
390def selinux_check(arch, fstypes): 237def selinux_check(arch, fstypes):
391 try: 238 try:
392 getenforce = find_binary_path('getenforce') 239 getenforce = find_binary_path('getenforce')
@@ -403,64 +250,6 @@ def selinux_check(arch, fstypes):
403 raise CreatorError("Can't create btrfs image if selinux is enabled," 250 raise CreatorError("Can't create btrfs image if selinux is enabled,"
404 " please run 'setenforce 0' to disable selinux") 251 " please run 'setenforce 0' to disable selinux")
405 252
406def get_image_type(path):
407 def _get_extension_name(path):
408 match = re.search("(?<=\.)\w+$", path)
409 if match:
410 return match.group(0)
411 else:
412 return None
413
414 if os.path.isdir(path):
415 _check_mic_chroot(path)
416 return "fs"
417
418 maptab = {
419 "tar": "loop",
420 "raw":"raw",
421 "vmdk":"vmdk",
422 "vdi":"vdi",
423 "iso":"livecd",
424 "usbimg":"liveusb",
425 }
426
427 extension = _get_extension_name(path)
428 if extension in maptab:
429 return maptab[extension]
430
431 fd = open(path, "rb")
432 file_header = fd.read(1024)
433 fd.close()
434 vdi_flag = "<<< Sun VirtualBox Disk Image >>>"
435 if file_header[0:len(vdi_flag)] == vdi_flag:
436 return maptab["vdi"]
437
438 output = runner.outs(['file', path])
439 isoptn = re.compile(r".*ISO 9660 CD-ROM filesystem.*(bootable).*")
440 usbimgptn = re.compile(r".*x86 boot sector.*active.*")
441 rawptn = re.compile(r".*x86 boot sector.*")
442 vmdkptn = re.compile(r".*VMware. disk image.*")
443 ext3fsimgptn = re.compile(r".*Linux.*ext3 filesystem data.*")
444 ext4fsimgptn = re.compile(r".*Linux.*ext4 filesystem data.*")
445 btrfsimgptn = re.compile(r".*BTRFS.*")
446 if isoptn.match(output):
447 return maptab["iso"]
448 elif usbimgptn.match(output):
449 return maptab["usbimg"]
450 elif rawptn.match(output):
451 return maptab["raw"]
452 elif vmdkptn.match(output):
453 return maptab["vmdk"]
454 elif ext3fsimgptn.match(output):
455 return "ext3fsimg"
456 elif ext4fsimgptn.match(output):
457 return "ext4fsimg"
458 elif btrfsimgptn.match(output):
459 return "btrfsimg"
460 else:
461 raise CreatorError("Cannot detect the type of image: %s" % path)
462
463
464def get_file_size(filename): 253def get_file_size(filename):
465 """ Return size in MB unit """ 254 """ Return size in MB unit """
466 cmd = ['du', "-s", "-b", "-B", "1M", filename] 255 cmd = ['du', "-s", "-b", "-B", "1M", filename]
@@ -482,583 +271,12 @@ def get_filesystem_avail(fs):
482 vfstat = os.statvfs(fs) 271 vfstat = os.statvfs(fs)
483 return vfstat.f_bavail * vfstat.f_bsize 272 return vfstat.f_bavail * vfstat.f_bsize
484 273
485def convert_image(srcimg, srcfmt, dstimg, dstfmt):
486 #convert disk format
487 if dstfmt != "raw":
488 raise CreatorError("Invalid destination image format: %s" % dstfmt)
489 msger.debug("converting %s image to %s" % (srcimg, dstimg))
490 if srcfmt == "vmdk":
491 path = find_binary_path("qemu-img")
492 argv = [path, "convert", "-f", "vmdk", srcimg, "-O", dstfmt, dstimg]
493 elif srcfmt == "vdi":
494 path = find_binary_path("VBoxManage")
495 argv = [path, "internalcommands", "converttoraw", srcimg, dstimg]
496 else:
497 raise CreatorError("Invalid soure image format: %s" % srcfmt)
498
499 rc = runner.show(argv)
500 if rc == 0:
501 msger.debug("convert successful")
502 if rc != 0:
503 raise CreatorError("Unable to convert disk to %s" % dstfmt)
504
505def uncompress_squashfs(squashfsimg, outdir):
506 """Uncompress file system from squshfs image"""
507 unsquashfs = find_binary_path("unsquashfs")
508 args = [ unsquashfs, "-d", outdir, squashfsimg ]
509 rc = runner.show(args)
510 if (rc != 0):
511 raise SquashfsError("Failed to uncompress %s." % squashfsimg)
512
513def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"): 274def mkdtemp(dir = "/var/tmp", prefix = "wic-tmp-"):
514 """ FIXME: use the dir in wic.conf instead """ 275 """ FIXME: use the dir in wic.conf instead """
515 276
516 makedirs(dir) 277 makedirs(dir)
517 return tempfile.mkdtemp(dir = dir, prefix = prefix) 278 return tempfile.mkdtemp(dir = dir, prefix = prefix)
518 279
519def get_repostrs_from_ks(ks):
520 def _get_temp_reponame(baseurl):
521 md5obj = hashlib.md5(baseurl)
522 tmpreponame = "%s" % md5obj.hexdigest()
523 return tmpreponame
524
525 kickstart_repos = []
526
527 for repodata in ks.handler.repo.repoList:
528 repo = {}
529 for attr in ('name',
530 'baseurl',
531 'mirrorlist',
532 'includepkgs', # val is list
533 'excludepkgs', # val is list
534 'cost', # int
535 'priority',# int
536 'save',
537 'proxy',
538 'proxyuser',
539 'proxypasswd',
540 'proxypasswd',
541 'debuginfo',
542 'source',
543 'gpgkey',
544 'ssl_verify'):
545 if hasattr(repodata, attr) and getattr(repodata, attr):
546 repo[attr] = getattr(repodata, attr)
547
548 if 'name' not in repo:
549 repo['name'] = _get_temp_reponame(repodata.baseurl)
550
551 kickstart_repos.append(repo)
552
553 return kickstart_repos
554
555def _get_uncompressed_data_from_url(url, filename, proxies):
556 filename = myurlgrab(url, filename, proxies)
557 suffix = None
558 if filename.endswith(".gz"):
559 suffix = ".gz"
560 runner.quiet(['gunzip', "-f", filename])
561 elif filename.endswith(".bz2"):
562 suffix = ".bz2"
563 runner.quiet(['bunzip2', "-f", filename])
564 if suffix:
565 filename = filename.replace(suffix, "")
566 return filename
567
568def _get_metadata_from_repo(baseurl, proxies, cachedir, reponame, filename,
569 sumtype=None, checksum=None):
570 url = os.path.join(baseurl, filename)
571 filename_tmp = str("%s/%s/%s" % (cachedir, reponame, os.path.basename(filename)))
572 if os.path.splitext(filename_tmp)[1] in (".gz", ".bz2"):
573 filename = os.path.splitext(filename_tmp)[0]
574 else:
575 filename = filename_tmp
576 if sumtype and checksum and os.path.exists(filename):
577 try:
578 sumcmd = find_binary_path("%ssum" % sumtype)
579 except:
580 file_checksum = None
581 else:
582 file_checksum = runner.outs([sumcmd, filename]).split()[0]
583
584 if file_checksum and file_checksum == checksum:
585 return filename
586
587 return _get_uncompressed_data_from_url(url,filename_tmp,proxies)
588
589def get_metadata_from_repos(repos, cachedir):
590 my_repo_metadata = []
591 for repo in repos:
592 reponame = repo['name']
593 baseurl = repo['baseurl']
594
595
596 if 'proxy' in repo:
597 proxy = repo['proxy']
598 else:
599 proxy = get_proxy_for(baseurl)
600
601 proxies = None
602 if proxy:
603 proxies = {str(baseurl.split(":")[0]):str(proxy)}
604
605 makedirs(os.path.join(cachedir, reponame))
606 url = os.path.join(baseurl, "repodata/repomd.xml")
607 filename = os.path.join(cachedir, reponame, 'repomd.xml')
608 repomd = myurlgrab(url, filename, proxies)
609 try:
610 root = xmlparse(repomd)
611 except SyntaxError:
612 raise CreatorError("repomd.xml syntax error.")
613
614 ns = root.getroot().tag
615 ns = ns[0:ns.rindex("}")+1]
616
617 filepaths = {}
618 checksums = {}
619 sumtypes = {}
620
621 for elm in root.getiterator("%sdata" % ns):
622 if elm.attrib["type"] == "patterns":
623 filepaths['patterns'] = elm.find("%slocation" % ns).attrib['href']
624 checksums['patterns'] = elm.find("%sopen-checksum" % ns).text
625 sumtypes['patterns'] = elm.find("%sopen-checksum" % ns).attrib['type']
626 break
627
628 for elm in root.getiterator("%sdata" % ns):
629 if elm.attrib["type"] in ("group_gz", "group"):
630 filepaths['comps'] = elm.find("%slocation" % ns).attrib['href']
631 checksums['comps'] = elm.find("%sopen-checksum" % ns).text
632 sumtypes['comps'] = elm.find("%sopen-checksum" % ns).attrib['type']
633 break
634
635 primary_type = None
636 for elm in root.getiterator("%sdata" % ns):
637 if elm.attrib["type"] in ("primary_db", "primary"):
638 primary_type = elm.attrib["type"]
639 filepaths['primary'] = elm.find("%slocation" % ns).attrib['href']
640 checksums['primary'] = elm.find("%sopen-checksum" % ns).text
641 sumtypes['primary'] = elm.find("%sopen-checksum" % ns).attrib['type']
642 break
643
644 if not primary_type:
645 continue
646
647 for item in ("primary", "patterns", "comps"):
648 if item not in filepaths:
649 filepaths[item] = None
650 continue
651 if not filepaths[item]:
652 continue
653 filepaths[item] = _get_metadata_from_repo(baseurl,
654 proxies,
655 cachedir,
656 reponame,
657 filepaths[item],
658 sumtypes[item],
659 checksums[item])
660
661 """ Get repo key """
662 try:
663 repokey = _get_metadata_from_repo(baseurl,
664 proxies,
665 cachedir,
666 reponame,
667 "repodata/repomd.xml.key")
668 except CreatorError:
669 repokey = None
670 msger.debug("\ncan't get %s/%s" % (baseurl, "repodata/repomd.xml.key"))
671
672 my_repo_metadata.append({"name":reponame,
673 "baseurl":baseurl,
674 "repomd":repomd,
675 "primary":filepaths['primary'],
676 "cachedir":cachedir,
677 "proxies":proxies,
678 "patterns":filepaths['patterns'],
679 "comps":filepaths['comps'],
680 "repokey":repokey})
681
682 return my_repo_metadata
683
684def get_rpmver_in_repo(repometadata):
685 for repo in repometadata:
686 if repo["primary"].endswith(".xml"):
687 root = xmlparse(repo["primary"])
688 ns = root.getroot().tag
689 ns = ns[0:ns.rindex("}")+1]
690
691 versionlist = []
692 for elm in root.getiterator("%spackage" % ns):
693 if elm.find("%sname" % ns).text == 'rpm':
694 for node in elm.getchildren():
695 if node.tag == "%sversion" % ns:
696 versionlist.append(node.attrib['ver'])
697
698 if versionlist:
699 return reversed(
700 sorted(
701 versionlist,
702 key = lambda ver: map(int, ver.split('.')))).next()
703
704 elif repo["primary"].endswith(".sqlite"):
705 con = sqlite.connect(repo["primary"])
706 for row in con.execute("select version from packages where "
707 "name=\"rpm\" ORDER by version DESC"):
708 con.close()
709 return row[0]
710
711 return None
712
713def get_arch(repometadata):
714 archlist = []
715 for repo in repometadata:
716 if repo["primary"].endswith(".xml"):
717 root = xmlparse(repo["primary"])
718 ns = root.getroot().tag
719 ns = ns[0:ns.rindex("}")+1]
720 for elm in root.getiterator("%spackage" % ns):
721 if elm.find("%sarch" % ns).text not in ("noarch", "src"):
722 arch = elm.find("%sarch" % ns).text
723 if arch not in archlist:
724 archlist.append(arch)
725 elif repo["primary"].endswith(".sqlite"):
726 con = sqlite.connect(repo["primary"])
727 for row in con.execute("select arch from packages where arch not in (\"src\", \"noarch\")"):
728 if row[0] not in archlist:
729 archlist.append(row[0])
730
731 con.close()
732
733 uniq_arch = []
734 for i in range(len(archlist)):
735 if archlist[i] not in rpmmisc.archPolicies.keys():
736 continue
737 need_append = True
738 j = 0
739 while j < len(uniq_arch):
740 if archlist[i] in rpmmisc.archPolicies[uniq_arch[j]].split(':'):
741 need_append = False
742 break
743 if uniq_arch[j] in rpmmisc.archPolicies[archlist[i]].split(':'):
744 if need_append:
745 uniq_arch[j] = archlist[i]
746 need_append = False
747 else:
748 uniq_arch.remove(uniq_arch[j])
749 continue
750 j += 1
751 if need_append:
752 uniq_arch.append(archlist[i])
753
754 return uniq_arch, archlist
755
756def get_package(pkg, repometadata, arch = None):
757 ver = ""
758 target_repo = None
759 if not arch:
760 arches = []
761 elif arch not in rpmmisc.archPolicies:
762 arches = [arch]
763 else:
764 arches = rpmmisc.archPolicies[arch].split(':')
765 arches.append('noarch')
766
767 for repo in repometadata:
768 if repo["primary"].endswith(".xml"):
769 root = xmlparse(repo["primary"])
770 ns = root.getroot().tag
771 ns = ns[0:ns.rindex("}")+1]
772 for elm in root.getiterator("%spackage" % ns):
773 if elm.find("%sname" % ns).text == pkg:
774 if elm.find("%sarch" % ns).text in arches:
775 version = elm.find("%sversion" % ns)
776 tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
777 if tmpver > ver:
778 ver = tmpver
779 location = elm.find("%slocation" % ns)
780 pkgpath = "%s" % location.attrib['href']
781 target_repo = repo
782 break
783 if repo["primary"].endswith(".sqlite"):
784 con = sqlite.connect(repo["primary"])
785 if arch:
786 sql = 'select version, release, location_href from packages ' \
787 'where name = "%s" and arch IN ("%s")' % \
788 (pkg, '","'.join(arches))
789 for row in con.execute(sql):
790 tmpver = "%s-%s" % (row[0], row[1])
791 if tmpver > ver:
792 ver = tmpver
793 pkgpath = "%s" % row[2]
794 target_repo = repo
795 break
796 else:
797 sql = 'select version, release, location_href from packages ' \
798 'where name = "%s"' % pkg
799 for row in con.execute(sql):
800 tmpver = "%s-%s" % (row[0], row[1])
801 if tmpver > ver:
802 ver = tmpver
803 pkgpath = "%s" % row[2]
804 target_repo = repo
805 break
806 con.close()
807 if target_repo:
808 makedirs("%s/packages/%s" % (target_repo["cachedir"], target_repo["name"]))
809 url = os.path.join(target_repo["baseurl"], pkgpath)
810 filename = str("%s/packages/%s/%s" % (target_repo["cachedir"], target_repo["name"], os.path.basename(pkgpath)))
811 if os.path.exists(filename):
812 ret = rpmmisc.checkRpmIntegrity('rpm', filename)
813 if ret == 0:
814 return filename
815
816 msger.warning("package %s is damaged: %s" %
817 (os.path.basename(filename), filename))
818 os.unlink(filename)
819
820 pkg = myurlgrab(str(url), filename, target_repo["proxies"])
821 return pkg
822 else:
823 return None
824
825def get_source_name(pkg, repometadata):
826
827 def get_bin_name(pkg):
828 m = RPM_RE.match(pkg)
829 if m:
830 return m.group(1)
831 return None
832
833 def get_src_name(srpm):
834 m = SRPM_RE.match(srpm)
835 if m:
836 return m.group(1)
837 return None
838
839 ver = ""
840 target_repo = None
841
842 pkg_name = get_bin_name(pkg)
843 if not pkg_name:
844 return None
845
846 for repo in repometadata:
847 if repo["primary"].endswith(".xml"):
848 root = xmlparse(repo["primary"])
849 ns = root.getroot().tag
850 ns = ns[0:ns.rindex("}")+1]
851 for elm in root.getiterator("%spackage" % ns):
852 if elm.find("%sname" % ns).text == pkg_name:
853 if elm.find("%sarch" % ns).text != "src":
854 version = elm.find("%sversion" % ns)
855 tmpver = "%s-%s" % (version.attrib['ver'], version.attrib['rel'])
856 if tmpver > ver:
857 ver = tmpver
858 fmt = elm.find("%sformat" % ns)
859 if fmt:
860 fns = fmt.getchildren()[0].tag
861 fns = fns[0:fns.rindex("}")+1]
862 pkgpath = fmt.find("%ssourcerpm" % fns).text
863 target_repo = repo
864 break
865
866 if repo["primary"].endswith(".sqlite"):
867 con = sqlite.connect(repo["primary"])
868 for row in con.execute("select version, release, rpm_sourcerpm from packages where name = \"%s\" and arch != \"src\"" % pkg_name):
869 tmpver = "%s-%s" % (row[0], row[1])
870 if tmpver > ver:
871 pkgpath = "%s" % row[2]
872 target_repo = repo
873 break
874 con.close()
875 if target_repo:
876 return get_src_name(pkgpath)
877 else:
878 return None
879
880def get_pkglist_in_patterns(group, patterns):
881 found = False
882 pkglist = []
883 try:
884 root = xmlparse(patterns)
885 except SyntaxError:
886 raise SyntaxError("%s syntax error." % patterns)
887
888 for elm in list(root.getroot()):
889 ns = elm.tag
890 ns = ns[0:ns.rindex("}")+1]
891 name = elm.find("%sname" % ns)
892 summary = elm.find("%ssummary" % ns)
893 if name.text == group or summary.text == group:
894 found = True
895 break
896
897 if not found:
898 return pkglist
899
900 found = False
901 for requires in list(elm):
902 if requires.tag.endswith("requires"):
903 found = True
904 break
905
906 if not found:
907 return pkglist
908
909 for pkg in list(requires):
910 pkgname = pkg.attrib["name"]
911 if pkgname not in pkglist:
912 pkglist.append(pkgname)
913
914 return pkglist
915
916def get_pkglist_in_comps(group, comps):
917 found = False
918 pkglist = []
919 try:
920 root = xmlparse(comps)
921 except SyntaxError:
922 raise SyntaxError("%s syntax error." % comps)
923
924 for elm in root.getiterator("group"):
925 id = elm.find("id")
926 name = elm.find("name")
927 if id.text == group or name.text == group:
928 packagelist = elm.find("packagelist")
929 found = True
930 break
931
932 if not found:
933 return pkglist
934
935 for require in elm.getiterator("packagereq"):
936 if require.tag.endswith("packagereq"):
937 pkgname = require.text
938 if pkgname not in pkglist:
939 pkglist.append(pkgname)
940
941 return pkglist
942
943def is_statically_linked(binary):
944 return ", statically linked, " in runner.outs(['file', binary])
945
946def setup_qemu_emulator(rootdir, arch):
947 # mount binfmt_misc if it doesn't exist
948 if not os.path.exists("/proc/sys/fs/binfmt_misc"):
949 modprobecmd = find_binary_path("modprobe")
950 runner.show([modprobecmd, "binfmt_misc"])
951 if not os.path.exists("/proc/sys/fs/binfmt_misc/register"):
952 mountcmd = find_binary_path("mount")
953 runner.show([mountcmd, "-t", "binfmt_misc", "none", "/proc/sys/fs/binfmt_misc"])
954
955 # qemu_emulator is a special case, we can't use find_binary_path
956 # qemu emulator should be a statically-linked executable file
957 qemu_emulator = "/usr/bin/qemu-arm"
958 if not os.path.exists(qemu_emulator) or not is_statically_linked(qemu_emulator):
959 qemu_emulator = "/usr/bin/qemu-arm-static"
960 if not os.path.exists(qemu_emulator):
961 raise CreatorError("Please install a statically-linked qemu-arm")
962
963 # qemu emulator version check
964 armv7_list = [arch for arch in rpmmisc.archPolicies.keys() if arch.startswith('armv7')]
965 if arch in armv7_list: # need qemu (>=0.13.0)
966 qemuout = runner.outs([qemu_emulator, "-h"])
967 m = re.search("version\s*([.\d]+)", qemuout)
968 if m:
969 qemu_version = m.group(1)
970 if qemu_version < "0.13":
971 raise CreatorError("Requires %s version >=0.13 for %s" % (qemu_emulator, arch))
972 else:
973 msger.warning("Can't get version info of %s, please make sure it's higher than 0.13.0" % qemu_emulator)
974
975 if not os.path.exists(rootdir + "/usr/bin"):
976 makedirs(rootdir + "/usr/bin")
977 shutil.copy(qemu_emulator, rootdir + "/usr/bin/qemu-arm-static")
978 qemu_emulator = "/usr/bin/qemu-arm-static"
979
980 # disable selinux, selinux will block qemu emulator to run
981 if os.path.exists("/usr/sbin/setenforce"):
982 msger.info('Try to disable selinux')
983 runner.show(["/usr/sbin/setenforce", "0"])
984
985 # unregister it if it has been registered and is a dynamically-linked executable
986 node = "/proc/sys/fs/binfmt_misc/arm"
987 if os.path.exists(node):
988 qemu_unregister_string = "-1\n"
989 fd = open("/proc/sys/fs/binfmt_misc/arm", "w")
990 fd.write(qemu_unregister_string)
991 fd.close()
992
993 # register qemu emulator for interpreting other arch executable file
994 if not os.path.exists(node):
995 qemu_arm_string = ":arm:M::\\x7fELF\\x01\\x01\\x01\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x02\\x00\\x28\\x00:\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\x00\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xff\\xfa\\xff\\xff\\xff:%s:\n" % qemu_emulator
996 fd = open("/proc/sys/fs/binfmt_misc/register", "w")
997 fd.write(qemu_arm_string)
998 fd.close()
999
1000 return qemu_emulator
1001
1002def SrcpkgsDownload(pkgs, repometadata, instroot, cachedir):
1003 def get_source_repometadata(repometadata):
1004 src_repometadata=[]
1005 for repo in repometadata:
1006 if repo["name"].endswith("-source"):
1007 src_repometadata.append(repo)
1008 if src_repometadata:
1009 return src_repometadata
1010 return None
1011
1012 def get_src_name(srpm):
1013 m = SRPM_RE.match(srpm)
1014 if m:
1015 return m.group(1)
1016 return None
1017
1018 src_repometadata = get_source_repometadata(repometadata)
1019
1020 if not src_repometadata:
1021 msger.warning("No source repo found")
1022 return None
1023
1024 src_pkgs = []
1025 lpkgs_dict = {}
1026 lpkgs_path = []
1027 for repo in src_repometadata:
1028 cachepath = "%s/%s/packages/*.src.rpm" %(cachedir, repo["name"])
1029 lpkgs_path += glob.glob(cachepath)
1030
1031 for lpkg in lpkgs_path:
1032 lpkg_name = get_src_name(os.path.basename(lpkg))
1033 lpkgs_dict[lpkg_name] = lpkg
1034 localpkgs = lpkgs_dict.keys()
1035
1036 cached_count = 0
1037 destdir = instroot+'/usr/src/SRPMS'
1038 if not os.path.exists(destdir):
1039 os.makedirs(destdir)
1040
1041 srcpkgset = set()
1042 for _pkg in pkgs:
1043 srcpkg_name = get_source_name(_pkg, repometadata)
1044 if not srcpkg_name:
1045 continue
1046 srcpkgset.add(srcpkg_name)
1047
1048 for pkg in list(srcpkgset):
1049 if pkg in localpkgs:
1050 cached_count += 1
1051 shutil.copy(lpkgs_dict[pkg], destdir)
1052 src_pkgs.append(os.path.basename(lpkgs_dict[pkg]))
1053 else:
1054 src_pkg = get_package(pkg, src_repometadata, 'src')
1055 if src_pkg:
1056 shutil.copy(src_pkg, destdir)
1057 src_pkgs.append(src_pkg)
1058 msger.info("%d source packages gotten from cache" % cached_count)
1059
1060 return src_pkgs
1061
1062def strip_end(text, suffix): 280def strip_end(text, suffix):
1063 if not text.endswith(suffix): 281 if not text.endswith(suffix):
1064 return text 282 return text
diff --git a/scripts/lib/mic/utils/proxy.py b/scripts/lib/mic/utils/proxy.py
deleted file mode 100644
index 91451a2d01..0000000000
--- a/scripts/lib/mic/utils/proxy.py
+++ /dev/null
@@ -1,183 +0,0 @@
1#!/usr/bin/python -tt
2#
3# Copyright (c) 2010, 2011 Intel, Inc.
4#
5# This program is free software; you can redistribute it and/or modify it
6# under the terms of the GNU General Public License as published by the Free
7# Software Foundation; version 2 of the License
8#
9# This program is distributed in the hope that it will be useful, but
10# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12# for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc., 59
16# Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17
18import os
19import urlparse
20
21_my_proxies = {}
22_my_noproxy = None
23_my_noproxy_list = []
24
25def set_proxy_environ():
26 global _my_noproxy, _my_proxies
27 if not _my_proxies:
28 return
29 for key in _my_proxies.keys():
30 os.environ[key + "_proxy"] = _my_proxies[key]
31 if not _my_noproxy:
32 return
33 os.environ["no_proxy"] = _my_noproxy
34
35def unset_proxy_environ():
36 for env in ('http_proxy',
37 'https_proxy',
38 'ftp_proxy',
39 'all_proxy'):
40 if env in os.environ:
41 del os.environ[env]
42
43 ENV=env.upper()
44 if ENV in os.environ:
45 del os.environ[ENV]
46
47def _set_proxies(proxy = None, no_proxy = None):
48 """Return a dictionary of scheme -> proxy server URL mappings.
49 """
50
51 global _my_noproxy, _my_proxies
52 _my_proxies = {}
53 _my_noproxy = None
54 proxies = []
55 if proxy:
56 proxies.append(("http_proxy", proxy))
57 if no_proxy:
58 proxies.append(("no_proxy", no_proxy))
59
60 # Get proxy settings from environment if not provided
61 if not proxy and not no_proxy:
62 proxies = os.environ.items()
63
64 # Remove proxy env variables, urllib2 can't handle them correctly
65 unset_proxy_environ()
66
67 for name, value in proxies:
68 name = name.lower()
69 if value and name[-6:] == '_proxy':
70 if name[0:2] != "no":
71 _my_proxies[name[:-6]] = value
72 else:
73 _my_noproxy = value
74
75def _ip_to_int(ip):
76 ipint=0
77 shift=24
78 for dec in ip.split("."):
79 ipint |= int(dec) << shift
80 shift -= 8
81 return ipint
82
83def _int_to_ip(val):
84 ipaddr=""
85 shift=0
86 for i in range(4):
87 dec = val >> shift
88 dec &= 0xff
89 ipaddr = ".%d%s" % (dec, ipaddr)
90 shift += 8
91 return ipaddr[1:]
92
93def _isip(host):
94 if host.replace(".", "").isdigit():
95 return True
96 return False
97
98def _set_noproxy_list():
99 global _my_noproxy, _my_noproxy_list
100 _my_noproxy_list = []
101 if not _my_noproxy:
102 return
103 for item in _my_noproxy.split(","):
104 item = item.strip()
105 if not item:
106 continue
107
108 if item[0] != '.' and item.find("/") == -1:
109 # Need to match it
110 _my_noproxy_list.append({"match":0,"needle":item})
111
112 elif item[0] == '.':
113 # Need to match at tail
114 _my_noproxy_list.append({"match":1,"needle":item})
115
116 elif item.find("/") > 3:
117 # IP/MASK, need to match at head
118 needle = item[0:item.find("/")].strip()
119 ip = _ip_to_int(needle)
120 netmask = 0
121 mask = item[item.find("/")+1:].strip()
122
123 if mask.isdigit():
124 netmask = int(mask)
125 netmask = ~((1<<(32-netmask)) - 1)
126 ip &= netmask
127 else:
128 shift=24
129 netmask=0
130 for dec in mask.split("."):
131 netmask |= int(dec) << shift
132 shift -= 8
133 ip &= netmask
134
135 _my_noproxy_list.append({"match":2,"needle":ip,"netmask":netmask})
136
137def _isnoproxy(url):
138 (scheme, host, path, parm, query, frag) = urlparse.urlparse(url)
139
140 if '@' in host:
141 user_pass, host = host.split('@', 1)
142
143 if ':' in host:
144 host, port = host.split(':', 1)
145
146 hostisip = _isip(host)
147 for item in _my_noproxy_list:
148 if hostisip and item["match"] <= 1:
149 continue
150
151 if item["match"] == 2 and hostisip:
152 if (_ip_to_int(host) & item["netmask"]) == item["needle"]:
153 return True
154
155 if item["match"] == 0:
156 if host == item["needle"]:
157 return True
158
159 if item["match"] == 1:
160 if host.rfind(item["needle"]) > 0:
161 return True
162
163 return False
164
165def set_proxies(proxy = None, no_proxy = None):
166 _set_proxies(proxy, no_proxy)
167 _set_noproxy_list()
168 set_proxy_environ()
169
170def get_proxy_for(url):
171 if url.startswith('file:') or _isnoproxy(url):
172 return None
173
174 type = url[0:url.index(":")]
175 proxy = None
176 if _my_proxies.has_key(type):
177 proxy = _my_proxies[type]
178 elif _my_proxies.has_key("http"):
179 proxy = _my_proxies["http"]
180 else:
181 proxy = None
182
183 return proxy