summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorAlexander Kanavin <alexander.kanavin@linux.intel.com>2017-02-13 16:44:48 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-03-14 14:42:17 +0000
commit67615e01751bdba4e2186c86c44bebd9ded5233b (patch)
treea0472e225aaa55eaadb10f419867fae3fcb31d0e /meta/lib
parentd4efcded26706f50f8ca98d76df2b349ed1f1792 (diff)
downloadpoky-67615e01751bdba4e2186c86c44bebd9ded5233b.tar.gz
rootfs_rpm.bbclass: migrate image creation to dnf
To properly look at this patch, you probably need a side-by-side diff viewing tool. (From OE-Core rev: 65581c68d130fa74d703f6c3c92560e053857ac7) Signed-off-by: Alexander Kanavin <alexander.kanavin@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oe/package_manager.py1192
-rw-r--r--meta/lib/oe/rootfs.py18
-rw-r--r--meta/lib/oe/sdk.py7
3 files changed, 229 insertions, 988 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 725997ce33..d51609189d 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -102,109 +102,14 @@ class Indexer(object, metaclass=ABCMeta):
102 102
103 103
104class RpmIndexer(Indexer): 104class RpmIndexer(Indexer):
105 def get_ml_prefix_and_os_list(self, arch_var=None, os_var=None):
106 package_archs = collections.OrderedDict()
107 target_os = collections.OrderedDict()
108
109 if arch_var is not None and os_var is not None:
110 package_archs['default'] = self.d.getVar(arch_var).split()
111 package_archs['default'].reverse()
112 target_os['default'] = self.d.getVar(os_var).strip()
113 else:
114 package_archs['default'] = self.d.getVar("PACKAGE_ARCHS").split()
115 # arch order is reversed. This ensures the -best- match is
116 # listed first!
117 package_archs['default'].reverse()
118 target_os['default'] = self.d.getVar("TARGET_OS").strip()
119 multilibs = self.d.getVar('MULTILIBS') or ""
120 for ext in multilibs.split():
121 eext = ext.split(':')
122 if len(eext) > 1 and eext[0] == 'multilib':
123 localdata = bb.data.createCopy(self.d)
124 default_tune_key = "DEFAULTTUNE_virtclass-multilib-" + eext[1]
125 default_tune = localdata.getVar(default_tune_key, False)
126 if default_tune is None:
127 default_tune_key = "DEFAULTTUNE_ML_" + eext[1]
128 default_tune = localdata.getVar(default_tune_key, False)
129 if default_tune:
130 localdata.setVar("DEFAULTTUNE", default_tune)
131 package_archs[eext[1]] = localdata.getVar('PACKAGE_ARCHS').split()
132 package_archs[eext[1]].reverse()
133 target_os[eext[1]] = localdata.getVar("TARGET_OS").strip()
134
135 ml_prefix_list = collections.OrderedDict()
136 for mlib in package_archs:
137 if mlib == 'default':
138 ml_prefix_list[mlib] = package_archs[mlib]
139 else:
140 ml_prefix_list[mlib] = list()
141 for arch in package_archs[mlib]:
142 if arch in ['all', 'noarch', 'any']:
143 ml_prefix_list[mlib].append(arch)
144 else:
145 ml_prefix_list[mlib].append(mlib + "_" + arch)
146
147 return (ml_prefix_list, target_os)
148
149 def write_index(self): 105 def write_index(self):
150 sdk_pkg_archs = (self.d.getVar('SDK_PACKAGE_ARCHS') or "").replace('-', '_').split()
151 all_mlb_pkg_archs = (self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS') or "").replace('-', '_').split()
152
153 mlb_prefix_list = self.get_ml_prefix_and_os_list()[0]
154
155 archs = set()
156 for item in mlb_prefix_list:
157 archs = archs.union(set(i.replace('-', '_') for i in mlb_prefix_list[item]))
158
159 if len(archs) == 0:
160 archs = archs.union(set(all_mlb_pkg_archs))
161
162 archs = archs.union(set(sdk_pkg_archs))
163
164 rpm_createrepo = bb.utils.which(os.environ['PATH'], "createrepo")
165 if not rpm_createrepo:
166 bb.error("Cannot rebuild index as createrepo was not found in %s" % os.environ['PATH'])
167 return
168
169 if self.d.getVar('PACKAGE_FEED_SIGN') == '1': 106 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
170 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND')) 107 raise NotImplementedError('Package feed signing not yet implementd for rpm')
171 else:
172 signer = None
173 index_cmds = []
174 repomd_files = []
175 rpm_dirs_found = False
176 for arch in archs:
177 dbpath = os.path.join(self.d.getVar('WORKDIR'), 'rpmdb', arch)
178 if os.path.exists(dbpath):
179 bb.utils.remove(dbpath, True)
180 arch_dir = os.path.join(self.deploy_dir, arch)
181 if not os.path.isdir(arch_dir):
182 continue
183
184 index_cmds.append("%s --dbpath %s --update -q %s" % \
185 (rpm_createrepo, dbpath, arch_dir))
186 repomd_files.append(os.path.join(arch_dir, 'repodata', 'repomd.xml'))
187
188 rpm_dirs_found = True
189
190 if not rpm_dirs_found:
191 bb.note("There are no packages in %s" % self.deploy_dir)
192 return
193 108
194 # Create repodata 109 createrepo_c = bb.utils.which(os.environ['PATH'], "createrepo_c")
195 result = oe.utils.multiprocess_exec(index_cmds, create_index) 110 result = create_index("%s --update -q %s" % (createrepo_c, self.deploy_dir))
196 if result: 111 if result:
197 bb.fatal('%s' % ('\n'.join(result))) 112 bb.fatal(result)
198 # Sign repomd
199 if signer:
200 for repomd in repomd_files:
201 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
202 is_ascii_sig = (feed_sig_type.upper() != "BIN")
203 signer.detach_sign(repomd,
204 self.d.getVar('PACKAGE_FEED_GPG_NAME'),
205 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
206 armor=is_ascii_sig)
207
208 113
209class OpkgIndexer(Indexer): 114class OpkgIndexer(Indexer):
210 def write_index(self): 115 def write_index(self):
@@ -347,117 +252,9 @@ class PkgsList(object, metaclass=ABCMeta):
347 def list_pkgs(self): 252 def list_pkgs(self):
348 pass 253 pass
349 254
350
351class RpmPkgsList(PkgsList): 255class RpmPkgsList(PkgsList):
352 def __init__(self, d, rootfs_dir, arch_var=None, os_var=None):
353 super(RpmPkgsList, self).__init__(d, rootfs_dir)
354
355 self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm")
356 self.image_rpmlib = os.path.join(self.rootfs_dir, 'var/lib/rpm')
357
358 self.ml_prefix_list, self.ml_os_list = \
359 RpmIndexer(d, rootfs_dir).get_ml_prefix_and_os_list(arch_var, os_var)
360
361 # Determine rpm version
362 try:
363 output = subprocess.check_output([self.rpm_cmd, "--version"], stderr=subprocess.STDOUT).decode("utf-8")
364 except subprocess.CalledProcessError as e:
365 bb.fatal("Getting rpm version failed. Command '%s' "
366 "returned %d:\n%s" % (self.rpm_cmd, e.returncode, e.output.decode("utf-8")))
367
368 '''
369 Translate the RPM/Smart format names to the OE multilib format names
370 '''
371 def _pkg_translate_smart_to_oe(self, pkg, arch):
372 new_pkg = pkg
373 new_arch = arch
374 fixed_arch = arch.replace('_', '-')
375 found = 0
376 for mlib in self.ml_prefix_list:
377 for cmp_arch in self.ml_prefix_list[mlib]:
378 fixed_cmp_arch = cmp_arch.replace('_', '-')
379 if fixed_arch == fixed_cmp_arch:
380 if mlib == 'default':
381 new_pkg = pkg
382 new_arch = cmp_arch
383 else:
384 new_pkg = mlib + '-' + pkg
385 # We need to strip off the ${mlib}_ prefix on the arch
386 new_arch = cmp_arch.replace(mlib + '_', '')
387
388 # Workaround for bug 3565. Simply look to see if we
389 # know of a package with that name, if not try again!
390 filename = os.path.join(self.d.getVar('PKGDATA_DIR'),
391 'runtime-reverse',
392 new_pkg)
393 if os.path.exists(filename):
394 found = 1
395 break
396
397 if found == 1 and fixed_arch == fixed_cmp_arch:
398 break
399 #bb.note('%s, %s -> %s, %s' % (pkg, arch, new_pkg, new_arch))
400 return new_pkg, new_arch
401
402 def _list_pkg_deps(self):
403 cmd = [bb.utils.which(os.getenv('PATH'), "rpmresolve"),
404 "-t", self.image_rpmlib]
405
406 try:
407 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8")
408 except subprocess.CalledProcessError as e:
409 bb.fatal("Cannot get the package dependencies. Command '%s' "
410 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
411
412 return output
413
414 def list_pkgs(self): 256 def list_pkgs(self):
415 cmd = [self.rpm_cmd, '--root', self.rootfs_dir] 257 return RpmPM(self.d, self.rootfs_dir, self.d.getVar('TARGET_VENDOR')).list_installed()
416 cmd.extend(['-D', '_dbpath /var/lib/rpm'])
417 cmd.extend(['-qa', '--qf', '[%{NAME} %{ARCH} %{VERSION} %{PACKAGEORIGIN}\n]'])
418
419 try:
420 tmp_output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).strip().decode("utf-8")
421 except subprocess.CalledProcessError as e:
422 bb.fatal("Cannot get the installed packages list. Command '%s' "
423 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
424
425 output = dict()
426 deps = dict()
427 dependencies = self._list_pkg_deps()
428
429 # Populate deps dictionary for better manipulation
430 for line in dependencies.splitlines():
431 try:
432 pkg, dep = line.split("|")
433 if not pkg in deps:
434 deps[pkg] = list()
435 if not dep in deps[pkg]:
436 deps[pkg].append(dep)
437 except:
438 # Ignore any other lines they're debug or errors
439 pass
440
441 for line in tmp_output.split('\n'):
442 if len(line.strip()) == 0:
443 continue
444 pkg = line.split()[0]
445 arch = line.split()[1]
446 ver = line.split()[2]
447 dep = deps.get(pkg, [])
448
449 # Skip GPG keys
450 if pkg == 'gpg-pubkey':
451 continue
452
453 pkgorigin = line.split()[3]
454 new_pkg, new_arch = self._pkg_translate_smart_to_oe(pkg, arch)
455
456 output[new_pkg] = {"arch":new_arch, "ver":ver,
457 "filename":pkgorigin, "deps":dep}
458
459 return output
460
461 258
462class OpkgPkgsList(PkgsList): 259class OpkgPkgsList(PkgsList):
463 def __init__(self, d, rootfs_dir, config_file): 260 def __init__(self, d, rootfs_dir, config_file):
@@ -553,6 +350,16 @@ class PackageManager(object, metaclass=ABCMeta):
553 pass 350 pass
554 351
555 """ 352 """
353 Returns the path to a tmpdir where resides the contents of a package.
354
355 Deleting the tmpdir is responsability of the caller.
356
357 """
358 @abstractmethod
359 def extract(self, pkg):
360 pass
361
362 """
556 Add remote package feeds into repository manager configuration. The parameters 363 Add remote package feeds into repository manager configuration. The parameters
557 for the feeds are set by feed_uris, feed_base_paths and feed_archs. 364 for the feeds are set by feed_uris, feed_base_paths and feed_archs.
558 See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS 365 See http://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PACKAGE_FEED_URIS
@@ -661,821 +468,264 @@ class RpmPM(PackageManager):
661 self.target_rootfs = target_rootfs 468 self.target_rootfs = target_rootfs
662 self.target_vendor = target_vendor 469 self.target_vendor = target_vendor
663 self.task_name = task_name 470 self.task_name = task_name
664 self.providename = providename 471 if arch_var == None:
665 self.fullpkglist = list() 472 self.archs = self.d.getVar('ALL_MULTILIB_PACKAGE_ARCHS').replace("-","_")
666 self.deploy_dir = self.d.getVar('DEPLOY_DIR_RPM') 473 else:
667 self.etcrpm_dir = os.path.join(self.target_rootfs, "etc/rpm") 474 self.archs = self.d.getVar(arch_var).replace("-","_")
668 self.install_dir_name = "oe_install" 475 if task_name == "host":
669 self.install_dir_path = os.path.join(self.target_rootfs, self.install_dir_name) 476 self.primary_arch = self.d.getVar('SDK_ARCH')
670 self.rpm_cmd = bb.utils.which(os.getenv('PATH'), "rpm") 477 else:
671 self.smart_cmd = bb.utils.which(os.getenv('PATH'), "smart") 478 self.primary_arch = self.d.getVar('MACHINE_ARCH')
672 # 0 = --log-level=warning, only warnings 479
673 # 1 = --log-level=info (includes information about executing scriptlets and their output), default 480 self.rpm_repo_dir = oe.path.join(self.d.getVar('WORKDIR'), "oe-rootfs-repo")
674 # 2 = --log-level=debug 481 bb.utils.mkdirhier(self.rpm_repo_dir)
675 # 3 = --log-level=debug plus dumps of scriplet content and command invocation 482 oe.path.symlink(self.d.getVar('DEPLOY_DIR_RPM'), oe.path.join(self.rpm_repo_dir, "rpm"), True)
676 self.debug_level = int(d.getVar('ROOTFS_RPM_DEBUG') or "1") 483
677 self.smart_opt = ["--log-level=%s" % 484 self.saved_packaging_data = self.d.expand('${T}/saved_packaging_data/%s' % self.task_name)
678 ("warning" if self.debug_level == 0 else 485 if not os.path.exists(self.d.expand('${T}/saved_packaging_data')):
679 "info" if self.debug_level == 1 else 486 bb.utils.mkdirhier(self.d.expand('${T}/saved_packaging_data'))
680 "debug"), "--data-dir=%s" % 487 self.packaging_data_dirs = ['var/lib/rpm', 'var/lib/dnf', 'var/cache/dnf']
681 os.path.join(target_rootfs, 'var/lib/smart')]
682 self.scriptlet_wrapper = self.d.expand('${WORKDIR}/scriptlet_wrapper')
683 self.solution_manifest = self.d.expand('${T}/saved/%s_solution' % 488 self.solution_manifest = self.d.expand('${T}/saved/%s_solution' %
684 self.task_name) 489 self.task_name)
685 self.saved_rpmlib = self.d.expand('${T}/saved/%s' % self.task_name)
686 self.image_rpmlib = os.path.join(self.target_rootfs, 'var/lib/rpm')
687
688 if not os.path.exists(self.d.expand('${T}/saved')): 490 if not os.path.exists(self.d.expand('${T}/saved')):
689 bb.utils.mkdirhier(self.d.expand('${T}/saved')) 491 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
690 492
691 packageindex_dir = os.path.join(self.d.getVar('WORKDIR'), 'rpms') 493 def _configure_dnf(self):
692 self.indexer = RpmIndexer(self.d, packageindex_dir) 494 # libsolv handles 'noarch' internally, we don't need to specify it explicitly
693 self.pkgs_list = RpmPkgsList(self.d, self.target_rootfs, arch_var, os_var) 495 archs = [i for i in self.archs.split() if i not in ["any", "all", "noarch"]]
496 # This prevents accidental matching against libsolv's built-in policies
497 if len(archs) <= 1:
498 archs = archs + ["bogusarch"]
499 archconfdir = "%s/%s" %(self.target_rootfs, "etc/dnf/vars/")
500 bb.utils.mkdirhier(archconfdir)
501 open(archconfdir + "arch", 'w').write(":".join(archs))
502
503 open(oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"), 'w').write("")
504
505
506 def _configure_rpm(self):
507 # We need to configure rpm to use our primary package architecture as the installation architecture,
508 # and to make it compatible with other package architectures that we use.
509 # Otherwise it will refuse to proceed with packages installation.
510 platformconfdir = "%s/%s" %(self.target_rootfs, "etc/rpm/")
511 rpmrcconfdir = "%s/%s" %(self.target_rootfs, "etc/")
512 bb.utils.mkdirhier(platformconfdir)
513 open(platformconfdir + "platform", 'w').write("%s-pc-linux" % self.primary_arch)
514 open(rpmrcconfdir + "rpmrc", 'w').write("arch_compat: %s: %s\n" % (self.primary_arch, self.archs if len(self.archs) > 0 else self.primary_arch))
515
516 open(platformconfdir + "macros", 'w').write("%_transaction_color 7\n")
517 if self.d.getVar('RPM_PREFER_ELF_ARCH'):
518 open(platformconfdir + "macros", 'a').write("%%_prefer_color %s" % (self.d.getVar('RPM_PREFER_ELF_ARCH')))
519 else:
520 open(platformconfdir + "macros", 'a').write("%_prefer_color 7")
521
522 if self.d.getVar('RPM_SIGN_PACKAGES') == '1':
523 raise NotImplementedError("Signature verification with rpm not yet supported.")
524
525 def create_configs(self):
526 self._configure_dnf()
527 self._configure_rpm()
694 528
695 self.ml_prefix_list, self.ml_os_list = self.indexer.get_ml_prefix_and_os_list(arch_var, os_var) 529 def write_index(self):
530 lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock"
531 lf = bb.utils.lockfile(lockfilename, False)
532 RpmIndexer(self.d, self.rpm_repo_dir).write_index()
533 bb.utils.unlockfile(lf)
696 534
697 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs): 535 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
698 if feed_uris == "": 536 if feed_uris == "":
699 return 537 return
700 538
701 arch_list = [] 539 raise NotImplementedError("Adding remote dnf feeds not yet supported.")
702 if feed_archs is not None:
703 # User define feed architectures
704 arch_list = feed_archs.split()
705 else:
706 # List must be prefered to least preferred order
707 default_platform_extra = list()
708 platform_extra = list()
709 bbextendvariant = self.d.getVar('BBEXTENDVARIANT') or ""
710 for mlib in self.ml_os_list:
711 for arch in self.ml_prefix_list[mlib]:
712 plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
713 if mlib == bbextendvariant:
714 if plt not in default_platform_extra:
715 default_platform_extra.append(plt)
716 else:
717 if plt not in platform_extra:
718 platform_extra.append(plt)
719 platform_extra = default_platform_extra + platform_extra
720
721 for canonical_arch in platform_extra:
722 arch = canonical_arch.split('-')[0]
723 if not os.path.exists(os.path.join(self.deploy_dir, arch)):
724 continue
725 arch_list.append(arch)
726 540
727 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split()) 541 def _prepare_pkg_transaction(self):
542 os.environ['D'] = self.target_rootfs
543 os.environ['OFFLINE_ROOT'] = self.target_rootfs
544 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
545 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
546 os.environ['INTERCEPT_DIR'] = oe.path.join(self.d.getVar('WORKDIR'),
547 "intercept_scripts")
548 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
728 549
729 uri_iterator = 0
730 channel_priority = 10 + 5 * len(feed_uris) * (len(arch_list) if arch_list else 1)
731
732 for uri in feed_uris:
733 if arch_list:
734 for arch in arch_list:
735 bb.note('Adding Smart channel url%d%s (%s)' %
736 (uri_iterator, arch, channel_priority))
737 self._invoke_smart(['channel', '--add', 'url%d-%s' % (uri_iterator, arch),
738 'type=rpm-md', 'baseurl=%s/%s' % (uri, arch), '-y'])
739 self._invoke_smart(['channel', '--set', 'url%d-%s' % (uri_iterator, arch),
740 'priority=%d' % channel_priority])
741 channel_priority -= 5
742 else:
743 bb.note('Adding Smart channel url%d (%s)' %
744 (uri_iterator, channel_priority))
745 self._invoke_smart(['channel', '--add', 'url%d' % uri_iterator,
746 'type=rpm-md', 'baseurl=%s' % uri, '-y'])
747 self._invoke_smart(['channel', '--set', 'url%d' % uri_iterator,
748 'priority=%d' % channel_priority])
749 channel_priority -= 5
750 550
751 uri_iterator += 1 551 def install(self, pkgs, attempt_only = False):
552 if len(pkgs) == 0:
553 return
554 self._prepare_pkg_transaction()
752 555
753 ''' 556 bad_recommendations = self.d.getVar('BAD_RECOMMENDATIONS')
754 Create configs for rpm and smart, and multilib is supported 557 package_exclude = self.d.getVar('PACKAGE_EXCLUDE')
755 ''' 558 exclude_pkgs = (bad_recommendations.split() if bad_recommendations else []) + (package_exlcude.split() if package_exclude else [])
756 def create_configs(self):
757 target_arch = self.d.getVar('TARGET_ARCH')
758 platform = '%s%s-%s' % (target_arch.replace('-', '_'),
759 self.target_vendor,
760 self.ml_os_list['default'])
761
762 # List must be prefered to least preferred order
763 default_platform_extra = list()
764 platform_extra = list()
765 bbextendvariant = self.d.getVar('BBEXTENDVARIANT') or ""
766 for mlib in self.ml_os_list:
767 for arch in self.ml_prefix_list[mlib]:
768 plt = arch.replace('-', '_') + '-.*-' + self.ml_os_list[mlib]
769 if mlib == bbextendvariant:
770 if plt not in default_platform_extra:
771 default_platform_extra.append(plt)
772 else:
773 if plt not in platform_extra:
774 platform_extra.append(plt)
775 platform_extra = default_platform_extra + platform_extra
776 559
777 self._create_configs(platform, platform_extra) 560 output = self._invoke_dnf((["--skip-broken"] if attempt_only else []) +
561 (["-x", ",".join(exclude_pkgs)] if len(exclude_pkgs) > 0 else []) +
562 (["--setopt=install_weak_deps=False"] if self.d.getVar('NO_RECOMMENDATIONS') == 1 else []) +
563 ["--nogpgcheck", "install"] +
564 pkgs)
778 565
779 #takes array args 566 failed_scriptlets_pkgnames = collections.OrderedDict()
780 def _invoke_smart(self, args): 567 for line in output.splitlines():
781 cmd = [self.smart_cmd] + self.smart_opt + args 568 if line.startswith("Non-fatal POSTIN scriptlet failure in rpm package"):
782 # bb.note(cmd) 569 failed_scriptlets_pkgnames[line.split()[-1]] = True
783 try:
784 complementary_pkgs = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8")
785 # bb.note(complementary_pkgs)
786 return complementary_pkgs
787 except subprocess.CalledProcessError as e:
788 bb.fatal("Could not invoke smart. Command "
789 "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
790 570
791 def _search_pkg_name_in_feeds(self, pkg, feed_archs): 571 for pkg in failed_scriptlets_pkgnames.keys():
792 for arch in feed_archs: 572 self.save_rpmpostinst(pkg)
793 arch = arch.replace('-', '_')
794 regex_match = re.compile(r"^%s-[^-]*-[^-]*@%s$" % \
795 (re.escape(pkg), re.escape(arch)))
796 for p in self.fullpkglist:
797 if regex_match.match(p) is not None:
798 # First found is best match
799 # bb.note('%s -> %s' % (pkg, pkg + '@' + arch))
800 return pkg + '@' + arch
801
802 # Search provides if not found by pkgname.
803 bb.note('Not found %s by name, searching provides ...' % pkg)
804 cmd = [self.smart_cmd] + self.smart_opt + ["query", "--provides", pkg,
805 "--show-format=$name-$version"]
806 bb.note('cmd: %s' % ' '.join(cmd))
807 ps = subprocess.Popen(cmd, stdout=subprocess.PIPE)
808 try:
809 output = subprocess.check_output(["sed", "-ne", "s/ *Provides://p"],
810 stdin=ps.stdout, stderr=subprocess.STDOUT).decode("utf-8")
811 # Found a provider
812 if output:
813 bb.note('Found providers for %s: %s' % (pkg, output))
814 for p in output.split():
815 for arch in feed_archs:
816 arch = arch.replace('-', '_')
817 if p.rstrip().endswith('@' + arch):
818 return p
819 except subprocess.CalledProcessError as e:
820 bb.error("Failed running smart query on package %s." % pkg)
821 573
822 return "" 574 def remove(self, pkgs, with_dependencies = True):
575 if len(pkgs) == 0:
576 return
577 self._prepare_pkg_transaction()
823 578
824 ''' 579 if with_dependencies:
825 Translate the OE multilib format names to the RPM/Smart format names 580 self._invoke_dnf(["remove"] + pkgs)
826 It searched the RPM/Smart format names in probable multilib feeds first, 581 else:
827 and then searched the default base feed. 582 cmd = bb.utils.which(os.getenv('PATH'), "rpm")
828 ''' 583 args = ["-e", "--nodeps", "--root=%s" %self.target_rootfs]
829 def _pkg_translate_oe_to_smart(self, pkgs, attempt_only=False):
830 new_pkgs = list()
831
832 for pkg in pkgs:
833 new_pkg = pkg
834 # Search new_pkg in probable multilibs first
835 for mlib in self.ml_prefix_list:
836 # Jump the default archs
837 if mlib == 'default':
838 continue
839 584
840 subst = pkg.replace(mlib + '-', '')
841 # if the pkg in this multilib feed
842 if subst != pkg:
843 feed_archs = self.ml_prefix_list[mlib]
844 new_pkg = self._search_pkg_name_in_feeds(subst, feed_archs)
845 if not new_pkg:
846 # Failed to translate, package not found!
847 err_msg = '%s not found in the %s feeds (%s) in %s.' % \
848 (pkg, mlib, " ".join(feed_archs), self.d.getVar('DEPLOY_DIR_RPM'))
849 if not attempt_only:
850 bb.error(err_msg)
851 bb.fatal("This is often caused by an empty package declared " \
852 "in a recipe's PACKAGES variable. (Empty packages are " \
853 "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)")
854 bb.warn(err_msg)
855 else:
856 new_pkgs.append(new_pkg)
857
858 break
859
860 # Apparently not a multilib package...
861 if pkg == new_pkg:
862 # Search new_pkg in default archs
863 default_archs = self.ml_prefix_list['default']
864 new_pkg = self._search_pkg_name_in_feeds(pkg, default_archs)
865 if not new_pkg:
866 err_msg = '%s not found in the feeds (%s) in %s.' % \
867 (pkg, " ".join(default_archs), self.d.getVar('DEPLOY_DIR_RPM'))
868 if not attempt_only:
869 bb.error(err_msg)
870 bb.fatal("This is often caused by an empty package declared " \
871 "in a recipe's PACKAGES variable. (Empty packages are " \
872 "not constructed unless ALLOW_EMPTY_<pkg> = '1' is used.)")
873 bb.warn(err_msg)
874 else:
875 new_pkgs.append(new_pkg)
876
877 return new_pkgs
878
879 def _create_configs(self, platform, platform_extra):
880 # Setup base system configuration
881 bb.note("configuring RPM platform settings")
882
883 # Configure internal RPM environment when using Smart
884 os.environ['RPM_ETCRPM'] = self.etcrpm_dir
885 bb.utils.mkdirhier(self.etcrpm_dir)
886
887 # Setup temporary directory -- install...
888 if os.path.exists(self.install_dir_path):
889 bb.utils.remove(self.install_dir_path, True)
890 bb.utils.mkdirhier(os.path.join(self.install_dir_path, 'tmp'))
891
892 channel_priority = 5
893 platform_dir = os.path.join(self.etcrpm_dir, "platform")
894 sdkos = self.d.getVar("SDK_OS")
895 with open(platform_dir, "w+") as platform_fd:
896 platform_fd.write(platform + '\n')
897 for pt in platform_extra:
898 channel_priority += 5
899 if sdkos:
900 tmp = re.sub("-%s$" % sdkos, "-%s\n" % sdkos, pt)
901 tmp = re.sub("-linux.*$", "-linux.*\n", tmp)
902 platform_fd.write(tmp)
903
904 # Tell RPM that the "/" directory exist and is available
905 bb.note("configuring RPM system provides")
906 sysinfo_dir = os.path.join(self.etcrpm_dir, "sysinfo")
907 bb.utils.mkdirhier(sysinfo_dir)
908 with open(os.path.join(sysinfo_dir, "Dirnames"), "w+") as dirnames:
909 dirnames.write("/\n")
910
911 if self.providename:
912 providename_dir = os.path.join(sysinfo_dir, "Providename")
913 if not os.path.exists(providename_dir):
914 providename_content = '\n'.join(self.providename)
915 providename_content += '\n'
916 open(providename_dir, "w+").write(providename_content)
917
918 # Configure RPM... we enforce these settings!
919 bb.note("configuring RPM DB settings")
920 # After change the __db.* cache size, log file will not be
921 # generated automatically, that will raise some warnings,
922 # so touch a bare log for rpm write into it.
923 rpmlib_log = os.path.join(self.image_rpmlib, 'log', 'log.0000000001')
924 if not os.path.exists(rpmlib_log):
925 bb.utils.mkdirhier(os.path.join(self.image_rpmlib, 'log'))
926 open(rpmlib_log, 'w+').close()
927
928 DB_CONFIG_CONTENT = "# ================ Environment\n" \
929 "set_data_dir .\n" \
930 "set_create_dir .\n" \
931 "set_lg_dir ./log\n" \
932 "set_tmp_dir ./tmp\n" \
933 "set_flags db_log_autoremove on\n" \
934 "\n" \
935 "# -- thread_count must be >= 8\n" \
936 "set_thread_count 64\n" \
937 "\n" \
938 "# ================ Logging\n" \
939 "\n" \
940 "# ================ Memory Pool\n" \
941 "set_cachesize 0 1048576 0\n" \
942 "set_mp_mmapsize 268435456\n" \
943 "\n" \
944 "# ================ Locking\n" \
945 "set_lk_max_locks 16384\n" \
946 "set_lk_max_lockers 16384\n" \
947 "set_lk_max_objects 16384\n" \
948 "mutex_set_max 163840\n" \
949 "\n" \
950 "# ================ Replication\n"
951
952 db_config_dir = os.path.join(self.image_rpmlib, 'DB_CONFIG')
953 if not os.path.exists(db_config_dir):
954 open(db_config_dir, 'w+').write(DB_CONFIG_CONTENT)
955
956 # Create database so that smart doesn't complain (lazy init)
957 cmd = [self.rpm_cmd, '--root', self.target_rootfs, '--dbpath', '/var/lib/rpm', '-qa']
958 try:
959 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
960 except subprocess.CalledProcessError as e:
961 bb.fatal("Create rpm database failed. Command '%s' "
962 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
963 # Import GPG key to RPM database of the target system
964 if self.d.getVar('RPM_SIGN_PACKAGES') == '1':
965 pubkey_path = self.d.getVar('RPM_GPG_PUBKEY')
966 cmd = [self.rpm_cmd, '--root', self.target_rootfs, '--dbpath', '/var/lib/rpm', '--import', pubkey_path]
967 try: 585 try:
968 subprocess.check_output(cmd, stderr=subprocess.STDOUT) 586 output = subprocess.check_output([cmd] + args + pkgs, stderr=subprocess.STDOUT).decode("utf-8")
969 except subprocess.CalledProcessError as e: 587 except subprocess.CalledProcessError as e:
970 bb.fatal("Import GPG key failed. Command '%s' " 588 bb.fatal("Could not invoke rpm. Command "
971 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 589 "'%s' returned %d:\n%s" % (' '.join([cmd] + args + pkgs), e.returncode, e.output.decode("utf-8")))
972
973
974 # Configure smart
975 bb.note("configuring Smart settings")
976 bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'),
977 True)
978 self._invoke_smart(['config', '--set', 'rpm-root=%s' % self.target_rootfs])
979 self._invoke_smart(['config', '--set', 'rpm-dbpath=/var/lib/rpm'])
980 self._invoke_smart(['config', '--set', 'rpm-extra-macros._var=%s' %
981 self.d.getVar('localstatedir')])
982 cmd = ["config", "--set", "rpm-extra-macros._tmppath=/%s/tmp" % self.install_dir_name]
983
984 prefer_color = self.d.getVar('RPM_PREFER_ELF_ARCH')
985 if prefer_color:
986 if prefer_color not in ['0', '1', '2', '4']:
987 bb.fatal("Invalid RPM_PREFER_ELF_ARCH: %s, it should be one of:\n"
988 "\t1: ELF32 wins\n"
989 "\t2: ELF64 wins\n"
990 "\t4: ELF64 N32 wins (mips64 or mips64el only)" %
991 prefer_color)
992 if prefer_color == "4" and self.d.getVar("TUNE_ARCH") not in \
993 ['mips64', 'mips64el']:
994 bb.fatal("RPM_PREFER_ELF_ARCH = \"4\" is for mips64 or mips64el "
995 "only.")
996 self._invoke_smart(['config', '--set', 'rpm-extra-macros._prefer_color=%s'
997 % prefer_color])
998
999 self._invoke_smart(cmd)
1000 self._invoke_smart(['config', '--set', 'rpm-ignoresize=1'])
1001
1002 # Write common configuration for host and target usage
1003 self._invoke_smart(['config', '--set', 'rpm-nolinktos=1'])
1004 self._invoke_smart(['config', '--set', 'rpm-noparentdirs=1'])
1005 check_signature = self.d.getVar('RPM_CHECK_SIGNATURES')
1006 if check_signature and check_signature.strip() == "0":
1007 self._invoke_smart(['config', '--set rpm-check-signatures=false'])
1008 for i in self.d.getVar('BAD_RECOMMENDATIONS').split():
1009 self._invoke_smart(['flag', '--set', 'ignore-recommends', i])
1010
1011 # Do the following configurations here, to avoid them being
1012 # saved for field upgrade
1013 if self.d.getVar('NO_RECOMMENDATIONS').strip() == "1":
1014 self._invoke_smart(['config', '--set', 'ignore-all-recommends=1'])
1015 pkg_exclude = self.d.getVar('PACKAGE_EXCLUDE') or ""
1016 for i in pkg_exclude.split():
1017 self._invoke_smart(['flag', '--set', 'exclude-packages', i])
1018
1019 # Optional debugging
1020 # self._invoke_smart(['config', '--set', 'rpm-log-level=debug'])
1021 # cmd = ['config', '--set', 'rpm-log-file=/tmp/smart-debug-logfile']
1022 # self._invoke_smart(cmd)
1023 ch_already_added = []
1024 for canonical_arch in platform_extra:
1025 arch = canonical_arch.split('-')[0]
1026 arch_channel = os.path.join(self.d.getVar('WORKDIR'), 'rpms', arch)
1027 oe.path.remove(arch_channel)
1028 deploy_arch_dir = os.path.join(self.deploy_dir, arch)
1029 if not os.path.exists(deploy_arch_dir):
1030 continue
1031
1032 lockfilename = self.d.getVar('DEPLOY_DIR_RPM') + "/rpm.lock"
1033 lf = bb.utils.lockfile(lockfilename, False)
1034 oe.path.copyhardlinktree(deploy_arch_dir, arch_channel)
1035 bb.utils.unlockfile(lf)
1036
1037 if not arch in ch_already_added:
1038 bb.note('Adding Smart channel %s (%s)' %
1039 (arch, channel_priority))
1040 self._invoke_smart(['channel', '--add', arch, 'type=rpm-md',
1041 'baseurl=%s' % arch_channel, '-y'])
1042 self._invoke_smart(['channel', '--set', arch, 'priority=%d' %
1043 channel_priority])
1044 channel_priority -= 5
1045
1046 ch_already_added.append(arch)
1047
1048 bb.note('adding Smart RPM DB channel')
1049 self._invoke_smart(['channel', '--add', 'rpmsys', 'type=rpm-sys', '-y'])
1050
1051 # Construct install scriptlet wrapper.
1052 # Scripts need to be ordered when executed, this ensures numeric order.
1053 # If we ever run into needing more the 899 scripts, we'll have to.
1054 # change num to start with 1000.
1055 #
1056 scriptletcmd = "$2 $1/$3 $4\n"
1057 scriptpath = "$1/$3"
1058
1059 # When self.debug_level >= 3, also dump the content of the
1060 # executed scriptlets and how they get invoked. We have to
1061 # replace "exit 1" and "ERR" because printing those as-is
1062 # would trigger a log analysis failure.
1063 if self.debug_level >= 3:
1064 dump_invocation = 'echo "Executing ${name} ${kind} with: ' + scriptletcmd + '"\n'
1065 dump_script = 'cat ' + scriptpath + '| sed -e "s/exit 1/exxxit 1/g" -e "s/ERR/IRR/g"; echo\n'
1066 else:
1067 dump_invocation = 'echo "Executing ${name} ${kind}"\n'
1068 dump_script = ''
1069
1070 SCRIPTLET_FORMAT = "#!/bin/bash\n" \
1071 "\n" \
1072 "export PATH=%s\n" \
1073 "export D=%s\n" \
1074 'export OFFLINE_ROOT="$D"\n' \
1075 'export IPKG_OFFLINE_ROOT="$D"\n' \
1076 'export OPKG_OFFLINE_ROOT="$D"\n' \
1077 "export INTERCEPT_DIR=%s\n" \
1078 "export NATIVE_ROOT=%s\n" \
1079 "\n" \
1080 "name=`head -1 " + scriptpath + " | cut -d\' \' -f 2`\n" \
1081 "kind=`head -1 " + scriptpath + " | cut -d\' \' -f 4`\n" \
1082 + dump_invocation \
1083 + dump_script \
1084 + scriptletcmd + \
1085 "ret=$?\n" \
1086 "echo Result of ${name} ${kind}: ${ret}\n" \
1087 "if [ ${ret} -ne 0 ]; then\n" \
1088 " if [ $4 -eq 1 ]; then\n" \
1089 " mkdir -p $1/etc/rpm-postinsts\n" \
1090 " num=100\n" \
1091 " while [ -e $1/etc/rpm-postinsts/${num}-* ]; do num=$((num + 1)); done\n" \
1092 ' echo "#!$2" > $1/etc/rpm-postinsts/${num}-${name}\n' \
1093 ' echo "# Arg: $4" >> $1/etc/rpm-postinsts/${num}-${name}\n' \
1094 " cat " + scriptpath + " >> $1/etc/rpm-postinsts/${num}-${name}\n" \
1095 " chmod +x $1/etc/rpm-postinsts/${num}-${name}\n" \
1096 ' echo "Info: deferring ${name} ${kind} install scriptlet to first boot"\n' \
1097 " else\n" \
1098 ' echo "Error: ${name} ${kind} remove scriptlet failed"\n' \
1099 " fi\n" \
1100 "fi\n"
1101
1102 intercept_dir = self.d.expand('${WORKDIR}/intercept_scripts')
1103 native_root = self.d.getVar('STAGING_DIR_NATIVE')
1104 scriptlet_content = SCRIPTLET_FORMAT % (os.environ['PATH'],
1105 self.target_rootfs,
1106 intercept_dir,
1107 native_root)
1108 open(self.scriptlet_wrapper, 'w+').write(scriptlet_content)
1109
1110 bb.note("configuring RPM cross-install scriptlet_wrapper")
1111 os.chmod(self.scriptlet_wrapper, 0o755)
1112 cmd = ['config', '--set', 'rpm-extra-macros._cross_scriptlet_wrapper=%s' %
1113 self.scriptlet_wrapper]
1114 self._invoke_smart(cmd)
1115
1116 # Debug to show smart config info
1117 # bb.note(self._invoke_smart(['config', '--show']))
1118
1119 def update(self):
1120 self._invoke_smart(['update', 'rpmsys'])
1121
1122 def get_rdepends_recursively(self, pkgs):
1123 # pkgs will be changed during the loop, so use [:] to make a copy.
1124 for pkg in pkgs[:]:
1125 sub_data = oe.packagedata.read_subpkgdata(pkg, self.d)
1126 sub_rdep = sub_data.get("RDEPENDS_" + pkg)
1127 if not sub_rdep:
1128 continue
1129 done = list(bb.utils.explode_dep_versions2(sub_rdep).keys())
1130 next = done
1131 # Find all the rdepends on dependency chain
1132 while next:
1133 new = []
1134 for sub_pkg in next:
1135 sub_data = oe.packagedata.read_subpkgdata(sub_pkg, self.d)
1136 sub_pkg_rdep = sub_data.get("RDEPENDS_" + sub_pkg)
1137 if not sub_pkg_rdep:
1138 continue
1139 for p in bb.utils.explode_dep_versions2(sub_pkg_rdep):
1140 # Already handled, skip it.
1141 if p in done or p in pkgs:
1142 continue
1143 # It's a new dep
1144 if oe.packagedata.has_subpkgdata(p, self.d):
1145 done.append(p)
1146 new.append(p)
1147 next = new
1148 pkgs.extend(done)
1149 return pkgs
1150
1151 '''
1152 Install pkgs with smart, the pkg name is oe format
1153 '''
1154 def install(self, pkgs, attempt_only=False):
1155
1156 if not pkgs:
1157 bb.note("There are no packages to install")
1158 return
1159 bb.note("Installing the following packages: %s" % ' '.join(pkgs))
1160 if not attempt_only:
1161 # Pull in multilib requires since rpm may not pull in them
1162 # correctly, for example,
1163 # lib32-packagegroup-core-standalone-sdk-target requires
1164 # lib32-libc6, but rpm may pull in libc6 rather than lib32-libc6
1165 # since it doesn't know mlprefix (lib32-), bitbake knows it and
1166 # can handle it well, find out the RDEPENDS on the chain will
1167 # fix the problem. Both do_rootfs and do_populate_sdk have this
1168 # issue.
1169 # The attempt_only packages don't need this since they are
1170 # based on the installed ones.
1171 #
1172 # Separate pkgs into two lists, one is multilib, the other one
1173 # is non-multilib.
1174 ml_pkgs = []
1175 non_ml_pkgs = pkgs[:]
1176 for pkg in pkgs:
1177 for mlib in (self.d.getVar("MULTILIB_VARIANTS") or "").split():
1178 if pkg.startswith(mlib + '-'):
1179 ml_pkgs.append(pkg)
1180 non_ml_pkgs.remove(pkg)
1181
1182 if len(ml_pkgs) > 0 and len(non_ml_pkgs) > 0:
1183 # Found both foo and lib-foo
1184 ml_pkgs = self.get_rdepends_recursively(ml_pkgs)
1185 non_ml_pkgs = self.get_rdepends_recursively(non_ml_pkgs)
1186 # Longer list makes smart slower, so only keep the pkgs
1187 # which have the same BPN, and smart can handle others
1188 # correctly.
1189 pkgs_new = []
1190 for pkg in non_ml_pkgs:
1191 for mlib in (self.d.getVar("MULTILIB_VARIANTS") or "").split():
1192 mlib_pkg = mlib + "-" + pkg
1193 if mlib_pkg in ml_pkgs:
1194 pkgs_new.append(pkg)
1195 pkgs_new.append(mlib_pkg)
1196 for pkg in pkgs:
1197 if pkg not in pkgs_new:
1198 pkgs_new.append(pkg)
1199 pkgs = pkgs_new
1200 new_depends = {}
1201 deps = bb.utils.explode_dep_versions2(" ".join(pkgs))
1202 for depend in deps:
1203 data = oe.packagedata.read_subpkgdata(depend, self.d)
1204 key = "PKG_%s" % depend
1205 if key in data:
1206 new_depend = data[key]
1207 else:
1208 new_depend = depend
1209 new_depends[new_depend] = deps[depend]
1210 pkgs = bb.utils.join_deps(new_depends, commasep=True).split(', ')
1211 pkgs = self._pkg_translate_oe_to_smart(pkgs, attempt_only)
1212 if not pkgs:
1213 bb.note("There are no packages to install")
1214 return
1215 if not attempt_only:
1216 bb.note('to be installed: %s' % ' '.join(pkgs))
1217 cmd = [self.smart_cmd] + self.smart_opt + ["install", "-y"] + pkgs
1218 bb.note(' '.join(cmd))
1219 else:
1220 bb.note('installing attempt only packages...')
1221 bb.note('Attempting %s' % ' '.join(pkgs))
1222 cmd = [self.smart_cmd] + self.smart_opt + ["install", "--attempt",
1223 "-y"] + pkgs
1224 try:
1225 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8")
1226 bb.note(output)
1227 except subprocess.CalledProcessError as e:
1228 bb.fatal("Unable to install packages. Command '%s' "
1229 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
1230
1231 '''
1232 Remove pkgs with smart, the pkg name is smart/rpm format
1233 '''
1234 def remove(self, pkgs, with_dependencies=True):
1235 bb.note('to be removed: ' + ' '.join(pkgs))
1236
1237 if not with_dependencies:
1238 cmd = [self.rpm_cmd] + ["-e", "--nodeps", "--root=%s" %
1239 self.target_rootfs, "--dbpath=/var/lib/rpm",
1240 "--define='_cross_scriptlet_wrapper %s'" %
1241 self.scriptlet_wrapper,
1242 "--define='_tmppath /%s/tmp'" % self.install_dir_name] + pkgs
1243 else:
1244 # for pkg in pkgs:
1245 # bb.note('Debug: What required: %s' % pkg)
1246 # bb.note(self._invoke_smart(['query', pkg, '--show-requiredby']))
1247 cmd = [self.smart_cmd] + self.smart_opt + ["remove", "-y"] + pkgs
1248 try:
1249 bb.note(' '.join(cmd))
1250 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8")
1251 bb.note(output)
1252 except subprocess.CalledProcessError as e:
1253 bb.note("Unable to remove packages. Command '%s' "
1254 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
1255 590
1256 def upgrade(self): 591 def upgrade(self):
1257 bb.note('smart upgrade') 592 self._prepare_pkg_transaction()
1258 self._invoke_smart(['upgrade']) 593 self._invoke_dnf(["upgrade"])
1259
1260 def write_index(self):
1261 result = self.indexer.write_index()
1262 594
1263 if result is not None: 595 def autoremove(self):
1264 bb.fatal(result) 596 self._prepare_pkg_transaction()
597 self._invoke_dnf(["autoremove"])
1265 598
1266 def remove_packaging_data(self): 599 def remove_packaging_data(self):
1267 bb.utils.remove(self.image_rpmlib, True) 600 self._invoke_dnf(["clean", "all"])
1268 bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'), 601 for dir in self.packaging_data_dirs:
1269 True) 602 bb.utils.remove(oe.path.join(self.target_rootfs, dir), True)
1270 bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/opkg'), True)
1271
1272 # remove temp directory
1273 bb.utils.remove(self.install_dir_path, True)
1274 603
1275 def backup_packaging_data(self): 604 def backup_packaging_data(self):
1276 # Save the rpmlib for increment rpm image generation 605 # Save the packaging dirs for increment rpm image generation
1277 if os.path.exists(self.saved_rpmlib): 606 if os.path.exists(self.saved_packaging_data):
1278 bb.utils.remove(self.saved_rpmlib, True) 607 bb.utils.remove(self.saved_packaging_data, True)
1279 shutil.copytree(self.image_rpmlib, 608 for i in self.packaging_data_dirs:
1280 self.saved_rpmlib, 609 source_dir = oe.path.join(self.target_rootfs, i)
1281 symlinks=True) 610 target_dir = oe.path.join(self.saved_packaging_data, i)
611 shutil.copytree(source_dir, target_dir, symlinks=True)
1282 612
1283 def recovery_packaging_data(self): 613 def recovery_packaging_data(self):
1284 # Move the rpmlib back 614 # Move the rpmlib back
1285 if os.path.exists(self.saved_rpmlib): 615 if os.path.exists(self.saved_packaging_data):
1286 if os.path.exists(self.image_rpmlib): 616 for i in self.packaging_data_dirs:
1287 bb.utils.remove(self.image_rpmlib, True) 617 target_dir = oe.path.join(self.target_rootfs, i)
1288 618 if os.path.exists(target_dir):
1289 bb.note('Recovery packaging data') 619 bb.utils.remove(target_dir, True)
1290 shutil.copytree(self.saved_rpmlib, 620 source_dir = oe.path.join(self.saved_packaging_data, i)
1291 self.image_rpmlib, 621 shutil.copytree(source_dir,
622 target_dir,
1292 symlinks=True) 623 symlinks=True)
1293 624
1294 def list_installed(self): 625 def list_installed(self):
1295 return self.pkgs_list.list_pkgs() 626 output = self._invoke_dnf(["repoquery", "--installed", "--queryformat", "Package: %{name} %{arch} %{version} %{sourcerpm}\nDependencies:\n%{requires}\nRecommendations:\n%{recommends}\nDependenciesEndHere:\n"],
1296 627 print_output = False)
1297 ''' 628 packages = {}
1298 If incremental install, we need to determine what we've got, 629 current_package = None
1299 what we need to add, and what to remove... 630 current_deps = None
1300 The dump_install_solution will dump and save the new install 631 current_state = "initial"
1301 solution. 632 for line in output.splitlines():
1302 ''' 633 if line.startswith("Package:"):
1303 def dump_install_solution(self, pkgs): 634 package_info = line.split(" ")[1:]
1304 bb.note('creating new install solution for incremental install') 635 current_package = package_info[0]
1305 if len(pkgs) == 0: 636 package_arch = package_info[1]
1306 return 637 package_version = package_info[2]
1307 638 package_srpm = package_info[3]
1308 pkgs = self._pkg_translate_oe_to_smart(pkgs, False) 639 packages[current_package] = {"arch":package_arch, "ver":package_version, "filename":package_srpm}
1309 install_pkgs = list() 640 current_deps = []
641 elif line.startswith("Dependencies:"):
642 current_state = "dependencies"
643 elif line.startswith("Recommendations"):
644 current_state = "recommendations"
645 elif line.startswith("DependenciesEndHere:"):
646 current_state = "initial"
647 packages[current_package]["deps"] = current_deps
648 elif len(line) > 0:
649 if current_state == "dependencies":
650 current_deps.append(line)
651 elif current_state == "recommendations":
652 current_deps.append("%s [REC]" % line)
653
654 return packages
1310 655
1311 cmd = [self.smart_cmd] + self.smart_opt + ['install', '-y', '--dump'] + pkgs 656 def update(self):
657 self._invoke_dnf(["makecache"])
658
659 def _invoke_dnf(self, dnf_args, fatal = True, print_output = True ):
660 os.environ['RPM_ETCCONFIGDIR'] = self.target_rootfs
661
662 dnf_cmd = bb.utils.which(os.getenv('PATH'), "dnf-2")
663 standard_dnf_args = (["-v", "--rpmverbosity=debug"] if self.d.getVar('ROOTFS_RPM_DEBUG') else []) + ["-y",
664 "-c", oe.path.join(self.target_rootfs, "etc/dnf/dnf.conf"),
665 "--setopt=reposdir=%s" %(oe.path.join(self.target_rootfs, "etc/yum.repos.d")),
666 "--repofrompath=oe-repo,%s" % (self.rpm_repo_dir),
667 "--installroot=%s" % (self.target_rootfs),
668 "--setopt=logdir=%s" % (self.d.getVar('T'))
669 ]
670 cmd = [dnf_cmd] + standard_dnf_args + dnf_args
1312 try: 671 try:
1313 # Disable rpmsys channel for the fake install 672 output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode("utf-8")
1314 self._invoke_smart(['channel', '--disable', 'rpmsys']) 673 if print_output:
1315 674 bb.note(output)
1316 output = subprocess.check_output(cmd,stderr=subprocess.STDOUT).decode('utf-8') 675 return output
1317 f = open(self.solution_manifest, 'w')
1318 f.write(output)
1319 f.close()
1320 with open(self.solution_manifest, 'r') as manifest:
1321 for pkg in manifest.read().split('\n'):
1322 if '@' in pkg:
1323 install_pkgs.append(pkg.strip())
1324 except subprocess.CalledProcessError as e: 676 except subprocess.CalledProcessError as e:
1325 bb.note("Unable to dump install packages. Command '%s' " 677 if print_output:
1326 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8"))) 678 (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command "
1327 # Recovery rpmsys channel 679 "'%s' returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
1328 self._invoke_smart(['channel', '--enable', 'rpmsys']) 680 else:
1329 return install_pkgs 681 (bb.note, bb.fatal)[fatal]("Could not invoke dnf. Command "
682 "'%s' returned %d:" % (' '.join(cmd), e.returncode))
683 return e.output.decode("utf-8")
684
685 def dump_install_solution(self, pkgs):
686 open(self.solution_manifest, 'w').write(" ".join(pkgs))
687 return pkgs
1330 688
1331 '''
1332 If incremental install, we need to determine what we've got,
1333 what we need to add, and what to remove...
1334 The load_old_install_solution will load the previous install
1335 solution
1336 '''
1337 def load_old_install_solution(self): 689 def load_old_install_solution(self):
1338 bb.note('load old install solution for incremental install')
1339 installed_pkgs = list()
1340 if not os.path.exists(self.solution_manifest): 690 if not os.path.exists(self.solution_manifest):
1341 bb.note('old install solution not exist') 691 return []
1342 return installed_pkgs
1343
1344 with open(self.solution_manifest, 'r') as manifest:
1345 for pkg in manifest.read().split('\n'):
1346 if '@' in pkg:
1347 installed_pkgs.append(pkg.strip())
1348
1349 return installed_pkgs
1350
1351 '''
1352 Dump all available packages in feeds, it should be invoked after the
1353 newest rpm index was created
1354 '''
1355 def dump_all_available_pkgs(self):
1356 available_manifest = self.d.expand('${T}/saved/available_pkgs.txt')
1357 available_pkgs = list()
1358 cmd = [self.smart_cmd] + self.smart_opt + ['query', '--output', available_manifest]
1359 try:
1360 subprocess.check_output(cmd, stderr=subprocess.STDOUT)
1361 with open(available_manifest, 'r') as manifest:
1362 for pkg in manifest.read().split('\n'):
1363 if '@' in pkg:
1364 available_pkgs.append(pkg.strip())
1365 except subprocess.CalledProcessError as e:
1366 bb.note("Unable to list all available packages. Command '%s' "
1367 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
1368 692
1369 self.fullpkglist = available_pkgs 693 return open(self.solution_manifest, 'r').read().split()
1370 694
1371 return 695 def _script_num_prefix(self, path):
696 files = os.listdir(path)
697 numbers = set()
698 numbers.add(99)
699 for f in files:
700 numbers.add(int(f.split("-")[0]))
701 return max(numbers) + 1
1372 702
1373 def save_rpmpostinst(self, pkg): 703 def save_rpmpostinst(self, pkg):
1374 mlibs = (self.d.getVar('MULTILIB_GLOBAL_VARIANTS', False) or "").split() 704 bb.note("Saving postinstall script of %s" % (pkg))
1375 705 cmd = bb.utils.which(os.getenv('PATH'), "rpm")
1376 new_pkg = pkg 706 args = ["-q", "--root=%s" % self.target_rootfs, "--queryformat", "%{postin}", pkg]
1377 # Remove any multilib prefix from the package name
1378 for mlib in mlibs:
1379 if mlib in pkg:
1380 new_pkg = pkg.replace(mlib + '-', '')
1381 break
1382
1383 bb.note(' * postponing %s' % new_pkg)
1384 saved_dir = self.target_rootfs + self.d.expand('${sysconfdir}/rpm-postinsts/') + new_pkg
1385
1386 cmd = self.rpm_cmd + ' -q --scripts --root ' + self.target_rootfs
1387 cmd += ' --dbpath=/var/lib/rpm ' + new_pkg
1388 cmd += ' | sed -n -e "/^postinstall scriptlet (using .*):$/,/^.* scriptlet (using .*):$/ {/.*/p}"'
1389 cmd += ' | sed -e "/postinstall scriptlet (using \(.*\)):$/d"'
1390 cmd += ' -e "/^.* scriptlet (using .*):$/d" > %s' % saved_dir
1391
1392 try:
1393 bb.note(cmd)
1394 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).strip().decode("utf-8")
1395 bb.note(output)
1396 os.chmod(saved_dir, 0o755)
1397 except subprocess.CalledProcessError as e:
1398 bb.fatal("Invoke save_rpmpostinst failed. Command '%s' "
1399 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
1400
1401 '''Write common configuration for target usage'''
1402 def rpm_setup_smart_target_config(self):
1403 bb.utils.remove(os.path.join(self.target_rootfs, 'var/lib/smart'),
1404 True)
1405
1406 self._invoke_smart(['config', '--set', 'rpm-nolinktos=1'])
1407 self._invoke_smart(['config', '--set', 'rpm-noparentdirs=1'])
1408 for i in self.d.getVar('BAD_RECOMMENDATIONS').split():
1409 self._invoke_smart(['flag', '--set', 'ignore-recommends', i])
1410 self._invoke_smart(['channel', '--add', 'rpmsys', 'type=rpm-sys', '-y'])
1411
1412 '''
1413 The rpm db lock files were produced after invoking rpm to query on
1414 build system, and they caused the rpm on target didn't work, so we
1415 need to unlock the rpm db by removing the lock files.
1416 '''
1417 def unlock_rpm_db(self):
1418 # Remove rpm db lock files
1419 rpm_db_locks = glob.glob('%s/var/lib/rpm/__db.*' % self.target_rootfs)
1420 for f in rpm_db_locks:
1421 bb.utils.remove(f, True)
1422 707
1423 """
1424 Returns a dictionary with the package info.
1425 """
1426 def package_info(self, pkg):
1427 cmd = [self.smart_cmd] + self.smart_opt + ['info', '--urls', pkg]
1428 try: 708 try:
1429 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT).decode("utf-8") 709 output = subprocess.check_output([cmd] + args,stderr=subprocess.STDOUT).decode("utf-8")
1430 except subprocess.CalledProcessError as e: 710 except subprocess.CalledProcessError as e:
1431 bb.fatal("Unable to list available packages. Command '%s' " 711 bb.fatal("Could not invoke rpm. Command "
1432 "returned %d:\n%s" % (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 712 "'%s' returned %d:\n%s" % (' '.join([cmd] + args), e.returncode, e.output.decode("utf-8")))
1433 713
1434 # Set default values to avoid UnboundLocalError 714 # may need to prepend #!/bin/sh to output
1435 arch = ""
1436 ver = ""
1437 filename = ""
1438 715
1439 #Parse output 716 target_path = oe.path.join(self.target_rootfs, self.d.expand('${sysconfdir}/rpm-postinsts/'))
1440 for line in output.splitlines(): 717 bb.utils.mkdirhier(target_path)
1441 line = line.rstrip() 718 num = self._script_num_prefix(target_path)
1442 if line.startswith("Name:"): 719 saved_script_name = oe.path.join(target_path, "%d-%s" % (num, pkg))
1443 pkg = line.split(": ")[1] 720 open(saved_script_name, 'w').write(output)
1444 elif line.startswith("Version:"): 721 os.chmod(saved_script_name, 0o755)
1445 tmp_str = line.split(": ")[1]
1446 ver, arch = tmp_str.split("@")
1447 break
1448
1449 # Get filename
1450 index = re.search("^URLs", output, re.MULTILINE)
1451 tmp_str = output[index.end():]
1452 for line in tmp_str.splitlines():
1453 if "/" in line:
1454 line = line.lstrip()
1455 filename = line.split(" ")[0]
1456 break
1457
1458 # To have the same data type than other package_info methods
1459 filepath = os.path.join(self.deploy_dir, arch, filename)
1460 pkg_dict = {}
1461 pkg_dict[pkg] = {"arch":arch, "ver":ver, "filename":filename,
1462 "filepath": filepath}
1463
1464 return pkg_dict
1465
1466 """
1467 Returns the path to a tmpdir where resides the contents of a package.
1468
1469 Deleting the tmpdir is responsability of the caller.
1470 722
1471 """
1472 def extract(self, pkg): 723 def extract(self, pkg):
1473 pkg_info = self.package_info(pkg) 724 output = self._invoke_dnf(["repoquery", "--queryformat", "%{location}", pkg])
1474 if not pkg_info: 725 pkg_name = output.splitlines()[-1]
1475 bb.fatal("Unable to get information for package '%s' while " 726 if not pkg_name.endswith(".rpm"):
1476 "trying to extract the package." % pkg) 727 bb.fatal("dnf could not find package %s in repository: %s" %(pkg, output))
1477 728 pkg_path = oe.path.join(self.rpm_repo_dir, pkg_name)
1478 pkg_path = pkg_info[pkg]["filepath"]
1479 729
1480 cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio") 730 cpio_cmd = bb.utils.which(os.getenv("PATH"), "cpio")
1481 rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio") 731 rpm2cpio_cmd = bb.utils.which(os.getenv("PATH"), "rpm2cpio")
@@ -1725,7 +975,7 @@ class OpkgPM(OpkgDpkgPM):
1725 for uri in feed_uris: 975 for uri in feed_uris:
1726 if archs: 976 if archs:
1727 for arch in archs: 977 for arch in archs:
1728 if (feed_archs is None) and (not os.path.exists(os.path.join(self.deploy_dir, arch))): 978 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))):
1729 continue 979 continue
1730 bb.note('Adding opkg feed url-%s-%d (%s)' % 980 bb.note('Adding opkg feed url-%s-%d (%s)' %
1731 (arch, uri_iterator, uri)) 981 (arch, uri_iterator, uri))
diff --git a/meta/lib/oe/rootfs.py b/meta/lib/oe/rootfs.py
index 74c643b3b9..5e1c09762a 100644
--- a/meta/lib/oe/rootfs.py
+++ b/meta/lib/oe/rootfs.py
@@ -431,6 +431,8 @@ class RpmRootfs(Rootfs):
431 bb.note('incremental removed: %s' % ' '.join(pkg_to_remove)) 431 bb.note('incremental removed: %s' % ' '.join(pkg_to_remove))
432 self.pm.remove(pkg_to_remove) 432 self.pm.remove(pkg_to_remove)
433 433
434 self.pm.autoremove()
435
434 def _create(self): 436 def _create(self):
435 pkgs_to_install = self.manifest.parse_initial_manifest() 437 pkgs_to_install = self.manifest.parse_initial_manifest()
436 rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS') 438 rpm_pre_process_cmds = self.d.getVar('RPM_PREPROCESS_COMMANDS')
@@ -444,8 +446,6 @@ class RpmRootfs(Rootfs):
444 if self.progress_reporter: 446 if self.progress_reporter:
445 self.progress_reporter.next_stage() 447 self.progress_reporter.next_stage()
446 448
447 self.pm.dump_all_available_pkgs()
448
449 if self.inc_rpm_image_gen == "1": 449 if self.inc_rpm_image_gen == "1":
450 self._create_incremental(pkgs_to_install) 450 self._create_incremental(pkgs_to_install)
451 451
@@ -480,15 +480,13 @@ class RpmRootfs(Rootfs):
480 if self.progress_reporter: 480 if self.progress_reporter:
481 self.progress_reporter.next_stage() 481 self.progress_reporter.next_stage()
482 482
483 self._setup_dbg_rootfs(['/etc/rpm', '/var/lib/rpm', '/var/lib/smart']) 483 self._setup_dbg_rootfs(['/etc', '/var/lib/rpm', '/var/cache/dnf', '/var/lib/dnf'])
484 484
485 execute_pre_post_process(self.d, rpm_post_process_cmds) 485 execute_pre_post_process(self.d, rpm_post_process_cmds)
486 486
487 if self.inc_rpm_image_gen == "1": 487 if self.inc_rpm_image_gen == "1":
488 self.pm.backup_packaging_data() 488 self.pm.backup_packaging_data()
489 489
490 self.pm.rpm_setup_smart_target_config()
491
492 if self.progress_reporter: 490 if self.progress_reporter:
493 self.progress_reporter.next_stage() 491 self.progress_reporter.next_stage()
494 492
@@ -526,15 +524,7 @@ class RpmRootfs(Rootfs):
526 self.pm.save_rpmpostinst(pkg) 524 self.pm.save_rpmpostinst(pkg)
527 525
528 def _cleanup(self): 526 def _cleanup(self):
529 # during the execution of postprocess commands, rpm is called several 527 pass
530 # times to get the files installed, dependencies, etc. This creates the
531 # __db.00* (Berkeley DB files that hold locks, rpm specific environment
532 # settings, etc.), that should not get into the final rootfs
533 self.pm.unlock_rpm_db()
534 if os.path.isdir(self.pm.install_dir_path + "/tmp") and not os.listdir(self.pm.install_dir_path + "/tmp"):
535 bb.utils.remove(self.pm.install_dir_path + "/tmp", True)
536 if os.path.isdir(self.pm.install_dir_path) and not os.listdir(self.pm.install_dir_path):
537 bb.utils.remove(self.pm.install_dir_path, True)
538 528
539class DpkgOpkgRootfs(Rootfs): 529class DpkgOpkgRootfs(Rootfs):
540 def __init__(self, d, progress_reporter=None, logcatcher=None): 530 def __init__(self, d, progress_reporter=None, logcatcher=None):
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index fef02d0777..deb823b6ec 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -130,7 +130,6 @@ class RpmSdk(Sdk):
130 130
131 pm.create_configs() 131 pm.create_configs()
132 pm.write_index() 132 pm.write_index()
133 pm.dump_all_available_pkgs()
134 pm.update() 133 pm.update()
135 134
136 pkgs = [] 135 pkgs = []
@@ -188,7 +187,9 @@ class RpmSdk(Sdk):
188 True).strip('/'), 187 True).strip('/'),
189 ) 188 )
190 self.mkdirhier(native_sysconf_dir) 189 self.mkdirhier(native_sysconf_dir)
191 for f in glob.glob(os.path.join(self.sdk_output, "etc", "*")): 190 for f in glob.glob(os.path.join(self.sdk_output, "etc", "rpm*")):
191 self.movefile(f, native_sysconf_dir)
192 for f in glob.glob(os.path.join(self.sdk_output, "etc", "dnf", "*")):
192 self.movefile(f, native_sysconf_dir) 193 self.movefile(f, native_sysconf_dir)
193 self.remove(os.path.join(self.sdk_output, "etc"), True) 194 self.remove(os.path.join(self.sdk_output, "etc"), True)
194 195
@@ -350,7 +351,7 @@ def sdk_list_installed_packages(d, target, rootfs_dir=None):
350 if img_type == "rpm": 351 if img_type == "rpm":
351 arch_var = ["SDK_PACKAGE_ARCHS", None][target is True] 352 arch_var = ["SDK_PACKAGE_ARCHS", None][target is True]
352 os_var = ["SDK_OS", None][target is True] 353 os_var = ["SDK_OS", None][target is True]
353 return RpmPkgsList(d, rootfs_dir, arch_var, os_var).list_pkgs() 354 return RpmPkgsList(d, rootfs_dir).list_pkgs()
354 elif img_type == "ipk": 355 elif img_type == "ipk":
355 conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True] 356 conf_file_var = ["IPKGCONF_SDK", "IPKGCONF_TARGET"][target is True]
356 return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs() 357 return OpkgPkgsList(d, rootfs_dir, d.getVar(conf_file_var)).list_pkgs()