summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe/package_manager
diff options
context:
space:
mode:
authorFredrik Gustafsson <fredrik.gustafsson@axis.com>2020-07-24 16:42:40 +0200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2020-07-27 19:58:10 +0100
commit5bc67f55028407de78ac09f97f9a47b165ae8760 (patch)
treec92c0c02ea9467495ec15774bd345953cf206dbd /meta/lib/oe/package_manager
parentc5bea36b7275f1692f5847af84d320b674a4c62e (diff)
downloadpoky-5bc67f55028407de78ac09f97f9a47b165ae8760.tar.gz
ipk: Move package manager to its own dir
This is part of a refactor that will split the package manager code so that it's possible to use other package managers in other layers. RP: Fixes to parse/build (From OE-Core rev: 3ef5a3c885e1010cddfe7eba1cd3728f15270d78) Signed-off-by: Fredrik Gustafsson <fredrigu@axis.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe/package_manager')
-rw-r--r--meta/lib/oe/package_manager/__init__.py433
-rw-r--r--meta/lib/oe/package_manager/ipk/__init__.py501
-rw-r--r--meta/lib/oe/package_manager/ipk/rootfs.py2
-rw-r--r--meta/lib/oe/package_manager/ipk/sdk.py2
4 files changed, 504 insertions, 434 deletions
diff --git a/meta/lib/oe/package_manager/__init__.py b/meta/lib/oe/package_manager/__init__.py
index 36606d8cb3..4deb12837a 100644
--- a/meta/lib/oe/package_manager/__init__.py
+++ b/meta/lib/oe/package_manager/__init__.py
@@ -149,59 +149,6 @@ class Indexer(object, metaclass=ABCMeta):
149 def write_index(self): 149 def write_index(self):
150 pass 150 pass
151 151
152class OpkgIndexer(Indexer):
153 def write_index(self):
154 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
155 "SDK_PACKAGE_ARCHS",
156 ]
157
158 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index")
159 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
160 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
161 else:
162 signer = None
163
164 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
165 open(os.path.join(self.deploy_dir, "Packages"), "w").close()
166
167 index_cmds = set()
168 index_sign_files = set()
169 for arch_var in arch_vars:
170 archs = self.d.getVar(arch_var)
171 if archs is None:
172 continue
173
174 for arch in archs.split():
175 pkgs_dir = os.path.join(self.deploy_dir, arch)
176 pkgs_file = os.path.join(pkgs_dir, "Packages")
177
178 if not os.path.isdir(pkgs_dir):
179 continue
180
181 if not os.path.exists(pkgs_file):
182 open(pkgs_file, "w").close()
183
184 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' %
185 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
186
187 index_sign_files.add(pkgs_file)
188
189 if len(index_cmds) == 0:
190 bb.note("There are no packages in %s!" % self.deploy_dir)
191 return
192
193 oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
194
195 if signer:
196 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
197 is_ascii_sig = (feed_sig_type.upper() != "BIN")
198 for f in index_sign_files:
199 signer.detach_sign(f,
200 self.d.getVar('PACKAGE_FEED_GPG_NAME'),
201 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
202 armor=is_ascii_sig)
203
204
205class DpkgIndexer(Indexer): 152class DpkgIndexer(Indexer):
206 def _create_configs(self): 153 def _create_configs(self):
207 bb.utils.mkdirhier(self.apt_conf_dir) 154 bb.utils.mkdirhier(self.apt_conf_dir)
@@ -286,33 +233,6 @@ class PkgsList(object, metaclass=ABCMeta):
286 def list_pkgs(self): 233 def list_pkgs(self):
287 pass 234 pass
288 235
289class OpkgPkgsList(PkgsList):
290 def __init__(self, d, rootfs_dir, config_file):
291 super(OpkgPkgsList, self).__init__(d, rootfs_dir)
292
293 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
294 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
295 self.opkg_args += self.d.getVar("OPKG_ARGS")
296
297 def list_pkgs(self, format=None):
298 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args)
299
300 # opkg returns success even when it printed some
301 # "Collected errors:" report to stderr. Mixing stderr into
302 # stdout then leads to random failures later on when
303 # parsing the output. To avoid this we need to collect both
304 # output streams separately and check for empty stderr.
305 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
306 cmd_output, cmd_stderr = p.communicate()
307 cmd_output = cmd_output.decode("utf-8")
308 cmd_stderr = cmd_stderr.decode("utf-8")
309 if p.returncode or cmd_stderr:
310 bb.fatal("Cannot get the installed packages list. Command '%s' "
311 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
312
313 return opkg_query(cmd_output)
314
315
316class DpkgPkgsList(PkgsList): 236class DpkgPkgsList(PkgsList):
317 237
318 def list_pkgs(self): 238 def list_pkgs(self):
@@ -763,358 +683,6 @@ class OpkgDpkgPM(PackageManager):
763 def _handle_intercept_failure(self, registered_pkgs): 683 def _handle_intercept_failure(self, registered_pkgs):
764 self.mark_packages("unpacked", registered_pkgs.split()) 684 self.mark_packages("unpacked", registered_pkgs.split())
765 685
766class OpkgPM(OpkgDpkgPM):
767 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True):
768 super(OpkgPM, self).__init__(d, target_rootfs)
769
770 self.config_file = config_file
771 self.pkg_archs = archs
772 self.task_name = task_name
773
774 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
775 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
776 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
777 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs)
778 self.opkg_args += self.d.getVar("OPKG_ARGS")
779
780 if prepare_index:
781 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
782
783 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
784 if opkg_lib_dir[0] == "/":
785 opkg_lib_dir = opkg_lib_dir[1:]
786
787 self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
788
789 bb.utils.mkdirhier(self.opkg_dir)
790
791 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
792 if not os.path.exists(self.d.expand('${T}/saved')):
793 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
794
795 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
796 if self.from_feeds:
797 self._create_custom_config()
798 else:
799 self._create_config()
800
801 self.indexer = OpkgIndexer(self.d, self.deploy_dir)
802
803 def mark_packages(self, status_tag, packages=None):
804 """
805 This function will change a package's status in /var/lib/opkg/status file.
806 If 'packages' is None then the new_status will be applied to all
807 packages
808 """
809 status_file = os.path.join(self.opkg_dir, "status")
810
811 with open(status_file, "r") as sf:
812 with open(status_file + ".tmp", "w+") as tmp_sf:
813 if packages is None:
814 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
815 r"Package: \1\n\2Status: \3%s" % status_tag,
816 sf.read()))
817 else:
818 if type(packages).__name__ != "list":
819 raise TypeError("'packages' should be a list object")
820
821 status = sf.read()
822 for pkg in packages:
823 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
824 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
825 status)
826
827 tmp_sf.write(status)
828
829 os.rename(status_file + ".tmp", status_file)
830
831 def _create_custom_config(self):
832 bb.note("Building from feeds activated!")
833
834 with open(self.config_file, "w+") as config_file:
835 priority = 1
836 for arch in self.pkg_archs.split():
837 config_file.write("arch %s %d\n" % (arch, priority))
838 priority += 5
839
840 for line in (self.d.getVar('IPK_FEED_URIS') or "").split():
841 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line)
842
843 if feed_match is not None:
844 feed_name = feed_match.group(1)
845 feed_uri = feed_match.group(2)
846
847 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri))
848
849 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri))
850
851 """
852 Allow to use package deploy directory contents as quick devel-testing
853 feed. This creates individual feed configs for each arch subdir of those
854 specified as compatible for the current machine.
855 NOTE: Development-helper feature, NOT a full-fledged feed.
856 """
857 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "":
858 for arch in self.pkg_archs.split():
859 cfg_file_name = os.path.join(self.target_rootfs,
860 self.d.getVar("sysconfdir"),
861 "opkg",
862 "local-%s-feed.conf" % arch)
863
864 with open(cfg_file_name, "w+") as cfg_file:
865 cfg_file.write("src/gz local-%s %s/%s" %
866 (arch,
867 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'),
868 arch))
869
870 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
871 # There is no command line option for this anymore, we need to add
872 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
873 # the default value of "/var/lib" as defined in opkg:
874 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
875 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
876 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
877 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
878 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
879 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
880
881
882 def _create_config(self):
883 with open(self.config_file, "w+") as config_file:
884 priority = 1
885 for arch in self.pkg_archs.split():
886 config_file.write("arch %s %d\n" % (arch, priority))
887 priority += 5
888
889 config_file.write("src oe file:%s\n" % self.deploy_dir)
890
891 for arch in self.pkg_archs.split():
892 pkgs_dir = os.path.join(self.deploy_dir, arch)
893 if os.path.isdir(pkgs_dir):
894 config_file.write("src oe-%s file:%s\n" %
895 (arch, pkgs_dir))
896
897 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
898 # There is no command line option for this anymore, we need to add
899 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
900 # the default value of "/var/lib" as defined in opkg:
901 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
902 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
903 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
904 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
905 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
906 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
907
908 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
909 if feed_uris == "":
910 return
911
912 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
913 % self.target_rootfs)
914
915 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True)
916
917 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
918 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split()
919
920 with open(rootfs_config, "w+") as config_file:
921 uri_iterator = 0
922 for uri in feed_uris:
923 if archs:
924 for arch in archs:
925 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))):
926 continue
927 bb.note('Adding opkg feed url-%s-%d (%s)' %
928 (arch, uri_iterator, uri))
929 config_file.write("src/gz uri-%s-%d %s/%s\n" %
930 (arch, uri_iterator, uri, arch))
931 else:
932 bb.note('Adding opkg feed url-%d (%s)' %
933 (uri_iterator, uri))
934 config_file.write("src/gz uri-%d %s\n" %
935 (uri_iterator, uri))
936
937 uri_iterator += 1
938
939 def update(self):
940 self.deploy_dir_lock()
941
942 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args)
943
944 try:
945 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
946 except subprocess.CalledProcessError as e:
947 self.deploy_dir_unlock()
948 bb.fatal("Unable to update the package index files. Command '%s' "
949 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
950
951 self.deploy_dir_unlock()
952
953 def install(self, pkgs, attempt_only=False):
954 if not pkgs:
955 return
956
957 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args)
958 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split():
959 cmd += " --add-exclude %s" % exclude
960 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split():
961 cmd += " --add-ignore-recommends %s" % bad_recommendation
962 cmd += " install "
963 cmd += " ".join(pkgs)
964
965 os.environ['D'] = self.target_rootfs
966 os.environ['OFFLINE_ROOT'] = self.target_rootfs
967 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
968 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
969 os.environ['INTERCEPT_DIR'] = self.intercepts_dir
970 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
971
972 try:
973 bb.note("Installing the following packages: %s" % ' '.join(pkgs))
974 bb.note(cmd)
975 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
976 bb.note(output)
977 failed_pkgs = []
978 for line in output.split('\n'):
979 if line.endswith("configuration required on target."):
980 bb.warn(line)
981 failed_pkgs.append(line.split(".")[0])
982 if failed_pkgs:
983 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
984 except subprocess.CalledProcessError as e:
985 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. "
986 "Command '%s' returned %d:\n%s" %
987 (cmd, e.returncode, e.output.decode("utf-8")))
988
989 def remove(self, pkgs, with_dependencies=True):
990 if not pkgs:
991 return
992
993 if with_dependencies:
994 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \
995 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
996 else:
997 cmd = "%s %s --force-depends remove %s" % \
998 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
999
1000 try:
1001 bb.note(cmd)
1002 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
1003 bb.note(output)
1004 except subprocess.CalledProcessError as e:
1005 bb.fatal("Unable to remove packages. Command '%s' "
1006 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
1007
1008 def write_index(self):
1009 self.deploy_dir_lock()
1010
1011 result = self.indexer.write_index()
1012
1013 self.deploy_dir_unlock()
1014
1015 if result is not None:
1016 bb.fatal(result)
1017
1018 def remove_packaging_data(self):
1019 bb.utils.remove(self.opkg_dir, True)
1020 # create the directory back, it's needed by PM lock
1021 bb.utils.mkdirhier(self.opkg_dir)
1022
1023 def remove_lists(self):
1024 if not self.from_feeds:
1025 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True)
1026
1027 def list_installed(self):
1028 return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs()
1029
1030 def dummy_install(self, pkgs):
1031 """
1032 The following function dummy installs pkgs and returns the log of output.
1033 """
1034 if len(pkgs) == 0:
1035 return
1036
1037 # Create an temp dir as opkg root for dummy installation
1038 temp_rootfs = self.d.expand('${T}/opkg')
1039 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
1040 if opkg_lib_dir[0] == "/":
1041 opkg_lib_dir = opkg_lib_dir[1:]
1042 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg')
1043 bb.utils.mkdirhier(temp_opkg_dir)
1044
1045 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
1046 opkg_args += self.d.getVar("OPKG_ARGS")
1047
1048 cmd = "%s %s update" % (self.opkg_cmd, opkg_args)
1049 try:
1050 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
1051 except subprocess.CalledProcessError as e:
1052 bb.fatal("Unable to update. Command '%s' "
1053 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
1054
1055 # Dummy installation
1056 cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
1057 opkg_args,
1058 ' '.join(pkgs))
1059 try:
1060 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
1061 except subprocess.CalledProcessError as e:
1062 bb.fatal("Unable to dummy install packages. Command '%s' "
1063 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
1064
1065 bb.utils.remove(temp_rootfs, True)
1066
1067 return output
1068
1069 def backup_packaging_data(self):
1070 # Save the opkglib for increment ipk image generation
1071 if os.path.exists(self.saved_opkg_dir):
1072 bb.utils.remove(self.saved_opkg_dir, True)
1073 shutil.copytree(self.opkg_dir,
1074 self.saved_opkg_dir,
1075 symlinks=True)
1076
1077 def recover_packaging_data(self):
1078 # Move the opkglib back
1079 if os.path.exists(self.saved_opkg_dir):
1080 if os.path.exists(self.opkg_dir):
1081 bb.utils.remove(self.opkg_dir, True)
1082
1083 bb.note('Recover packaging data')
1084 shutil.copytree(self.saved_opkg_dir,
1085 self.opkg_dir,
1086 symlinks=True)
1087
1088 def package_info(self, pkg):
1089 """
1090 Returns a dictionary with the package info.
1091 """
1092 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
1093 pkg_info = super(OpkgPM, self).package_info(pkg, cmd)
1094
1095 pkg_arch = pkg_info[pkg]["arch"]
1096 pkg_filename = pkg_info[pkg]["filename"]
1097 pkg_info[pkg]["filepath"] = \
1098 os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
1099
1100 return pkg_info
1101
1102 def extract(self, pkg):
1103 """
1104 Returns the path to a tmpdir where resides the contents of a package.
1105
1106 Deleting the tmpdir is responsability of the caller.
1107 """
1108 pkg_info = self.package_info(pkg)
1109 if not pkg_info:
1110 bb.fatal("Unable to get information for package '%s' while "
1111 "trying to extract the package." % pkg)
1112
1113 tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
1114 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
1115
1116 return tmp_dir
1117
1118class DpkgPM(OpkgDpkgPM): 686class DpkgPM(OpkgDpkgPM):
1119 def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True): 687 def __init__(self, d, target_rootfs, archs, base_archs, apt_conf_dir=None, deb_repo_workdir="oe-rootfs-repo", filterbydependencies=True):
1120 super(DpkgPM, self).__init__(d, target_rootfs) 688 super(DpkgPM, self).__init__(d, target_rootfs)
@@ -1445,6 +1013,7 @@ class DpkgPM(OpkgDpkgPM):
1445 1013
1446def generate_index_files(d): 1014def generate_index_files(d):
1447 from oe.package_manager.rpm import RpmSubdirIndexer 1015 from oe.package_manager.rpm import RpmSubdirIndexer
1016 from oe.package_manager.ipk import OpkgIndexer
1448 1017
1449 classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split() 1018 classes = d.getVar('PACKAGE_CLASSES').replace("package_", "").split()
1450 1019
diff --git a/meta/lib/oe/package_manager/ipk/__init__.py b/meta/lib/oe/package_manager/ipk/__init__.py
index a2094304c9..34500b8a0a 100644
--- a/meta/lib/oe/package_manager/ipk/__init__.py
+++ b/meta/lib/oe/package_manager/ipk/__init__.py
@@ -1,3 +1,504 @@
1# 1#
2# SPDX-License-Identifier: GPL-2.0-only 2# SPDX-License-Identifier: GPL-2.0-only
3# 3#
4
5from oe.package_manager import *
6
7class OpkgIndexer(Indexer):
8 def write_index(self):
9 arch_vars = ["ALL_MULTILIB_PACKAGE_ARCHS",
10 "SDK_PACKAGE_ARCHS",
11 ]
12
13 opkg_index_cmd = bb.utils.which(os.getenv('PATH'), "opkg-make-index")
14 if self.d.getVar('PACKAGE_FEED_SIGN') == '1':
15 signer = get_signer(self.d, self.d.getVar('PACKAGE_FEED_GPG_BACKEND'))
16 else:
17 signer = None
18
19 if not os.path.exists(os.path.join(self.deploy_dir, "Packages")):
20 open(os.path.join(self.deploy_dir, "Packages"), "w").close()
21
22 index_cmds = set()
23 index_sign_files = set()
24 for arch_var in arch_vars:
25 archs = self.d.getVar(arch_var)
26 if archs is None:
27 continue
28
29 for arch in archs.split():
30 pkgs_dir = os.path.join(self.deploy_dir, arch)
31 pkgs_file = os.path.join(pkgs_dir, "Packages")
32
33 if not os.path.isdir(pkgs_dir):
34 continue
35
36 if not os.path.exists(pkgs_file):
37 open(pkgs_file, "w").close()
38
39 index_cmds.add('%s --checksum md5 --checksum sha256 -r %s -p %s -m %s' %
40 (opkg_index_cmd, pkgs_file, pkgs_file, pkgs_dir))
41
42 index_sign_files.add(pkgs_file)
43
44 if len(index_cmds) == 0:
45 bb.note("There are no packages in %s!" % self.deploy_dir)
46 return
47
48 oe.utils.multiprocess_launch(create_index, index_cmds, self.d)
49
50 if signer:
51 feed_sig_type = self.d.getVar('PACKAGE_FEED_GPG_SIGNATURE_TYPE')
52 is_ascii_sig = (feed_sig_type.upper() != "BIN")
53 for f in index_sign_files:
54 signer.detach_sign(f,
55 self.d.getVar('PACKAGE_FEED_GPG_NAME'),
56 self.d.getVar('PACKAGE_FEED_GPG_PASSPHRASE_FILE'),
57 armor=is_ascii_sig)
58
59class OpkgPkgsList(PkgsList):
60 def __init__(self, d, rootfs_dir, config_file):
61 super(OpkgPkgsList, self).__init__(d, rootfs_dir)
62
63 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
64 self.opkg_args = "-f %s -o %s " % (config_file, rootfs_dir)
65 self.opkg_args += self.d.getVar("OPKG_ARGS")
66
67 def list_pkgs(self, format=None):
68 cmd = "%s %s status" % (self.opkg_cmd, self.opkg_args)
69
70 # opkg returns success even when it printed some
71 # "Collected errors:" report to stderr. Mixing stderr into
72 # stdout then leads to random failures later on when
73 # parsing the output. To avoid this we need to collect both
74 # output streams separately and check for empty stderr.
75 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
76 cmd_output, cmd_stderr = p.communicate()
77 cmd_output = cmd_output.decode("utf-8")
78 cmd_stderr = cmd_stderr.decode("utf-8")
79 if p.returncode or cmd_stderr:
80 bb.fatal("Cannot get the installed packages list. Command '%s' "
81 "returned %d and stderr:\n%s" % (cmd, p.returncode, cmd_stderr))
82
83 return opkg_query(cmd_output)
84
85
86
87class OpkgDpkgPM(PackageManager):
88 def __init__(self, d, target_rootfs):
89 """
90 This is an abstract class. Do not instantiate this directly.
91 """
92 super(OpkgDpkgPM, self).__init__(d, target_rootfs)
93
94 def package_info(self, pkg, cmd):
95 """
96 Returns a dictionary with the package info.
97
98 This method extracts the common parts for Opkg and Dpkg
99 """
100
101 try:
102 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True).decode("utf-8")
103 except subprocess.CalledProcessError as e:
104 bb.fatal("Unable to list available packages. Command '%s' "
105 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
106 return opkg_query(output)
107
108 def extract(self, pkg, pkg_info):
109 """
110 Returns the path to a tmpdir where resides the contents of a package.
111
112 Deleting the tmpdir is responsability of the caller.
113
114 This method extracts the common parts for Opkg and Dpkg
115 """
116
117 ar_cmd = bb.utils.which(os.getenv("PATH"), "ar")
118 tar_cmd = bb.utils.which(os.getenv("PATH"), "tar")
119 pkg_path = pkg_info[pkg]["filepath"]
120
121 if not os.path.isfile(pkg_path):
122 bb.fatal("Unable to extract package for '%s'."
123 "File %s doesn't exists" % (pkg, pkg_path))
124
125 tmp_dir = tempfile.mkdtemp()
126 current_dir = os.getcwd()
127 os.chdir(tmp_dir)
128 data_tar = 'data.tar.xz'
129
130 try:
131 cmd = [ar_cmd, 'x', pkg_path]
132 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
133 cmd = [tar_cmd, 'xf', data_tar]
134 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
135 except subprocess.CalledProcessError as e:
136 bb.utils.remove(tmp_dir, recurse=True)
137 bb.fatal("Unable to extract %s package. Command '%s' "
138 "returned %d:\n%s" % (pkg_path, ' '.join(cmd), e.returncode, e.output.decode("utf-8")))
139 except OSError as e:
140 bb.utils.remove(tmp_dir, recurse=True)
141 bb.fatal("Unable to extract %s package. Command '%s' "
142 "returned %d:\n%s at %s" % (pkg_path, ' '.join(cmd), e.errno, e.strerror, e.filename))
143
144 bb.note("Extracted %s to %s" % (pkg_path, tmp_dir))
145 bb.utils.remove(os.path.join(tmp_dir, "debian-binary"))
146 bb.utils.remove(os.path.join(tmp_dir, "control.tar.gz"))
147 os.chdir(current_dir)
148
149 return tmp_dir
150
151 def _handle_intercept_failure(self, registered_pkgs):
152 self.mark_packages("unpacked", registered_pkgs.split())
153
154class OpkgPM(OpkgDpkgPM):
155 def __init__(self, d, target_rootfs, config_file, archs, task_name='target', ipk_repo_workdir="oe-rootfs-repo", filterbydependencies=True, prepare_index=True):
156 super(OpkgPM, self).__init__(d, target_rootfs)
157
158 self.config_file = config_file
159 self.pkg_archs = archs
160 self.task_name = task_name
161
162 self.deploy_dir = oe.path.join(self.d.getVar('WORKDIR'), ipk_repo_workdir)
163 self.deploy_lock_file = os.path.join(self.deploy_dir, "deploy.lock")
164 self.opkg_cmd = bb.utils.which(os.getenv('PATH'), "opkg")
165 self.opkg_args = "--volatile-cache -f %s -t %s -o %s " % (self.config_file, self.d.expand('${T}/ipktemp/') ,target_rootfs)
166 self.opkg_args += self.d.getVar("OPKG_ARGS")
167
168 if prepare_index:
169 create_packages_dir(self.d, self.deploy_dir, d.getVar("DEPLOY_DIR_IPK"), "package_write_ipk", filterbydependencies)
170
171 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
172 if opkg_lib_dir[0] == "/":
173 opkg_lib_dir = opkg_lib_dir[1:]
174
175 self.opkg_dir = os.path.join(target_rootfs, opkg_lib_dir, "opkg")
176
177 bb.utils.mkdirhier(self.opkg_dir)
178
179 self.saved_opkg_dir = self.d.expand('${T}/saved/%s' % self.task_name)
180 if not os.path.exists(self.d.expand('${T}/saved')):
181 bb.utils.mkdirhier(self.d.expand('${T}/saved'))
182
183 self.from_feeds = (self.d.getVar('BUILD_IMAGES_FROM_FEEDS') or "") == "1"
184 if self.from_feeds:
185 self._create_custom_config()
186 else:
187 self._create_config()
188
189 self.indexer = OpkgIndexer(self.d, self.deploy_dir)
190
191 def mark_packages(self, status_tag, packages=None):
192 """
193 This function will change a package's status in /var/lib/opkg/status file.
194 If 'packages' is None then the new_status will be applied to all
195 packages
196 """
197 status_file = os.path.join(self.opkg_dir, "status")
198
199 with open(status_file, "r") as sf:
200 with open(status_file + ".tmp", "w+") as tmp_sf:
201 if packages is None:
202 tmp_sf.write(re.sub(r"Package: (.*?)\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)",
203 r"Package: \1\n\2Status: \3%s" % status_tag,
204 sf.read()))
205 else:
206 if type(packages).__name__ != "list":
207 raise TypeError("'packages' should be a list object")
208
209 status = sf.read()
210 for pkg in packages:
211 status = re.sub(r"Package: %s\n((?:[^\n]+\n)*?)Status: (.*)(?:unpacked|installed)" % pkg,
212 r"Package: %s\n\1Status: \2%s" % (pkg, status_tag),
213 status)
214
215 tmp_sf.write(status)
216
217 os.rename(status_file + ".tmp", status_file)
218
219 def _create_custom_config(self):
220 bb.note("Building from feeds activated!")
221
222 with open(self.config_file, "w+") as config_file:
223 priority = 1
224 for arch in self.pkg_archs.split():
225 config_file.write("arch %s %d\n" % (arch, priority))
226 priority += 5
227
228 for line in (self.d.getVar('IPK_FEED_URIS') or "").split():
229 feed_match = re.match(r"^[ \t]*(.*)##([^ \t]*)[ \t]*$", line)
230
231 if feed_match is not None:
232 feed_name = feed_match.group(1)
233 feed_uri = feed_match.group(2)
234
235 bb.note("Add %s feed with URL %s" % (feed_name, feed_uri))
236
237 config_file.write("src/gz %s %s\n" % (feed_name, feed_uri))
238
239 """
240 Allow to use package deploy directory contents as quick devel-testing
241 feed. This creates individual feed configs for each arch subdir of those
242 specified as compatible for the current machine.
243 NOTE: Development-helper feature, NOT a full-fledged feed.
244 """
245 if (self.d.getVar('FEED_DEPLOYDIR_BASE_URI') or "") != "":
246 for arch in self.pkg_archs.split():
247 cfg_file_name = os.path.join(self.target_rootfs,
248 self.d.getVar("sysconfdir"),
249 "opkg",
250 "local-%s-feed.conf" % arch)
251
252 with open(cfg_file_name, "w+") as cfg_file:
253 cfg_file.write("src/gz local-%s %s/%s" %
254 (arch,
255 self.d.getVar('FEED_DEPLOYDIR_BASE_URI'),
256 arch))
257
258 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
259 # There is no command line option for this anymore, we need to add
260 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
261 # the default value of "/var/lib" as defined in opkg:
262 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
263 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
264 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
265 cfg_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
266 cfg_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
267 cfg_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
268
269
270 def _create_config(self):
271 with open(self.config_file, "w+") as config_file:
272 priority = 1
273 for arch in self.pkg_archs.split():
274 config_file.write("arch %s %d\n" % (arch, priority))
275 priority += 5
276
277 config_file.write("src oe file:%s\n" % self.deploy_dir)
278
279 for arch in self.pkg_archs.split():
280 pkgs_dir = os.path.join(self.deploy_dir, arch)
281 if os.path.isdir(pkgs_dir):
282 config_file.write("src oe-%s file:%s\n" %
283 (arch, pkgs_dir))
284
285 if self.d.getVar('OPKGLIBDIR') != '/var/lib':
286 # There is no command line option for this anymore, we need to add
287 # info_dir and status_file to config file, if OPKGLIBDIR doesn't have
288 # the default value of "/var/lib" as defined in opkg:
289 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_LISTS_DIR VARDIR "/lib/opkg/lists"
290 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_INFO_DIR VARDIR "/lib/opkg/info"
291 # libopkg/opkg_conf.h:#define OPKG_CONF_DEFAULT_STATUS_FILE VARDIR "/lib/opkg/status"
292 config_file.write("option info_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'info'))
293 config_file.write("option lists_dir %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'lists'))
294 config_file.write("option status_file %s\n" % os.path.join(self.d.getVar('OPKGLIBDIR'), 'opkg', 'status'))
295
296 def insert_feeds_uris(self, feed_uris, feed_base_paths, feed_archs):
297 if feed_uris == "":
298 return
299
300 rootfs_config = os.path.join('%s/etc/opkg/base-feeds.conf'
301 % self.target_rootfs)
302
303 os.makedirs('%s/etc/opkg' % self.target_rootfs, exist_ok=True)
304
305 feed_uris = self.construct_uris(feed_uris.split(), feed_base_paths.split())
306 archs = self.pkg_archs.split() if feed_archs is None else feed_archs.split()
307
308 with open(rootfs_config, "w+") as config_file:
309 uri_iterator = 0
310 for uri in feed_uris:
311 if archs:
312 for arch in archs:
313 if (feed_archs is None) and (not os.path.exists(oe.path.join(self.deploy_dir, arch))):
314 continue
315 bb.note('Adding opkg feed url-%s-%d (%s)' %
316 (arch, uri_iterator, uri))
317 config_file.write("src/gz uri-%s-%d %s/%s\n" %
318 (arch, uri_iterator, uri, arch))
319 else:
320 bb.note('Adding opkg feed url-%d (%s)' %
321 (uri_iterator, uri))
322 config_file.write("src/gz uri-%d %s\n" %
323 (uri_iterator, uri))
324
325 uri_iterator += 1
326
327 def update(self):
328 self.deploy_dir_lock()
329
330 cmd = "%s %s update" % (self.opkg_cmd, self.opkg_args)
331
332 try:
333 subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT)
334 except subprocess.CalledProcessError as e:
335 self.deploy_dir_unlock()
336 bb.fatal("Unable to update the package index files. Command '%s' "
337 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
338
339 self.deploy_dir_unlock()
340
341 def install(self, pkgs, attempt_only=False):
342 if not pkgs:
343 return
344
345 cmd = "%s %s" % (self.opkg_cmd, self.opkg_args)
346 for exclude in (self.d.getVar("PACKAGE_EXCLUDE") or "").split():
347 cmd += " --add-exclude %s" % exclude
348 for bad_recommendation in (self.d.getVar("BAD_RECOMMENDATIONS") or "").split():
349 cmd += " --add-ignore-recommends %s" % bad_recommendation
350 cmd += " install "
351 cmd += " ".join(pkgs)
352
353 os.environ['D'] = self.target_rootfs
354 os.environ['OFFLINE_ROOT'] = self.target_rootfs
355 os.environ['IPKG_OFFLINE_ROOT'] = self.target_rootfs
356 os.environ['OPKG_OFFLINE_ROOT'] = self.target_rootfs
357 os.environ['INTERCEPT_DIR'] = self.intercepts_dir
358 os.environ['NATIVE_ROOT'] = self.d.getVar('STAGING_DIR_NATIVE')
359
360 try:
361 bb.note("Installing the following packages: %s" % ' '.join(pkgs))
362 bb.note(cmd)
363 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
364 bb.note(output)
365 failed_pkgs = []
366 for line in output.split('\n'):
367 if line.endswith("configuration required on target."):
368 bb.warn(line)
369 failed_pkgs.append(line.split(".")[0])
370 if failed_pkgs:
371 failed_postinsts_abort(failed_pkgs, self.d.expand("${T}/log.do_${BB_CURRENTTASK}"))
372 except subprocess.CalledProcessError as e:
373 (bb.fatal, bb.warn)[attempt_only]("Unable to install packages. "
374 "Command '%s' returned %d:\n%s" %
375 (cmd, e.returncode, e.output.decode("utf-8")))
376
377 def remove(self, pkgs, with_dependencies=True):
378 if not pkgs:
379 return
380
381 if with_dependencies:
382 cmd = "%s %s --force-remove --force-removal-of-dependent-packages remove %s" % \
383 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
384 else:
385 cmd = "%s %s --force-depends remove %s" % \
386 (self.opkg_cmd, self.opkg_args, ' '.join(pkgs))
387
388 try:
389 bb.note(cmd)
390 output = subprocess.check_output(cmd.split(), stderr=subprocess.STDOUT).decode("utf-8")
391 bb.note(output)
392 except subprocess.CalledProcessError as e:
393 bb.fatal("Unable to remove packages. Command '%s' "
394 "returned %d:\n%s" % (e.cmd, e.returncode, e.output.decode("utf-8")))
395
396 def write_index(self):
397 self.deploy_dir_lock()
398
399 result = self.indexer.write_index()
400
401 self.deploy_dir_unlock()
402
403 if result is not None:
404 bb.fatal(result)
405
406 def remove_packaging_data(self):
407 bb.utils.remove(self.opkg_dir, True)
408 # create the directory back, it's needed by PM lock
409 bb.utils.mkdirhier(self.opkg_dir)
410
411 def remove_lists(self):
412 if not self.from_feeds:
413 bb.utils.remove(os.path.join(self.opkg_dir, "lists"), True)
414
415 def list_installed(self):
416 return OpkgPkgsList(self.d, self.target_rootfs, self.config_file).list_pkgs()
417
418 def dummy_install(self, pkgs):
419 """
420 The following function dummy installs pkgs and returns the log of output.
421 """
422 if len(pkgs) == 0:
423 return
424
425 # Create an temp dir as opkg root for dummy installation
426 temp_rootfs = self.d.expand('${T}/opkg')
427 opkg_lib_dir = self.d.getVar('OPKGLIBDIR')
428 if opkg_lib_dir[0] == "/":
429 opkg_lib_dir = opkg_lib_dir[1:]
430 temp_opkg_dir = os.path.join(temp_rootfs, opkg_lib_dir, 'opkg')
431 bb.utils.mkdirhier(temp_opkg_dir)
432
433 opkg_args = "-f %s -o %s " % (self.config_file, temp_rootfs)
434 opkg_args += self.d.getVar("OPKG_ARGS")
435
436 cmd = "%s %s update" % (self.opkg_cmd, opkg_args)
437 try:
438 subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
439 except subprocess.CalledProcessError as e:
440 bb.fatal("Unable to update. Command '%s' "
441 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
442
443 # Dummy installation
444 cmd = "%s %s --noaction install %s " % (self.opkg_cmd,
445 opkg_args,
446 ' '.join(pkgs))
447 try:
448 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
449 except subprocess.CalledProcessError as e:
450 bb.fatal("Unable to dummy install packages. Command '%s' "
451 "returned %d:\n%s" % (cmd, e.returncode, e.output.decode("utf-8")))
452
453 bb.utils.remove(temp_rootfs, True)
454
455 return output
456
457 def backup_packaging_data(self):
458 # Save the opkglib for increment ipk image generation
459 if os.path.exists(self.saved_opkg_dir):
460 bb.utils.remove(self.saved_opkg_dir, True)
461 shutil.copytree(self.opkg_dir,
462 self.saved_opkg_dir,
463 symlinks=True)
464
465 def recover_packaging_data(self):
466 # Move the opkglib back
467 if os.path.exists(self.saved_opkg_dir):
468 if os.path.exists(self.opkg_dir):
469 bb.utils.remove(self.opkg_dir, True)
470
471 bb.note('Recover packaging data')
472 shutil.copytree(self.saved_opkg_dir,
473 self.opkg_dir,
474 symlinks=True)
475
476 def package_info(self, pkg):
477 """
478 Returns a dictionary with the package info.
479 """
480 cmd = "%s %s info %s" % (self.opkg_cmd, self.opkg_args, pkg)
481 pkg_info = super(OpkgPM, self).package_info(pkg, cmd)
482
483 pkg_arch = pkg_info[pkg]["arch"]
484 pkg_filename = pkg_info[pkg]["filename"]
485 pkg_info[pkg]["filepath"] = \
486 os.path.join(self.deploy_dir, pkg_arch, pkg_filename)
487
488 return pkg_info
489
490 def extract(self, pkg):
491 """
492 Returns the path to a tmpdir where resides the contents of a package.
493
494 Deleting the tmpdir is responsability of the caller.
495 """
496 pkg_info = self.package_info(pkg)
497 if not pkg_info:
498 bb.fatal("Unable to get information for package '%s' while "
499 "trying to extract the package." % pkg)
500
501 tmp_dir = super(OpkgPM, self).extract(pkg, pkg_info)
502 bb.utils.remove(os.path.join(tmp_dir, "data.tar.xz"))
503
504 return tmp_dir
diff --git a/meta/lib/oe/package_manager/ipk/rootfs.py b/meta/lib/oe/package_manager/ipk/rootfs.py
index 373fcbd93a..34dbcc27e6 100644
--- a/meta/lib/oe/package_manager/ipk/rootfs.py
+++ b/meta/lib/oe/package_manager/ipk/rootfs.py
@@ -5,10 +5,10 @@
5import re 5import re
6import filecmp 6import filecmp
7from oe.rootfs import Rootfs 7from oe.rootfs import Rootfs
8from oe.package_manager import OpkgPM
9from oe.manifest import Manifest 8from oe.manifest import Manifest
10from oe.utils import execute_pre_post_process 9from oe.utils import execute_pre_post_process
11from oe.package_manager.ipk.manifest import OpkgManifest 10from oe.package_manager.ipk.manifest import OpkgManifest
11from oe.package_manager.ipk import OpkgPM
12 12
13class DpkgOpkgRootfs(Rootfs): 13class DpkgOpkgRootfs(Rootfs):
14 def __init__(self, d, progress_reporter=None, logcatcher=None): 14 def __init__(self, d, progress_reporter=None, logcatcher=None):
diff --git a/meta/lib/oe/package_manager/ipk/sdk.py b/meta/lib/oe/package_manager/ipk/sdk.py
index 3d59f7a517..e6d7b04bb8 100644
--- a/meta/lib/oe/package_manager/ipk/sdk.py
+++ b/meta/lib/oe/package_manager/ipk/sdk.py
@@ -5,7 +5,7 @@
5from oe.utils import execute_pre_post_process 5from oe.utils import execute_pre_post_process
6from oe.sdk import Sdk 6from oe.sdk import Sdk
7from oe.manifest import Manifest 7from oe.manifest import Manifest
8from oe.package_manager import OpkgPM 8from oe.package_manager.ipk import OpkgPM
9import shutil 9import shutil
10import glob 10import glob
11 11