summaryrefslogtreecommitdiffstats
path: root/meta/lib/oe
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-13 11:40:03 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-15 09:44:33 +0100
commit65c8154f2169e6e901dd1e38d3cc893b019a8e11 (patch)
tree30a4beab6df2bd8b631638b608c0731d3427a24b /meta/lib/oe
parent20c8be711f3ffbc9f7916c9e101db19bb0281efb (diff)
downloadpoky-65c8154f2169e6e901dd1e38d3cc893b019a8e11.tar.gz
image: Add locale archive optimisation
Refactor the locale archive function from the SDK to also make it work during general image creation. This reduces the size of the locales from 900MB to 220MB in core-image-lsb-sdk. The exception handling around subprocess was dropped as the standard subprocess exception printing is better handled than the catchall exception. (From OE-Core rev: 8ffd93bdb09b0a4a84b27dafcd684c6abba392ed) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib/oe')
-rw-r--r--meta/lib/oe/package_manager.py49
-rw-r--r--meta/lib/oe/sdk.py51
2 files changed, 53 insertions, 47 deletions
diff --git a/meta/lib/oe/package_manager.py b/meta/lib/oe/package_manager.py
index 64c8a91216..6011e87350 100644
--- a/meta/lib/oe/package_manager.py
+++ b/meta/lib/oe/package_manager.py
@@ -12,6 +12,7 @@ import oe.path
12import string 12import string
13from oe.gpg_sign import get_signer 13from oe.gpg_sign import get_signer
14import hashlib 14import hashlib
15import fnmatch
15 16
16# this can be used by all PM backends to create the index files in parallel 17# this can be used by all PM backends to create the index files in parallel
17def create_index(arg): 18def create_index(arg):
@@ -89,6 +90,47 @@ def failed_postinsts_warn(pkgs, log_path):
89If deferring to first boot wasn't the intent, then scriptlet failure may mean an issue in the recipe, or a regression elsewhere. 90If deferring to first boot wasn't the intent, then scriptlet failure may mean an issue in the recipe, or a regression elsewhere.
90Details of the failure are in %s.""" %(pkgs, log_path)) 91Details of the failure are in %s.""" %(pkgs, log_path))
91 92
93def generate_locale_archive(d, rootfs, target_arch, localedir):
94 # Pretty sure we don't need this for locale archive generation but
95 # keeping it to be safe...
96 locale_arch_options = { \
97 "arm": ["--uint32-align=4", "--little-endian"],
98 "armeb": ["--uint32-align=4", "--big-endian"],
99 "aarch64": ["--uint32-align=4", "--little-endian"],
100 "aarch64_be": ["--uint32-align=4", "--big-endian"],
101 "sh4": ["--uint32-align=4", "--big-endian"],
102 "powerpc": ["--uint32-align=4", "--big-endian"],
103 "powerpc64": ["--uint32-align=4", "--big-endian"],
104 "mips": ["--uint32-align=4", "--big-endian"],
105 "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
106 "mips64": ["--uint32-align=4", "--big-endian"],
107 "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
108 "mipsel": ["--uint32-align=4", "--little-endian"],
109 "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
110 "mips64el": ["--uint32-align=4", "--little-endian"],
111 "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
112 "i586": ["--uint32-align=4", "--little-endian"],
113 "i686": ["--uint32-align=4", "--little-endian"],
114 "x86_64": ["--uint32-align=4", "--little-endian"]
115 }
116 if target_arch in locale_arch_options:
117 arch_options = locale_arch_options[target_arch]
118 else:
119 bb.error("locale_arch_options not found for target_arch=" + target_arch)
120 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
121
122 # Need to set this so cross-localedef knows where the archive is
123 env = dict(os.environ)
124 env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
125
126 for name in os.listdir(localedir):
127 path = os.path.join(localedir, name)
128 if os.path.isdir(path):
129 cmd = ["cross-localedef", "--verbose"]
130 cmd += arch_options
131 cmd += ["--add-to-archive", path]
132 subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
133
92class Indexer(object, metaclass=ABCMeta): 134class Indexer(object, metaclass=ABCMeta):
93 def __init__(self, d, deploy_dir): 135 def __init__(self, d, deploy_dir):
94 self.d = d 136 self.d = d
@@ -536,6 +578,13 @@ class PackageManager(object, metaclass=ABCMeta):
536 "'%s' returned %d:\n%s" % 578 "'%s' returned %d:\n%s" %
537 (' '.join(cmd), e.returncode, e.output.decode("utf-8"))) 579 (' '.join(cmd), e.returncode, e.output.decode("utf-8")))
538 580
581 target_arch = self.d.getVar('TARGET_ARCH')
582 localedir = oe.path.join(self.target_rootfs, self.d.getVar("libdir"), "locale")
583 if os.path.exists(localedir) and os.listdir(localedir):
584 generate_locale_archive(self.d, self.target_rootfs, target_arch, localedir)
585 # And now delete the binary locales
586 self.remove(fnmatch.filter(self.list_installed(), "glibc-binary-localedata-*"), False)
587
539 def deploy_dir_lock(self): 588 def deploy_dir_lock(self):
540 if self.deploy_dir is None: 589 if self.deploy_dir is None:
541 raise RuntimeError("deploy_dir is not set!") 590 raise RuntimeError("deploy_dir is not set!")
diff --git a/meta/lib/oe/sdk.py b/meta/lib/oe/sdk.py
index 0d39ea8a91..f20441ccf6 100644
--- a/meta/lib/oe/sdk.py
+++ b/meta/lib/oe/sdk.py
@@ -7,52 +7,6 @@ import shutil
7import glob 7import glob
8import traceback 8import traceback
9 9
10def generate_locale_archive(d, rootfs):
11 # Pretty sure we don't need this for SDK archive generation but
12 # keeping it to be safe...
13 target_arch = d.getVar('SDK_ARCH')
14 locale_arch_options = { \
15 "arm": ["--uint32-align=4", "--little-endian"],
16 "armeb": ["--uint32-align=4", "--big-endian"],
17 "aarch64": ["--uint32-align=4", "--little-endian"],
18 "aarch64_be": ["--uint32-align=4", "--big-endian"],
19 "sh4": ["--uint32-align=4", "--big-endian"],
20 "powerpc": ["--uint32-align=4", "--big-endian"],
21 "powerpc64": ["--uint32-align=4", "--big-endian"],
22 "mips": ["--uint32-align=4", "--big-endian"],
23 "mipsisa32r6": ["--uint32-align=4", "--big-endian"],
24 "mips64": ["--uint32-align=4", "--big-endian"],
25 "mipsisa64r6": ["--uint32-align=4", "--big-endian"],
26 "mipsel": ["--uint32-align=4", "--little-endian"],
27 "mipsisa32r6el": ["--uint32-align=4", "--little-endian"],
28 "mips64el": ["--uint32-align=4", "--little-endian"],
29 "mipsisa64r6el": ["--uint32-align=4", "--little-endian"],
30 "i586": ["--uint32-align=4", "--little-endian"],
31 "i686": ["--uint32-align=4", "--little-endian"],
32 "x86_64": ["--uint32-align=4", "--little-endian"]
33 }
34 if target_arch in locale_arch_options:
35 arch_options = locale_arch_options[target_arch]
36 else:
37 bb.error("locale_arch_options not found for target_arch=" + target_arch)
38 bb.fatal("unknown arch:" + target_arch + " for locale_arch_options")
39
40 localedir = oe.path.join(rootfs, d.getVar("libdir_nativesdk"), "locale")
41 # Need to set this so cross-localedef knows where the archive is
42 env = dict(os.environ)
43 env["LOCALEARCHIVE"] = oe.path.join(localedir, "locale-archive")
44
45 for name in os.listdir(localedir):
46 path = os.path.join(localedir, name)
47 if os.path.isdir(path):
48 try:
49 cmd = ["cross-localedef", "--verbose"]
50 cmd += arch_options
51 cmd += ["--add-to-archive", path]
52 subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT)
53 except Exception as e:
54 bb.fatal("Cannot create locale archive: %s" % e.output)
55
56class Sdk(object, metaclass=ABCMeta): 10class Sdk(object, metaclass=ABCMeta):
57 def __init__(self, d, manifest_dir): 11 def __init__(self, d, manifest_dir):
58 self.d = d 12 self.d = d
@@ -144,7 +98,10 @@ class Sdk(object, metaclass=ABCMeta):
144 for lang in linguas.split(): 98 for lang in linguas.split():
145 pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang) 99 pm.install("nativesdk-glibc-binary-localedata-%s.utf-8" % lang)
146 # Generate a locale archive of them 100 # Generate a locale archive of them
147 generate_locale_archive(self.d, oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)) 101 target_arch = self.d.getVar('SDK_ARCH')
102 rootfs = oe.path.join(self.sdk_host_sysroot, self.sdk_native_path)
103 localedir = oe.path.join(rootfs, self.d.getVar("libdir_nativesdk"), "locale")
104 generate_locale_archive(self.d, rootfs, target_arch, localedir)
148 # And now delete the binary locales 105 # And now delete the binary locales
149 pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8") 106 pkgs = fnmatch.filter(pm.list_installed(), "nativesdk-glibc-binary-localedata-*.utf-8")
150 pm.remove(pkgs) 107 pm.remove(pkgs)