summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2017-08-21 17:39:49 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2017-08-23 08:47:02 +0100
commitb1f237ebd0d4180c5d23a0ecd9aaf7193c63a48a (patch)
treed49aba7187426b2f1d049787681c7140ebd1a4dd
parenta7baa47c876c7895499909731aaa451c6009610a (diff)
downloadpoky-b1f237ebd0d4180c5d23a0ecd9aaf7193c63a48a.tar.gz
recipetool: allow plugins to set LICENSE and LIC_FILES_CHKSUM
We were being a bit prescriptive in setting LICENSE and LIC_FILES_CHKSUM. We can't always trust what's in the metadata accompanying some source which plugins will almost always be pulling from, however we do want to allow plugins to set the LICENSE and LIC_FILES_CHKSUM values. Merge what we find in our license file scan with what the plugin sends back. Additionally, plugins can now add a "license" item to the handled list in order to inhibit the normal LICENSE / LIC_FILES_CHKSUM handling if they have already taken care of it completely. Thanks to Mark Horn <mark.d.horn@intel.com> for prompting, testing and fixing this patch. (From OE-Core rev: 1df60b09f7a60427795ec828c9c7180e4e52f98c) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/recipetool/create.py115
-rw-r--r--scripts/lib/recipetool/create_npm.py44
2 files changed, 79 insertions, 80 deletions
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 40bd3c820b..26011451dd 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -618,9 +618,10 @@ def create_recipe(args):
618 # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments 618 # We need a blank line here so that patch_recipe_lines can rewind before the LICENSE comments
619 lines_before.append('') 619 lines_before.append('')
620 620
621 handled = [] 621 # We'll come back and replace this later in handle_license_vars()
622 licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data) 622 lines_before.append('##LICENSE_PLACEHOLDER##')
623 623
624 handled = []
624 classes = [] 625 classes = []
625 626
626 # FIXME This is kind of a hack, we probably ought to be using bitbake to do this 627 # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
@@ -751,6 +752,8 @@ def create_recipe(args):
751 if name_pv and not realpv: 752 if name_pv and not realpv:
752 realpv = name_pv 753 realpv = name_pv
753 754
755 licvalues = handle_license_vars(srctree_use, lines_before, handled, extravalues, tinfoil.config_data)
756
754 if not outfile: 757 if not outfile:
755 if not pn: 758 if not pn:
756 log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool) 759 log_error_cond('Unable to determine short program name from source tree - please specify name with -N/--name or output file name with -o/--outfile', args.devtool)
@@ -843,9 +846,6 @@ def create_recipe(args):
843 outlines.extend(lines_after) 846 outlines.extend(lines_after)
844 847
845 if extravalues: 848 if extravalues:
846 if 'LICENSE' in extravalues and not licvalues:
847 # Don't blow away 'CLOSED' value that comments say we set
848 del extravalues['LICENSE']
849 _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False) 849 _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
850 850
851 if args.extract_to: 851 if args.extract_to:
@@ -889,55 +889,94 @@ def check_single_file(fn, fetchuri):
889 logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri) 889 logger.error('Fetching "%s" returned a single HTML page - check the URL is correct and functional' % fetchuri)
890 sys.exit(1) 890 sys.exit(1)
891 891
892def split_value(value):
893 if isinstance(value, str):
894 return value.split()
895 else:
896 return value
892 897
893def handle_license_vars(srctree, lines_before, handled, extravalues, d): 898def handle_license_vars(srctree, lines_before, handled, extravalues, d):
899 lichandled = [x for x in handled if x[0] == 'license']
900 if lichandled:
901 # Someone else has already handled the license vars, just return their value
902 return lichandled[0][1]
903
894 licvalues = guess_license(srctree, d) 904 licvalues = guess_license(srctree, d)
905 licenses = []
895 lic_files_chksum = [] 906 lic_files_chksum = []
896 lic_unknown = [] 907 lic_unknown = []
908 lines = []
897 if licvalues: 909 if licvalues:
898 licenses = []
899 for licvalue in licvalues: 910 for licvalue in licvalues:
900 if not licvalue[0] in licenses: 911 if not licvalue[0] in licenses:
901 licenses.append(licvalue[0]) 912 licenses.append(licvalue[0])
902 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2])) 913 lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
903 if licvalue[0] == 'Unknown': 914 if licvalue[0] == 'Unknown':
904 lic_unknown.append(licvalue[1]) 915 lic_unknown.append(licvalue[1])
905 lines_before.append('# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
906 lines_before.append('# your responsibility to verify that the values are complete and correct.')
907 if len(licvalues) > 1:
908 lines_before.append('#')
909 lines_before.append('# NOTE: multiple licenses have been detected; they have been separated with &')
910 lines_before.append('# in the LICENSE value for now since it is a reasonable assumption that all')
911 lines_before.append('# of the licenses apply. If instead there is a choice between the multiple')
912 lines_before.append('# licenses then you should change the value to separate the licenses with |')
913 lines_before.append('# instead of &. If there is any doubt, check the accompanying documentation')
914 lines_before.append('# to determine which situation is applicable.')
915 if lic_unknown: 916 if lic_unknown:
916 lines_before.append('#') 917 lines.append('#')
917 lines_before.append('# The following license files were not able to be identified and are') 918 lines.append('# The following license files were not able to be identified and are')
918 lines_before.append('# represented as "Unknown" below, you will need to check them yourself:') 919 lines.append('# represented as "Unknown" below, you will need to check them yourself:')
919 for licfile in lic_unknown: 920 for licfile in lic_unknown:
920 lines_before.append('# %s' % licfile) 921 lines.append('# %s' % licfile)
921 lines_before.append('#') 922
922 else: 923 extra_license = split_value(extravalues.pop('LICENSE', []))
923 lines_before.append('# Unable to find any files that looked like license statements. Check the accompanying') 924 if '&' in extra_license:
924 lines_before.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.') 925 extra_license.remove('&')
925 lines_before.append('#') 926 if extra_license:
926 lines_before.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
927 lines_before.append('# this is not accurate with respect to the licensing of the software being built (it')
928 lines_before.append('# will not be in most cases) you must specify the correct value before using this')
929 lines_before.append('# recipe for anything other than initial testing/development!')
930 licenses = ['CLOSED']
931 pkg_license = extravalues.pop('LICENSE', None)
932 if pkg_license:
933 if licenses == ['Unknown']: 927 if licenses == ['Unknown']:
934 lines_before.append('# NOTE: The following LICENSE value was determined from the original package metadata') 928 licenses = extra_license
935 licenses = [pkg_license]
936 else: 929 else:
937 lines_before.append('# NOTE: Original package metadata indicates license is: %s' % pkg_license) 930 for item in extra_license:
938 lines_before.append('LICENSE = "%s"' % ' & '.join(licenses)) 931 if item not in licenses:
939 lines_before.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum)) 932 licenses.append(item)
940 lines_before.append('') 933 extra_lic_files_chksum = split_value(extravalues.pop('LIC_FILES_CHKSUM', []))
934 for item in extra_lic_files_chksum:
935 if item not in lic_files_chksum:
936 lic_files_chksum.append(item)
937
938 if lic_files_chksum:
939 # We are going to set the vars, so prepend the standard disclaimer
940 lines.insert(0, '# WARNING: the following LICENSE and LIC_FILES_CHKSUM values are best guesses - it is')
941 lines.insert(1, '# your responsibility to verify that the values are complete and correct.')
942 else:
943 # Without LIC_FILES_CHKSUM we set LICENSE = "CLOSED" to allow the
944 # user to get started easily
945 lines.append('# Unable to find any files that looked like license statements. Check the accompanying')
946 lines.append('# documentation and source headers and set LICENSE and LIC_FILES_CHKSUM accordingly.')
947 lines.append('#')
948 lines.append('# NOTE: LICENSE is being set to "CLOSED" to allow you to at least start building - if')
949 lines.append('# this is not accurate with respect to the licensing of the software being built (it')
950 lines.append('# will not be in most cases) you must specify the correct value before using this')
951 lines.append('# recipe for anything other than initial testing/development!')
952 licenses = ['CLOSED']
953
954 if extra_license and sorted(licenses) != sorted(extra_license):
955 lines.append('# NOTE: Original package / source metadata indicates license is: %s' % ' & '.join(extra_license))
956
957 if len(licenses) > 1:
958 lines.append('#')
959 lines.append('# NOTE: multiple licenses have been detected; they have been separated with &')
960 lines.append('# in the LICENSE value for now since it is a reasonable assumption that all')
961 lines.append('# of the licenses apply. If instead there is a choice between the multiple')
962 lines.append('# licenses then you should change the value to separate the licenses with |')
963 lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
964 lines.append('# to determine which situation is applicable.')
965
966 lines.append('LICENSE = "%s"' % ' & '.join(licenses))
967 lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n '.join(lic_files_chksum))
968 lines.append('')
969
970 # Replace the placeholder so we get the values in the right place in the recipe file
971 try:
972 pos = lines_before.index('##LICENSE_PLACEHOLDER##')
973 except ValueError:
974 pos = -1
975 if pos == -1:
976 lines_before.extend(lines)
977 else:
978 lines_before[pos:pos+1] = lines
979
941 handled.append(('license', licvalues)) 980 handled.append(('license', licvalues))
942 return licvalues 981 return licvalues
943 982
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 885d5438e3..07fcf4d883 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -164,37 +164,6 @@ class NpmRecipeHandler(RecipeHandler):
164 lines_before.append(line) 164 lines_before.append(line)
165 return updated 165 return updated
166 166
167 def _replace_license_vars(self, srctree, lines_before, handled, extravalues, d):
168 for item in handled:
169 if isinstance(item, tuple):
170 if item[0] == 'license':
171 del item
172 break
173
174 calledvars = []
175 def varfunc(varname, origvalue, op, newlines):
176 if varname in ['LICENSE', 'LIC_FILES_CHKSUM']:
177 for i, e in enumerate(reversed(newlines)):
178 if not e.startswith('#'):
179 stop = i
180 while stop > 0:
181 newlines.pop()
182 stop -= 1
183 break
184 calledvars.append(varname)
185 if len(calledvars) > 1:
186 # The second time around, put the new license text in
187 insertpos = len(newlines)
188 handle_license_vars(srctree, newlines, handled, extravalues, d)
189 return None, None, 0, True
190 return origvalue, None, 0, True
191 updated, newlines = bb.utils.edit_metadata(lines_before, ['LICENSE', 'LIC_FILES_CHKSUM'], varfunc)
192 if updated:
193 del lines_before[:]
194 lines_before.extend(newlines)
195 else:
196 raise Exception('Did not find license variables')
197
198 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues): 167 def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
199 import bb.utils 168 import bb.utils
200 import oe 169 import oe
@@ -228,10 +197,7 @@ class NpmRecipeHandler(RecipeHandler):
228 197
229 fetchdev = extravalues['fetchdev'] or None 198 fetchdev = extravalues['fetchdev'] or None
230 deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev) 199 deps, optdeps, devdeps = self.get_npm_package_dependencies(data, fetchdev)
231 updated = self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree) 200 self._handle_dependencies(d, deps, optdeps, devdeps, lines_before, srctree)
232 if updated:
233 # We need to redo the license stuff
234 self._replace_license_vars(srctree, lines_before, handled, extravalues, d)
235 201
236 # Shrinkwrap 202 # Shrinkwrap
237 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm') 203 localfilesdir = tempfile.mkdtemp(prefix='recipetool-npm')
@@ -267,13 +233,7 @@ class NpmRecipeHandler(RecipeHandler):
267 all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense])) 233 all_licenses = list(set([item.replace('_', ' ') for pkglicense in pkglicenses.values() for item in pkglicense]))
268 if '&' in all_licenses: 234 if '&' in all_licenses:
269 all_licenses.remove('&') 235 all_licenses.remove('&')
270 # Go back and update the LICENSE value since we have a bit more 236 extravalues['LICENSE'] = ' & '.join(all_licenses)
271 # information than when that was written out (and we know all apply
272 # vs. there being a choice, so we can join them with &)
273 for i, line in enumerate(lines_before):
274 if line.startswith('LICENSE = '):
275 lines_before[i] = 'LICENSE = "%s"' % ' & '.join(all_licenses)
276 break
277 237
278 # Need to move S setting after inherit npm 238 # Need to move S setting after inherit npm
279 for i, line in enumerate(lines_before): 239 for i, line in enumerate(lines_before):