summaryrefslogtreecommitdiffstats
path: root/meta
diff options
context:
space:
mode:
authorliangcao <liangcao@unomaha.edu>2013-08-23 14:40:35 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2013-08-26 13:19:41 +0100
commita0904066865c9792033d6c87c270966113b6ae66 (patch)
tree79f27c3335d8f434bdfc7b3bba139dfae03a19fa /meta
parentdaedc2fda2383f8ef678170d6337cc70d1444e25 (diff)
downloadpoky-a0904066865c9792033d6c87c270966113b6ae66.tar.gz
SPDX:real-time license scanning and SPDX output.
SPDX integrates real-time license scanning, generates SPDX standard output and license verification information during the OE-Core build process. The existing module includes scanning patched packages and creating package and file level SPDX documents. (From OE-Core rev: 7a37cc81fb95d56b5ac5e5ca22a1900e45717911) Signed-off-by: liangcao <liangcao@unomaha.edu> Signed-off-by: Elizabeth Flanagan <elizabeth.flanagan@intel.com> Signed-off-by: Saul Wold <sgw@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta')
-rw-r--r--meta/classes/spdx.bbclass321
-rw-r--r--meta/conf/licenses.conf51
2 files changed, 371 insertions, 1 deletions
diff --git a/meta/classes/spdx.bbclass b/meta/classes/spdx.bbclass
new file mode 100644
index 0000000000..bde6e496ff
--- /dev/null
+++ b/meta/classes/spdx.bbclass
@@ -0,0 +1,321 @@
1# This class integrates real-time license scanning, generation of SPDX standard
2# output and verifiying license info during the building process.
3# It is a combination of efforts from the OE-Core, SPDX and Fossology projects.
4#
5# For more information on FOSSology:
6# http://www.fossology.org
7#
8# For more information on FOSSologySPDX commandline:
9# https://github.com/spdx-tools/fossology-spdx/wiki/Fossology-SPDX-Web-API
10#
11# For more information on SPDX:
12# http://www.spdx.org
13#
14
15# SPDX file will be output to the path which is defined as[SPDX_MANIFEST_DIR]
16# in ./meta/conf/licenses.conf.
17
18SPDXOUTPUTDIR = "${WORKDIR}/spdx_output_dir"
19SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir"
20
21python do_spdx () {
22 import os, sys
23 import json
24
25 info = {}
26 info['workdir'] = (d.getVar('WORKDIR', True) or "")
27 info['sourcedir'] = (d.getVar('S', True) or "")
28 info['pn'] = (d.getVar( 'PN', True ) or "")
29 info['pv'] = (d.getVar( 'PV', True ) or "")
30 info['src_uri'] = (d.getVar( 'SRC_URI', True ) or "")
31 info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
32 info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
33
34 spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "")
35 manifest_dir = (d.getVar('SPDX_MANIFEST_DIR', True) or "")
36 info['outfile'] = os.path.join(manifest_dir, info['pn'] + ".spdx" )
37 sstatefile = os.path.join(spdx_sstate_dir,
38 info['pn'] + info['pv'] + ".spdx" )
39 info['spdx_temp_dir'] = (d.getVar('SPDX_TEMP_DIR', True) or "")
40 info['tar_file'] = os.path.join( info['workdir'], info['pn'] + ".tar.gz" )
41
42
43 ## get everything from cache. use it to decide if
44 ## something needs to be rerun
45 cur_ver_code = get_ver_code( info['sourcedir'] )
46 cache_cur = False
47 if not os.path.exists( spdx_sstate_dir ):
48 bb.mkdirhier( spdx_sstate_dir )
49 if not os.path.exists( info['spdx_temp_dir'] ):
50 bb.mkdirhier( info['spdx_temp_dir'] )
51 if os.path.exists( sstatefile ):
52 ## cache for this package exists. read it in
53 cached_spdx = get_cached_spdx( sstatefile )
54
55 if cached_spdx['PackageVerificationCode'] == cur_ver_code:
56 bb.warn(info['pn'] + "'s ver code same as cache's. do nothing")
57 cache_cur = True
58 else:
59 local_file_info = setup_foss_scan( info,
60 True, cached_spdx['Files'] )
61 else:
62 local_file_info = setup_foss_scan( info, False, None )
63
64 if cache_cur:
65 spdx_file_info = cached_spdx['Files']
66 else:
67 ## setup fossology command
68 foss_server = (d.getVar('FOSS_SERVER', True) or "")
69 foss_flags = (d.getVar('FOSS_WGET_FLAGS', True) or "")
70 foss_command = "wget %s --post-file=%s %s"\
71 % (foss_flags,info['tar_file'],foss_server)
72
73 #bb.warn(info['pn'] + json.dumps(local_file_info))
74 foss_file_info = run_fossology( foss_command )
75 spdx_file_info = create_spdx_doc( local_file_info, foss_file_info )
76 ## write to cache
77 write_cached_spdx(sstatefile,cur_ver_code,spdx_file_info)
78
79 ## Get document and package level information
80 spdx_header_info = get_header_info(info, cur_ver_code, spdx_file_info)
81
82 ## CREATE MANIFEST
83 create_manifest(info,spdx_header_info,spdx_file_info)
84
85 ## clean up the temp stuff
86 remove_dir_tree( info['spdx_temp_dir'] )
87 if os.path.exists(info['tar_file']):
88 remove_file( info['tar_file'] )
89}
90addtask spdx after do_patch before do_configure
91
92def create_manifest(info,header,files):
93 with open(info['outfile'], 'w') as f:
94 f.write(header + '\n')
95 for chksum, block in files.iteritems():
96 for key, value in block.iteritems():
97 f.write(key + ": " + value)
98 f.write('\n')
99 f.write('\n')
100
101def get_cached_spdx( sstatefile ):
102 import json
103 cached_spdx_info = {}
104 with open( sstatefile, 'r' ) as f:
105 try:
106 cached_spdx_info = json.load(f)
107 except ValueError as e:
108 cached_spdx_info = None
109 return cached_spdx_info
110
111def write_cached_spdx( sstatefile, ver_code, files ):
112 import json
113 spdx_doc = {}
114 spdx_doc['PackageVerificationCode'] = ver_code
115 spdx_doc['Files'] = {}
116 spdx_doc['Files'] = files
117 with open( sstatefile, 'w' ) as f:
118 f.write(json.dumps(spdx_doc))
119
120def setup_foss_scan( info, cache, cached_files ):
121 import errno, shutil
122 import tarfile
123 file_info = {}
124 cache_dict = {}
125
126 for f_dir, f in list_files( info['sourcedir'] ):
127 full_path = os.path.join( f_dir, f )
128 abs_path = os.path.join(info['sourcedir'], full_path)
129 dest_dir = os.path.join( info['spdx_temp_dir'], f_dir )
130 dest_path = os.path.join( info['spdx_temp_dir'], full_path )
131 try:
132 stats = os.stat(abs_path)
133 except OSError as e:
134 bb.warn( "Stat failed" + str(e) + "\n")
135 continue
136
137 checksum = hash_file( abs_path )
138 mtime = time.asctime(time.localtime(stats.st_mtime))
139
140 ## retain cache information if it exists
141 file_info[checksum] = {}
142 if cache and checksum in cached_files:
143 file_info[checksum] = cached_files[checksum]
144 else:
145 file_info[checksum]['FileName'] = full_path
146
147 try:
148 os.makedirs( dest_dir )
149 except OSError as e:
150 if e.errno == errno.EEXIST and os.path.isdir(dest_dir):
151 pass
152 else:
153 bb.warn( "mkdir failed " + str(e) + "\n" )
154 continue
155
156 if(cache and checksum not in cached_files) or not cache:
157 try:
158 shutil.copyfile( abs_path, dest_path )
159 except shutil.Error as e:
160 bb.warn( str(e) + "\n" )
161 except IOError as e:
162 bb.warn( str(e) + "\n" )
163
164 with tarfile.open( info['tar_file'], "w:gz" ) as tar:
165 tar.add( info['spdx_temp_dir'], arcname=os.path.basename(info['spdx_temp_dir']) )
166 tar.close()
167
168 return file_info
169
170
171def remove_dir_tree( dir_name ):
172 import shutil
173 try:
174 shutil.rmtree( dir_name )
175 except:
176 pass
177
178def remove_file( file_name ):
179 try:
180 os.remove( file_name )
181 except OSError as e:
182 pass
183
184def list_files( dir ):
185 for root, subFolders, files in os.walk( dir ):
186 for f in files:
187 rel_root = os.path.relpath( root, dir )
188 yield rel_root, f
189 return
190
191def hash_file( file_name ):
192 try:
193 f = open( file_name, 'rb' )
194 data_string = f.read()
195 except:
196 return None
197 finally:
198 f.close()
199 sha1 = hash_string( data_string )
200 return sha1
201
202def hash_string( data ):
203 import hashlib
204 sha1 = hashlib.sha1()
205 sha1.update( data )
206 return sha1.hexdigest()
207
208def run_fossology( foss_command ):
209 import string, re
210 import subprocess
211
212 p = subprocess.Popen(foss_command.split(),
213 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
214 foss_output, foss_error = p.communicate()
215
216 records = []
217 records = re.findall('FileName:.*?</text>', foss_output, re.S)
218
219 file_info = {}
220 for rec in records:
221 rec = string.replace( rec, '\r', '' )
222 chksum = re.findall( 'FileChecksum: SHA1: (.*)\n', rec)[0]
223 file_info[chksum] = {}
224 file_info[chksum]['FileCopyrightText'] = re.findall( 'FileCopyrightText: '
225 + '(.*?</text>)', rec, re.S )[0]
226 fields = ['FileType','LicenseConcluded',
227 'LicenseInfoInFile','FileName']
228 for field in fields:
229 file_info[chksum][field] = re.findall(field + ': (.*)', rec)[0]
230
231 return file_info
232
233def create_spdx_doc( file_info, scanned_files ):
234 import json
235 ## push foss changes back into cache
236 for chksum, lic_info in scanned_files.iteritems():
237 if chksum in file_info:
238 file_info[chksum]['FileName'] = file_info[chksum]['FileName']
239 file_info[chksum]['FileType'] = lic_info['FileType']
240 file_info[chksum]['FileChecksum: SHA1'] = chksum
241 file_info[chksum]['LicenseInfoInFile'] = lic_info['LicenseInfoInFile']
242 file_info[chksum]['LicenseConcluded'] = lic_info['LicenseConcluded']
243 file_info[chksum]['FileCopyrightText'] = lic_info['FileCopyrightText']
244 else:
245 bb.warn(lic_info['FileName'] + " : " + chksum
246 + " : is not in the local file info: "
247 + json.dumps(lic_info,indent=1))
248 return file_info
249
250def get_ver_code( dirname ):
251 chksums = []
252 for f_dir, f in list_files( dirname ):
253 try:
254 stats = os.stat(os.path.join(dirname,f_dir,f))
255 except OSError as e:
256 bb.warn( "Stat failed" + str(e) + "\n")
257 continue
258 chksums.append(hash_file(os.path.join(dirname,f_dir,f)))
259 ver_code_string = ''.join( chksums ).lower()
260 ver_code = hash_string( ver_code_string )
261 return ver_code
262
263def get_header_info( info, spdx_verification_code, spdx_files ):
264 """
265 Put together the header SPDX information.
266 Eventually this needs to become a lot less
267 of a hardcoded thing.
268 """
269 from datetime import datetime
270 import os
271 head = []
272 DEFAULT = "NOASSERTION"
273
274 #spdx_verification_code = get_ver_code( info['sourcedir'] )
275 package_checksum = ''
276 if os.path.exists(info['tar_file']):
277 package_checksum = hash_file( info['tar_file'] )
278 else:
279 package_checksum = DEFAULT
280
281 ## document level information
282 head.append("SPDXVersion: " + info['spdx_version'])
283 head.append("DataLicense: " + info['data_license'])
284 head.append("DocumentComment: <text>SPDX for "
285 + info['pn'] + " version " + info['pv'] + "</text>")
286 head.append("")
287
288 ## Creator information
289 now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
290 head.append("## Creation Information")
291 head.append("Creator: fossology-spdx")
292 head.append("Created: " + now)
293 head.append("CreatorComment: <text>UNO</text>")
294 head.append("")
295
296 ## package level information
297 head.append("## Package Information")
298 head.append("PackageName: " + info['pn'])
299 head.append("PackageVersion: " + info['pv'])
300 head.append("PackageDownloadLocation: " + DEFAULT)
301 head.append("PackageSummary: <text></text>")
302 head.append("PackageFileName: " + os.path.basename(info['tar_file']))
303 head.append("PackageSupplier: Person:" + DEFAULT)
304 head.append("PackageOriginator: Person:" + DEFAULT)
305 head.append("PackageChecksum: SHA1: " + package_checksum)
306 head.append("PackageVerificationCode: " + spdx_verification_code)
307 head.append("PackageDescription: <text>" + info['pn']
308 + " version " + info['pv'] + "</text>")
309 head.append("")
310 head.append("PackageCopyrightText: <text>" + DEFAULT + "</text>")
311 head.append("")
312 head.append("PackageLicenseDeclared: " + DEFAULT)
313 head.append("PackageLicenseConcluded: " + DEFAULT)
314 head.append("PackageLicenseInfoFromFiles: " + DEFAULT)
315 head.append("")
316
317 ## header for file level
318 head.append("## File Information")
319 head.append("")
320
321 return '\n'.join(head)
diff --git a/meta/conf/licenses.conf b/meta/conf/licenses.conf
index 922b84c924..b41d0a89a9 100644
--- a/meta/conf/licenses.conf
+++ b/meta/conf/licenses.conf
@@ -113,6 +113,55 @@ SPDXLICENSEMAP[SGIv1] = "SGI-1"
113# Set if you want the license.manifest copied to the image 113# Set if you want the license.manifest copied to the image
114#COPY_LIC_MANIFEST = "1" 114#COPY_LIC_MANIFEST = "1"
115 115
116# If you want the pkg licenses copied over as well you must set 116# If you want the pkg licenses copied over as well you must set
117# both COPY_LIC_MANIFEST and COPY_LIC_DIRS 117# both COPY_LIC_MANIFEST and COPY_LIC_DIRS
118#COPY_LIC_DIRS = "1" 118#COPY_LIC_DIRS = "1"
119
120## SPDX temporary directory
121SPDX_TEMP_DIR = "${WORKDIR}/spdx_temp"
122SPDX_MANIFEST_DIR = "/home/yocto/fossology_scans"
123
124## SPDX Format info
125SPDX_VERSION = "SPDX-1.1"
126DATA_LICENSE = "CC0-1.0"
127
128## Fossology scan information
129# You can set option to control if the copyright information will be skipped
130# during the identification process.
131#
132# It is defined as [FOSS_COPYRIGHT] in ./meta/conf/licenses.conf.
133# FOSS_COPYRIGHT = "true"
134# NO copyright will be processed. That means only license information will be
135# identified and output to SPDX file
136# FOSS_COPYRIGHT = "false"
137# Copyright will be identified and output to SPDX file along with license
138# information. The process will take more time than not processing copyright
139# information.
140#
141
142FOSS_COPYRIGHT = "true"
143
144# A option defined as[FOSS_RECURSIVE_UNPACK] in ./meta/conf/licenses.conf. is
145# used to control if FOSSology server need recursively unpack tar.gz file which
146# is sent from do_spdx task.
147#
148# FOSS_RECURSIVE_UNPACK = "false":
149# FOSSology server does NOT recursively unpack. In the current release, this
150# is the default choice because recursively unpack will not necessarily break
151# down original compressed files.
152# FOSS_RECURSIVE_UNPACK = "true":
153# FOSSology server recursively unpack components.
154#
155
156FOSS_RECURSIVE_UNPACK = "false"
157
158# FOSSologySPDX instance server.
159# For more information on FOSSologySPDX commandline:
160# https://github.com/spdx-tools/fossology-spdx/wiki/Fossology-SPDX-Web-API
161#
162
163FOSS_SERVER = "http://localhost//?mod=spdx_license_once&noCopyright=${FOSS_COPYRIGHT}&recursiveUnpack=${FOSS_RECURSIVE_UNPACK}"
164
165FOSS_WGET_FLAGS = "-qO - --no-check-certificate --timeout=0"
166
167