summaryrefslogtreecommitdiffstats
path: root/meta/classes/spdx.bbclass
diff options
context:
space:
mode:
Diffstat (limited to 'meta/classes/spdx.bbclass')
-rw-r--r--meta/classes/spdx.bbclass325
1 files changed, 325 insertions, 0 deletions
diff --git a/meta/classes/spdx.bbclass b/meta/classes/spdx.bbclass
new file mode 100644
index 0000000000..bccc230d8c
--- /dev/null
+++ b/meta/classes/spdx.bbclass
@@ -0,0 +1,325 @@
1# This class integrates real-time license scanning, generation of SPDX standard
2# output and verifiying license info during the building process.
3# It is a combination of efforts from the OE-Core, SPDX and Fossology projects.
4#
5# For more information on FOSSology:
6# http://www.fossology.org
7#
8# For more information on FOSSologySPDX commandline:
9# https://github.com/spdx-tools/fossology-spdx/wiki/Fossology-SPDX-Web-API
10#
11# For more information on SPDX:
12# http://www.spdx.org
13#
14
15# SPDX file will be output to the path which is defined as[SPDX_MANIFEST_DIR]
16# in ./meta/conf/licenses.conf.
17
18SPDXOUTPUTDIR = "${WORKDIR}/spdx_output_dir"
19SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir"
20
21# If ${S} isn't actually the top-level source directory, set SPDX_S to point at
22# the real top-level directory.
23SPDX_S ?= "${S}"
24
25python do_spdx () {
26 import os, sys
27 import json
28
29 info = {}
30 info['workdir'] = (d.getVar('WORKDIR', True) or "")
31 info['sourcedir'] = (d.getVar('SPDX_S', True) or "")
32 info['pn'] = (d.getVar( 'PN', True ) or "")
33 info['pv'] = (d.getVar( 'PV', True ) or "")
34 info['src_uri'] = (d.getVar( 'SRC_URI', True ) or "")
35 info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '')
36 info['data_license'] = (d.getVar('DATA_LICENSE', True) or '')
37
38 spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "")
39 manifest_dir = (d.getVar('SPDX_MANIFEST_DIR', True) or "")
40 info['outfile'] = os.path.join(manifest_dir, info['pn'] + ".spdx" )
41 sstatefile = os.path.join(spdx_sstate_dir,
42 info['pn'] + info['pv'] + ".spdx" )
43 info['spdx_temp_dir'] = (d.getVar('SPDX_TEMP_DIR', True) or "")
44 info['tar_file'] = os.path.join( info['workdir'], info['pn'] + ".tar.gz" )
45
46
47 ## get everything from cache. use it to decide if
48 ## something needs to be rerun
49 cur_ver_code = get_ver_code( info['sourcedir'] )
50 cache_cur = False
51 if not os.path.exists( spdx_sstate_dir ):
52 bb.utils.mkdirhier( spdx_sstate_dir )
53 if not os.path.exists( info['spdx_temp_dir'] ):
54 bb.utils.mkdirhier( info['spdx_temp_dir'] )
55 if os.path.exists( sstatefile ):
56 ## cache for this package exists. read it in
57 cached_spdx = get_cached_spdx( sstatefile )
58
59 if cached_spdx['PackageVerificationCode'] == cur_ver_code:
60 bb.warn(info['pn'] + "'s ver code same as cache's. do nothing")
61 cache_cur = True
62 else:
63 local_file_info = setup_foss_scan( info,
64 True, cached_spdx['Files'] )
65 else:
66 local_file_info = setup_foss_scan( info, False, None )
67
68 if cache_cur:
69 spdx_file_info = cached_spdx['Files']
70 else:
71 ## setup fossology command
72 foss_server = (d.getVar('FOSS_SERVER', True) or "")
73 foss_flags = (d.getVar('FOSS_WGET_FLAGS', True) or "")
74 foss_command = "wget %s --post-file=%s %s"\
75 % (foss_flags,info['tar_file'],foss_server)
76
77 #bb.warn(info['pn'] + json.dumps(local_file_info))
78 foss_file_info = run_fossology( foss_command )
79 spdx_file_info = create_spdx_doc( local_file_info, foss_file_info )
80 ## write to cache
81 write_cached_spdx(sstatefile,cur_ver_code,spdx_file_info)
82
83 ## Get document and package level information
84 spdx_header_info = get_header_info(info, cur_ver_code, spdx_file_info)
85
86 ## CREATE MANIFEST
87 create_manifest(info,spdx_header_info,spdx_file_info)
88
89 ## clean up the temp stuff
90 remove_dir_tree( info['spdx_temp_dir'] )
91 if os.path.exists(info['tar_file']):
92 remove_file( info['tar_file'] )
93}
94addtask spdx after do_patch before do_configure
95
96def create_manifest(info,header,files):
97 with open(info['outfile'], 'w') as f:
98 f.write(header + '\n')
99 for chksum, block in files.iteritems():
100 for key, value in block.iteritems():
101 f.write(key + ": " + value)
102 f.write('\n')
103 f.write('\n')
104
105def get_cached_spdx( sstatefile ):
106 import json
107 cached_spdx_info = {}
108 with open( sstatefile, 'r' ) as f:
109 try:
110 cached_spdx_info = json.load(f)
111 except ValueError as e:
112 cached_spdx_info = None
113 return cached_spdx_info
114
115def write_cached_spdx( sstatefile, ver_code, files ):
116 import json
117 spdx_doc = {}
118 spdx_doc['PackageVerificationCode'] = ver_code
119 spdx_doc['Files'] = {}
120 spdx_doc['Files'] = files
121 with open( sstatefile, 'w' ) as f:
122 f.write(json.dumps(spdx_doc))
123
124def setup_foss_scan( info, cache, cached_files ):
125 import errno, shutil
126 import tarfile
127 file_info = {}
128 cache_dict = {}
129
130 for f_dir, f in list_files( info['sourcedir'] ):
131 full_path = os.path.join( f_dir, f )
132 abs_path = os.path.join(info['sourcedir'], full_path)
133 dest_dir = os.path.join( info['spdx_temp_dir'], f_dir )
134 dest_path = os.path.join( info['spdx_temp_dir'], full_path )
135 try:
136 stats = os.stat(abs_path)
137 except OSError as e:
138 bb.warn( "Stat failed" + str(e) + "\n")
139 continue
140
141 checksum = hash_file( abs_path )
142 mtime = time.asctime(time.localtime(stats.st_mtime))
143
144 ## retain cache information if it exists
145 file_info[checksum] = {}
146 if cache and checksum in cached_files:
147 file_info[checksum] = cached_files[checksum]
148 else:
149 file_info[checksum]['FileName'] = full_path
150
151 try:
152 os.makedirs( dest_dir )
153 except OSError as e:
154 if e.errno == errno.EEXIST and os.path.isdir(dest_dir):
155 pass
156 else:
157 bb.warn( "mkdir failed " + str(e) + "\n" )
158 continue
159
160 if(cache and checksum not in cached_files) or not cache:
161 try:
162 shutil.copyfile( abs_path, dest_path )
163 except shutil.Error as e:
164 bb.warn( str(e) + "\n" )
165 except IOError as e:
166 bb.warn( str(e) + "\n" )
167
168 with tarfile.open( info['tar_file'], "w:gz" ) as tar:
169 tar.add( info['spdx_temp_dir'], arcname=os.path.basename(info['spdx_temp_dir']) )
170 tar.close()
171
172 return file_info
173
174
175def remove_dir_tree( dir_name ):
176 import shutil
177 try:
178 shutil.rmtree( dir_name )
179 except:
180 pass
181
182def remove_file( file_name ):
183 try:
184 os.remove( file_name )
185 except OSError as e:
186 pass
187
188def list_files( dir ):
189 for root, subFolders, files in os.walk( dir ):
190 for f in files:
191 rel_root = os.path.relpath( root, dir )
192 yield rel_root, f
193 return
194
195def hash_file( file_name ):
196 try:
197 f = open( file_name, 'rb' )
198 data_string = f.read()
199 except:
200 return None
201 finally:
202 f.close()
203 sha1 = hash_string( data_string )
204 return sha1
205
206def hash_string( data ):
207 import hashlib
208 sha1 = hashlib.sha1()
209 sha1.update( data )
210 return sha1.hexdigest()
211
212def run_fossology( foss_command ):
213 import string, re
214 import subprocess
215
216 p = subprocess.Popen(foss_command.split(),
217 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
218 foss_output, foss_error = p.communicate()
219
220 records = []
221 records = re.findall('FileName:.*?</text>', foss_output, re.S)
222
223 file_info = {}
224 for rec in records:
225 rec = string.replace( rec, '\r', '' )
226 chksum = re.findall( 'FileChecksum: SHA1: (.*)\n', rec)[0]
227 file_info[chksum] = {}
228 file_info[chksum]['FileCopyrightText'] = re.findall( 'FileCopyrightText: '
229 + '(.*?</text>)', rec, re.S )[0]
230 fields = ['FileType','LicenseConcluded',
231 'LicenseInfoInFile','FileName']
232 for field in fields:
233 file_info[chksum][field] = re.findall(field + ': (.*)', rec)[0]
234
235 return file_info
236
237def create_spdx_doc( file_info, scanned_files ):
238 import json
239 ## push foss changes back into cache
240 for chksum, lic_info in scanned_files.iteritems():
241 if chksum in file_info:
242 file_info[chksum]['FileName'] = file_info[chksum]['FileName']
243 file_info[chksum]['FileType'] = lic_info['FileType']
244 file_info[chksum]['FileChecksum: SHA1'] = chksum
245 file_info[chksum]['LicenseInfoInFile'] = lic_info['LicenseInfoInFile']
246 file_info[chksum]['LicenseConcluded'] = lic_info['LicenseConcluded']
247 file_info[chksum]['FileCopyrightText'] = lic_info['FileCopyrightText']
248 else:
249 bb.warn(lic_info['FileName'] + " : " + chksum
250 + " : is not in the local file info: "
251 + json.dumps(lic_info,indent=1))
252 return file_info
253
254def get_ver_code( dirname ):
255 chksums = []
256 for f_dir, f in list_files( dirname ):
257 try:
258 stats = os.stat(os.path.join(dirname,f_dir,f))
259 except OSError as e:
260 bb.warn( "Stat failed" + str(e) + "\n")
261 continue
262 chksums.append(hash_file(os.path.join(dirname,f_dir,f)))
263 ver_code_string = ''.join( chksums ).lower()
264 ver_code = hash_string( ver_code_string )
265 return ver_code
266
267def get_header_info( info, spdx_verification_code, spdx_files ):
268 """
269 Put together the header SPDX information.
270 Eventually this needs to become a lot less
271 of a hardcoded thing.
272 """
273 from datetime import datetime
274 import os
275 head = []
276 DEFAULT = "NOASSERTION"
277
278 #spdx_verification_code = get_ver_code( info['sourcedir'] )
279 package_checksum = ''
280 if os.path.exists(info['tar_file']):
281 package_checksum = hash_file( info['tar_file'] )
282 else:
283 package_checksum = DEFAULT
284
285 ## document level information
286 head.append("SPDXVersion: " + info['spdx_version'])
287 head.append("DataLicense: " + info['data_license'])
288 head.append("DocumentComment: <text>SPDX for "
289 + info['pn'] + " version " + info['pv'] + "</text>")
290 head.append("")
291
292 ## Creator information
293 now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
294 head.append("## Creation Information")
295 head.append("Creator: fossology-spdx")
296 head.append("Created: " + now)
297 head.append("CreatorComment: <text>UNO</text>")
298 head.append("")
299
300 ## package level information
301 head.append("## Package Information")
302 head.append("PackageName: " + info['pn'])
303 head.append("PackageVersion: " + info['pv'])
304 head.append("PackageDownloadLocation: " + DEFAULT)
305 head.append("PackageSummary: <text></text>")
306 head.append("PackageFileName: " + os.path.basename(info['tar_file']))
307 head.append("PackageSupplier: Person:" + DEFAULT)
308 head.append("PackageOriginator: Person:" + DEFAULT)
309 head.append("PackageChecksum: SHA1: " + package_checksum)
310 head.append("PackageVerificationCode: " + spdx_verification_code)
311 head.append("PackageDescription: <text>" + info['pn']
312 + " version " + info['pv'] + "</text>")
313 head.append("")
314 head.append("PackageCopyrightText: <text>" + DEFAULT + "</text>")
315 head.append("")
316 head.append("PackageLicenseDeclared: " + DEFAULT)
317 head.append("PackageLicenseConcluded: " + DEFAULT)
318 head.append("PackageLicenseInfoFromFiles: " + DEFAULT)
319 head.append("")
320
321 ## header for file level
322 head.append("## File Information")
323 head.append("")
324
325 return '\n'.join(head)