diff options
author | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
---|---|---|
committer | Tudor Florea <tudor.florea@enea.com> | 2015-10-09 22:59:03 +0200 |
commit | 972dcfcdbfe75dcfeb777150c136576cf1a71e99 (patch) | |
tree | 97a61cd7e293d7ae9d56ef7ed0f81253365bb026 /meta/classes/spdx.bbclass | |
download | poky-972dcfcdbfe75dcfeb777150c136576cf1a71e99.tar.gz |
initial commit for Enea Linux 5.0 arm
Signed-off-by: Tudor Florea <tudor.florea@enea.com>
Diffstat (limited to 'meta/classes/spdx.bbclass')
-rw-r--r-- | meta/classes/spdx.bbclass | 325 |
1 files changed, 325 insertions, 0 deletions
diff --git a/meta/classes/spdx.bbclass b/meta/classes/spdx.bbclass new file mode 100644 index 0000000000..bccc230d8c --- /dev/null +++ b/meta/classes/spdx.bbclass | |||
@@ -0,0 +1,325 @@ | |||
1 | # This class integrates real-time license scanning, generation of SPDX standard | ||
2 | # output and verifiying license info during the building process. | ||
3 | # It is a combination of efforts from the OE-Core, SPDX and Fossology projects. | ||
4 | # | ||
5 | # For more information on FOSSology: | ||
6 | # http://www.fossology.org | ||
7 | # | ||
8 | # For more information on FOSSologySPDX commandline: | ||
9 | # https://github.com/spdx-tools/fossology-spdx/wiki/Fossology-SPDX-Web-API | ||
10 | # | ||
11 | # For more information on SPDX: | ||
12 | # http://www.spdx.org | ||
13 | # | ||
14 | |||
15 | # SPDX file will be output to the path which is defined as[SPDX_MANIFEST_DIR] | ||
16 | # in ./meta/conf/licenses.conf. | ||
17 | |||
18 | SPDXOUTPUTDIR = "${WORKDIR}/spdx_output_dir" | ||
19 | SPDXSSTATEDIR = "${WORKDIR}/spdx_sstate_dir" | ||
20 | |||
21 | # If ${S} isn't actually the top-level source directory, set SPDX_S to point at | ||
22 | # the real top-level directory. | ||
23 | SPDX_S ?= "${S}" | ||
24 | |||
25 | python do_spdx () { | ||
26 | import os, sys | ||
27 | import json | ||
28 | |||
29 | info = {} | ||
30 | info['workdir'] = (d.getVar('WORKDIR', True) or "") | ||
31 | info['sourcedir'] = (d.getVar('SPDX_S', True) or "") | ||
32 | info['pn'] = (d.getVar( 'PN', True ) or "") | ||
33 | info['pv'] = (d.getVar( 'PV', True ) or "") | ||
34 | info['src_uri'] = (d.getVar( 'SRC_URI', True ) or "") | ||
35 | info['spdx_version'] = (d.getVar('SPDX_VERSION', True) or '') | ||
36 | info['data_license'] = (d.getVar('DATA_LICENSE', True) or '') | ||
37 | |||
38 | spdx_sstate_dir = (d.getVar('SPDXSSTATEDIR', True) or "") | ||
39 | manifest_dir = (d.getVar('SPDX_MANIFEST_DIR', True) or "") | ||
40 | info['outfile'] = os.path.join(manifest_dir, info['pn'] + ".spdx" ) | ||
41 | sstatefile = os.path.join(spdx_sstate_dir, | ||
42 | info['pn'] + info['pv'] + ".spdx" ) | ||
43 | info['spdx_temp_dir'] = (d.getVar('SPDX_TEMP_DIR', True) or "") | ||
44 | info['tar_file'] = os.path.join( info['workdir'], info['pn'] + ".tar.gz" ) | ||
45 | |||
46 | |||
47 | ## get everything from cache. use it to decide if | ||
48 | ## something needs to be rerun | ||
49 | cur_ver_code = get_ver_code( info['sourcedir'] ) | ||
50 | cache_cur = False | ||
51 | if not os.path.exists( spdx_sstate_dir ): | ||
52 | bb.utils.mkdirhier( spdx_sstate_dir ) | ||
53 | if not os.path.exists( info['spdx_temp_dir'] ): | ||
54 | bb.utils.mkdirhier( info['spdx_temp_dir'] ) | ||
55 | if os.path.exists( sstatefile ): | ||
56 | ## cache for this package exists. read it in | ||
57 | cached_spdx = get_cached_spdx( sstatefile ) | ||
58 | |||
59 | if cached_spdx['PackageVerificationCode'] == cur_ver_code: | ||
60 | bb.warn(info['pn'] + "'s ver code same as cache's. do nothing") | ||
61 | cache_cur = True | ||
62 | else: | ||
63 | local_file_info = setup_foss_scan( info, | ||
64 | True, cached_spdx['Files'] ) | ||
65 | else: | ||
66 | local_file_info = setup_foss_scan( info, False, None ) | ||
67 | |||
68 | if cache_cur: | ||
69 | spdx_file_info = cached_spdx['Files'] | ||
70 | else: | ||
71 | ## setup fossology command | ||
72 | foss_server = (d.getVar('FOSS_SERVER', True) or "") | ||
73 | foss_flags = (d.getVar('FOSS_WGET_FLAGS', True) or "") | ||
74 | foss_command = "wget %s --post-file=%s %s"\ | ||
75 | % (foss_flags,info['tar_file'],foss_server) | ||
76 | |||
77 | #bb.warn(info['pn'] + json.dumps(local_file_info)) | ||
78 | foss_file_info = run_fossology( foss_command ) | ||
79 | spdx_file_info = create_spdx_doc( local_file_info, foss_file_info ) | ||
80 | ## write to cache | ||
81 | write_cached_spdx(sstatefile,cur_ver_code,spdx_file_info) | ||
82 | |||
83 | ## Get document and package level information | ||
84 | spdx_header_info = get_header_info(info, cur_ver_code, spdx_file_info) | ||
85 | |||
86 | ## CREATE MANIFEST | ||
87 | create_manifest(info,spdx_header_info,spdx_file_info) | ||
88 | |||
89 | ## clean up the temp stuff | ||
90 | remove_dir_tree( info['spdx_temp_dir'] ) | ||
91 | if os.path.exists(info['tar_file']): | ||
92 | remove_file( info['tar_file'] ) | ||
93 | } | ||
94 | addtask spdx after do_patch before do_configure | ||
95 | |||
96 | def create_manifest(info,header,files): | ||
97 | with open(info['outfile'], 'w') as f: | ||
98 | f.write(header + '\n') | ||
99 | for chksum, block in files.iteritems(): | ||
100 | for key, value in block.iteritems(): | ||
101 | f.write(key + ": " + value) | ||
102 | f.write('\n') | ||
103 | f.write('\n') | ||
104 | |||
105 | def get_cached_spdx( sstatefile ): | ||
106 | import json | ||
107 | cached_spdx_info = {} | ||
108 | with open( sstatefile, 'r' ) as f: | ||
109 | try: | ||
110 | cached_spdx_info = json.load(f) | ||
111 | except ValueError as e: | ||
112 | cached_spdx_info = None | ||
113 | return cached_spdx_info | ||
114 | |||
115 | def write_cached_spdx( sstatefile, ver_code, files ): | ||
116 | import json | ||
117 | spdx_doc = {} | ||
118 | spdx_doc['PackageVerificationCode'] = ver_code | ||
119 | spdx_doc['Files'] = {} | ||
120 | spdx_doc['Files'] = files | ||
121 | with open( sstatefile, 'w' ) as f: | ||
122 | f.write(json.dumps(spdx_doc)) | ||
123 | |||
124 | def setup_foss_scan( info, cache, cached_files ): | ||
125 | import errno, shutil | ||
126 | import tarfile | ||
127 | file_info = {} | ||
128 | cache_dict = {} | ||
129 | |||
130 | for f_dir, f in list_files( info['sourcedir'] ): | ||
131 | full_path = os.path.join( f_dir, f ) | ||
132 | abs_path = os.path.join(info['sourcedir'], full_path) | ||
133 | dest_dir = os.path.join( info['spdx_temp_dir'], f_dir ) | ||
134 | dest_path = os.path.join( info['spdx_temp_dir'], full_path ) | ||
135 | try: | ||
136 | stats = os.stat(abs_path) | ||
137 | except OSError as e: | ||
138 | bb.warn( "Stat failed" + str(e) + "\n") | ||
139 | continue | ||
140 | |||
141 | checksum = hash_file( abs_path ) | ||
142 | mtime = time.asctime(time.localtime(stats.st_mtime)) | ||
143 | |||
144 | ## retain cache information if it exists | ||
145 | file_info[checksum] = {} | ||
146 | if cache and checksum in cached_files: | ||
147 | file_info[checksum] = cached_files[checksum] | ||
148 | else: | ||
149 | file_info[checksum]['FileName'] = full_path | ||
150 | |||
151 | try: | ||
152 | os.makedirs( dest_dir ) | ||
153 | except OSError as e: | ||
154 | if e.errno == errno.EEXIST and os.path.isdir(dest_dir): | ||
155 | pass | ||
156 | else: | ||
157 | bb.warn( "mkdir failed " + str(e) + "\n" ) | ||
158 | continue | ||
159 | |||
160 | if(cache and checksum not in cached_files) or not cache: | ||
161 | try: | ||
162 | shutil.copyfile( abs_path, dest_path ) | ||
163 | except shutil.Error as e: | ||
164 | bb.warn( str(e) + "\n" ) | ||
165 | except IOError as e: | ||
166 | bb.warn( str(e) + "\n" ) | ||
167 | |||
168 | with tarfile.open( info['tar_file'], "w:gz" ) as tar: | ||
169 | tar.add( info['spdx_temp_dir'], arcname=os.path.basename(info['spdx_temp_dir']) ) | ||
170 | tar.close() | ||
171 | |||
172 | return file_info | ||
173 | |||
174 | |||
175 | def remove_dir_tree( dir_name ): | ||
176 | import shutil | ||
177 | try: | ||
178 | shutil.rmtree( dir_name ) | ||
179 | except: | ||
180 | pass | ||
181 | |||
182 | def remove_file( file_name ): | ||
183 | try: | ||
184 | os.remove( file_name ) | ||
185 | except OSError as e: | ||
186 | pass | ||
187 | |||
188 | def list_files( dir ): | ||
189 | for root, subFolders, files in os.walk( dir ): | ||
190 | for f in files: | ||
191 | rel_root = os.path.relpath( root, dir ) | ||
192 | yield rel_root, f | ||
193 | return | ||
194 | |||
195 | def hash_file( file_name ): | ||
196 | try: | ||
197 | f = open( file_name, 'rb' ) | ||
198 | data_string = f.read() | ||
199 | except: | ||
200 | return None | ||
201 | finally: | ||
202 | f.close() | ||
203 | sha1 = hash_string( data_string ) | ||
204 | return sha1 | ||
205 | |||
206 | def hash_string( data ): | ||
207 | import hashlib | ||
208 | sha1 = hashlib.sha1() | ||
209 | sha1.update( data ) | ||
210 | return sha1.hexdigest() | ||
211 | |||
212 | def run_fossology( foss_command ): | ||
213 | import string, re | ||
214 | import subprocess | ||
215 | |||
216 | p = subprocess.Popen(foss_command.split(), | ||
217 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) | ||
218 | foss_output, foss_error = p.communicate() | ||
219 | |||
220 | records = [] | ||
221 | records = re.findall('FileName:.*?</text>', foss_output, re.S) | ||
222 | |||
223 | file_info = {} | ||
224 | for rec in records: | ||
225 | rec = string.replace( rec, '\r', '' ) | ||
226 | chksum = re.findall( 'FileChecksum: SHA1: (.*)\n', rec)[0] | ||
227 | file_info[chksum] = {} | ||
228 | file_info[chksum]['FileCopyrightText'] = re.findall( 'FileCopyrightText: ' | ||
229 | + '(.*?</text>)', rec, re.S )[0] | ||
230 | fields = ['FileType','LicenseConcluded', | ||
231 | 'LicenseInfoInFile','FileName'] | ||
232 | for field in fields: | ||
233 | file_info[chksum][field] = re.findall(field + ': (.*)', rec)[0] | ||
234 | |||
235 | return file_info | ||
236 | |||
237 | def create_spdx_doc( file_info, scanned_files ): | ||
238 | import json | ||
239 | ## push foss changes back into cache | ||
240 | for chksum, lic_info in scanned_files.iteritems(): | ||
241 | if chksum in file_info: | ||
242 | file_info[chksum]['FileName'] = file_info[chksum]['FileName'] | ||
243 | file_info[chksum]['FileType'] = lic_info['FileType'] | ||
244 | file_info[chksum]['FileChecksum: SHA1'] = chksum | ||
245 | file_info[chksum]['LicenseInfoInFile'] = lic_info['LicenseInfoInFile'] | ||
246 | file_info[chksum]['LicenseConcluded'] = lic_info['LicenseConcluded'] | ||
247 | file_info[chksum]['FileCopyrightText'] = lic_info['FileCopyrightText'] | ||
248 | else: | ||
249 | bb.warn(lic_info['FileName'] + " : " + chksum | ||
250 | + " : is not in the local file info: " | ||
251 | + json.dumps(lic_info,indent=1)) | ||
252 | return file_info | ||
253 | |||
254 | def get_ver_code( dirname ): | ||
255 | chksums = [] | ||
256 | for f_dir, f in list_files( dirname ): | ||
257 | try: | ||
258 | stats = os.stat(os.path.join(dirname,f_dir,f)) | ||
259 | except OSError as e: | ||
260 | bb.warn( "Stat failed" + str(e) + "\n") | ||
261 | continue | ||
262 | chksums.append(hash_file(os.path.join(dirname,f_dir,f))) | ||
263 | ver_code_string = ''.join( chksums ).lower() | ||
264 | ver_code = hash_string( ver_code_string ) | ||
265 | return ver_code | ||
266 | |||
267 | def get_header_info( info, spdx_verification_code, spdx_files ): | ||
268 | """ | ||
269 | Put together the header SPDX information. | ||
270 | Eventually this needs to become a lot less | ||
271 | of a hardcoded thing. | ||
272 | """ | ||
273 | from datetime import datetime | ||
274 | import os | ||
275 | head = [] | ||
276 | DEFAULT = "NOASSERTION" | ||
277 | |||
278 | #spdx_verification_code = get_ver_code( info['sourcedir'] ) | ||
279 | package_checksum = '' | ||
280 | if os.path.exists(info['tar_file']): | ||
281 | package_checksum = hash_file( info['tar_file'] ) | ||
282 | else: | ||
283 | package_checksum = DEFAULT | ||
284 | |||
285 | ## document level information | ||
286 | head.append("SPDXVersion: " + info['spdx_version']) | ||
287 | head.append("DataLicense: " + info['data_license']) | ||
288 | head.append("DocumentComment: <text>SPDX for " | ||
289 | + info['pn'] + " version " + info['pv'] + "</text>") | ||
290 | head.append("") | ||
291 | |||
292 | ## Creator information | ||
293 | now = datetime.now().strftime('%Y-%m-%dT%H:%M:%S') | ||
294 | head.append("## Creation Information") | ||
295 | head.append("Creator: fossology-spdx") | ||
296 | head.append("Created: " + now) | ||
297 | head.append("CreatorComment: <text>UNO</text>") | ||
298 | head.append("") | ||
299 | |||
300 | ## package level information | ||
301 | head.append("## Package Information") | ||
302 | head.append("PackageName: " + info['pn']) | ||
303 | head.append("PackageVersion: " + info['pv']) | ||
304 | head.append("PackageDownloadLocation: " + DEFAULT) | ||
305 | head.append("PackageSummary: <text></text>") | ||
306 | head.append("PackageFileName: " + os.path.basename(info['tar_file'])) | ||
307 | head.append("PackageSupplier: Person:" + DEFAULT) | ||
308 | head.append("PackageOriginator: Person:" + DEFAULT) | ||
309 | head.append("PackageChecksum: SHA1: " + package_checksum) | ||
310 | head.append("PackageVerificationCode: " + spdx_verification_code) | ||
311 | head.append("PackageDescription: <text>" + info['pn'] | ||
312 | + " version " + info['pv'] + "</text>") | ||
313 | head.append("") | ||
314 | head.append("PackageCopyrightText: <text>" + DEFAULT + "</text>") | ||
315 | head.append("") | ||
316 | head.append("PackageLicenseDeclared: " + DEFAULT) | ||
317 | head.append("PackageLicenseConcluded: " + DEFAULT) | ||
318 | head.append("PackageLicenseInfoFromFiles: " + DEFAULT) | ||
319 | head.append("") | ||
320 | |||
321 | ## header for file level | ||
322 | head.append("## File Information") | ||
323 | head.append("") | ||
324 | |||
325 | return '\n'.join(head) | ||