summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/classes/sstate.bbclass80
-rw-r--r--meta/lib/oe/sstatesig.py13
2 files changed, 45 insertions, 48 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index a0ca199867..11222223a9 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -823,29 +823,26 @@ sstate_unpack_package () {
823 823
824BB_HASHCHECK_FUNCTION = "sstate_checkhashes" 824BB_HASHCHECK_FUNCTION = "sstate_checkhashes"
825 825
826def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *, sq_unihash=None): 826def sstate_checkhashes(sq_data, d, siginfo=False, currentcount=0, **kwargs):
827 827 found = set()
828 ret = [] 828 missed = set()
829 missed = []
830 extension = ".tgz" 829 extension = ".tgz"
831 if siginfo: 830 if siginfo:
832 extension = extension + ".siginfo" 831 extension = extension + ".siginfo"
833 832
834 def gethash(task): 833 def gethash(task):
835 if sq_unihash is not None: 834 return sq_data['unihash'][task]
836 return sq_unihash[task]
837 return sq_hash[task]
838 835
839 def getpathcomponents(task, d): 836 def getpathcomponents(task, d):
840 # Magic data from BB_HASHFILENAME 837 # Magic data from BB_HASHFILENAME
841 splithashfn = sq_hashfn[task].split(" ") 838 splithashfn = sq_data['hashfn'][task].split(" ")
842 spec = splithashfn[1] 839 spec = splithashfn[1]
843 if splithashfn[0] == "True": 840 if splithashfn[0] == "True":
844 extrapath = d.getVar("NATIVELSBSTRING") + "/" 841 extrapath = d.getVar("NATIVELSBSTRING") + "/"
845 else: 842 else:
846 extrapath = "" 843 extrapath = ""
847 844
848 tname = sq_task[task][3:] 845 tname = bb.runqueue.taskname_from_tid(task)[3:]
849 846
850 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]: 847 if tname in ["fetch", "unpack", "patch", "populate_lic", "preconfigure"] and splithashfn[2]:
851 spec = splithashfn[2] 848 spec = splithashfn[2]
@@ -854,18 +851,18 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
854 return spec, extrapath, tname 851 return spec, extrapath, tname
855 852
856 853
857 for task in range(len(sq_fn)): 854 for tid in sq_data['hash']:
858 855
859 spec, extrapath, tname = getpathcomponents(task, d) 856 spec, extrapath, tname = getpathcomponents(tid, d)
860 857
861 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension) 858 sstatefile = d.expand("${SSTATE_DIR}/" + extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + extension)
862 859
863 if os.path.exists(sstatefile): 860 if os.path.exists(sstatefile):
864 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile) 861 bb.debug(2, "SState: Found valid sstate file %s" % sstatefile)
865 ret.append(task) 862 found.add(tid)
866 continue 863 continue
867 else: 864 else:
868 missed.append(task) 865 missed.add(tid)
869 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile) 866 bb.debug(2, "SState: Looked for but didn't find file %s" % sstatefile)
870 867
871 mirrors = d.getVar("SSTATE_MIRRORS") 868 mirrors = d.getVar("SSTATE_MIRRORS")
@@ -895,7 +892,7 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
895 thread_worker.connection_cache.close_connections() 892 thread_worker.connection_cache.close_connections()
896 893
897 def checkstatus(thread_worker, arg): 894 def checkstatus(thread_worker, arg):
898 (task, sstatefile) = arg 895 (tid, sstatefile) = arg
899 896
900 localdata2 = bb.data.createCopy(localdata) 897 localdata2 = bb.data.createCopy(localdata)
901 srcuri = "file://" + sstatefile 898 srcuri = "file://" + sstatefile
@@ -907,22 +904,22 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
907 connection_cache=thread_worker.connection_cache) 904 connection_cache=thread_worker.connection_cache)
908 fetcher.checkstatus() 905 fetcher.checkstatus()
909 bb.debug(2, "SState: Successful fetch test for %s" % srcuri) 906 bb.debug(2, "SState: Successful fetch test for %s" % srcuri)
910 ret.append(task) 907 found.add(tid)
911 if task in missed: 908 if tid in missed:
912 missed.remove(task) 909 missed.remove(tid)
913 except: 910 except:
914 missed.append(task) 911 missed.add(tid)
915 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri) 912 bb.debug(2, "SState: Unsuccessful fetch test for %s" % srcuri)
916 pass 913 pass
917 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d) 914 bb.event.fire(bb.event.ProcessProgress(msg, len(tasklist) - thread_worker.tasks.qsize()), d)
918 915
919 tasklist = [] 916 tasklist = []
920 for task in range(len(sq_fn)): 917 for tid in sq_data['hash']:
921 if task in ret: 918 if tid in found:
922 continue 919 continue
923 spec, extrapath, tname = getpathcomponents(task, d) 920 spec, extrapath, tname = getpathcomponents(tid, d)
924 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + extension) 921 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + extension)
925 tasklist.append((task, sstatefile)) 922 tasklist.append((tid, sstatefile))
926 923
927 if tasklist: 924 if tasklist:
928 msg = "Checking sstate mirror object availability" 925 msg = "Checking sstate mirror object availability"
@@ -943,38 +940,37 @@ def sstate_checkhashes(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=False, *,
943 bb.event.fire(bb.event.ProcessFinished(msg), d) 940 bb.event.fire(bb.event.ProcessFinished(msg), d)
944 941
945 # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting 942 # Likely checking an individual task hash again for multiconfig sharing of sstate tasks so skip reporting
946 if len(sq_fn) == 1: 943 if len(sq_data['hash']) == 1:
947 return ret 944 return found
948 945
949 inheritlist = d.getVar("INHERIT") 946 inheritlist = d.getVar("INHERIT")
950 if "toaster" in inheritlist: 947 if "toaster" in inheritlist:
951 evdata = {'missed': [], 'found': []}; 948 evdata = {'missed': [], 'found': []};
952 for task in missed: 949 for tid in missed:
953 spec, extrapath, tname = getpathcomponents(task, d) 950 spec, extrapath, tname = getpathcomponents(tid, d)
954 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz") 951 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + ".tgz")
955 evdata['missed'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) ) 952 evdata['missed'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
956 for task in ret: 953 for tid in found:
957 spec, extrapath, tname = getpathcomponents(task, d) 954 spec, extrapath, tname = getpathcomponents(tid, d)
958 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(task), d) + "_" + tname + ".tgz") 955 sstatefile = d.expand(extrapath + generate_sstatefn(spec, gethash(tid), d) + "_" + tname + ".tgz")
959 evdata['found'].append( (sq_fn[task], sq_task[task], gethash(task), sstatefile ) ) 956 evdata['found'].append((bb.runqueue.fn_from_tid(tid), bb.runqueue.taskname_from_tid(tid), gethash(tid), sstatefile ) )
960 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d) 957 bb.event.fire(bb.event.MetadataEvent("MissedSstate", evdata), d)
961 958
962 # Print some summary statistics about the current task completion and how much sstate 959 # Print some summary statistics about the current task completion and how much sstate
963 # reuse there was. Avoid divide by zero errors. 960 # reuse there was. Avoid divide by zero errors.
964 total = len(sq_fn) 961 total = len(sq_data['hash'])
965 currentcount = d.getVar("BB_SETSCENE_STAMPCURRENT_COUNT") or 0
966 complete = 0 962 complete = 0
967 if currentcount: 963 if currentcount:
968 complete = (len(ret) + currentcount) / (total + currentcount) * 100 964 complete = (len(found) + currentcount) / (total + currentcount) * 100
969 match = 0 965 match = 0
970 if total: 966 if total:
971 match = len(ret) / total * 100 967 match = len(found) / total * 100
972 bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(ret), len(missed), currentcount, match, complete)) 968 bb.plain("Sstate summary: Wanted %d Found %d Missed %d Current %d (%d%% match, %d%% complete)" % (total, len(found), len(missed), currentcount, match, complete))
973 969
974 if hasattr(bb.parse.siggen, "checkhashes"): 970 if hasattr(bb.parse.siggen, "checkhashes"):
975 bb.parse.siggen.checkhashes(missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d) 971 bb.parse.siggen.checkhashes(sq_data, missed, found, d)
976 972
977 return ret 973 return found
978 974
979BB_SETSCENE_DEPVALID = "setscene_depvalid" 975BB_SETSCENE_DEPVALID = "setscene_depvalid"
980 976
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 3f926e3e51..80aafa0c2d 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -235,19 +235,20 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
235 for (pn, task, fn, taskhash) in sorted(tasks): 235 for (pn, task, fn, taskhash) in sorted(tasks):
236 f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash)) 236 f.write('%s.%s %s %s\n' % (pn, task, fn, taskhash))
237 237
238 def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d): 238 def checkhashes(self, sq_data, missed, found, d):
239 warn_msgs = [] 239 warn_msgs = []
240 error_msgs = [] 240 error_msgs = []
241 sstate_missing_msgs = [] 241 sstate_missing_msgs = []
242 242
243 for task in range(len(sq_fn)): 243 for tid in sq_data['hash']:
244 if task not in ret: 244 if tid not in found:
245 for pn in self.lockedsigs: 245 for pn in self.lockedsigs:
246 if sq_hash[task] in iter(self.lockedsigs[pn].values()): 246 taskname = bb.runqueue.taskname_from_tid(tid)
247 if sq_task[task] == 'do_shared_workdir': 247 if sq_data['hash'][tid] in iter(self.lockedsigs[pn].values()):
248 if taskname == 'do_shared_workdir':
248 continue 249 continue
249 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" 250 sstate_missing_msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?"
250 % (pn, sq_task[task], sq_hash[task])) 251 % (pn, taskname, sq_data['hash'][tid]))
251 252
252 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK") 253 checklevel = d.getVar("SIGGEN_LOCKEDSIGS_TASKSIG_CHECK")
253 if checklevel == 'warn': 254 if checklevel == 'warn':