summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <rpurdie@linux.intel.com>2009-07-08 22:46:09 +0100
committerRichard Purdie <rpurdie@linux.intel.com>2009-07-08 22:46:09 +0100
commit433f50435e2227c66114223a2e2c9c88a5ffeed3 (patch)
tree6ae23a6032aa661d348254ab6afec727f2f3b9ed
parent67d169aa1ce9ce435989e1416b94f64652b1883d (diff)
downloadpoky-433f50435e2227c66114223a2e2c9c88a5ffeed3.tar.gz
bitbake-dev: Turn parsing into a server idle callback allowing the client to interrupt parsing and improving user interactvity. Also now specify whether async commands need the cache or not
Signed-off-by: Richard Purdie <rpurdie@linux.intel.com>
-rw-r--r--bitbake-dev/lib/bb/command.py23
-rw-r--r--bitbake-dev/lib/bb/cooker.py191
2 files changed, 122 insertions, 92 deletions
diff --git a/bitbake-dev/lib/bb/command.py b/bitbake-dev/lib/bb/command.py
index b94756649b..8667736fa1 100644
--- a/bitbake-dev/lib/bb/command.py
+++ b/bitbake-dev/lib/bb/command.py
@@ -57,16 +57,18 @@ class Command:
57 async_cmds[command] = (method) 57 async_cmds[command] = (method)
58 58
59 def runCommand(self, commandline): 59 def runCommand(self, commandline):
60 bb.debug("Running command %s" % commandline)
60 try: 61 try:
61 command = commandline.pop(0) 62 command = commandline.pop(0)
62 if command in CommandsSync.__dict__: 63 if command in CommandsSync.__dict__:
63 # Can run online commands straight away 64 # Can run synchronous commands straight away
64 return getattr(CommandsSync, command)(self.cmds_sync, self, commandline) 65 return getattr(CommandsSync, command)(self.cmds_sync, self, commandline)
65 if self.currentAsyncCommand is not None: 66 if self.currentAsyncCommand is not None:
66 return "Busy (%s in progress)" % self.currentAsyncCommand[0] 67 return "Busy (%s in progress)" % self.currentAsyncCommand[0]
67 if command not in CommandsAsync.__dict__: 68 if command not in CommandsAsync.__dict__:
68 return "No such command" 69 return "No such command"
69 self.currentAsyncCommand = (command, commandline) 70 self.currentAsyncCommand = (command, commandline)
71 self.cooker.server.register_idle_function(self.cooker.runCommands, self.cooker)
70 return True 72 return True
71 except: 73 except:
72 import traceback 74 import traceback
@@ -76,10 +78,20 @@ class Command:
76 try: 78 try:
77 if self.currentAsyncCommand is not None: 79 if self.currentAsyncCommand is not None:
78 (command, options) = self.currentAsyncCommand 80 (command, options) = self.currentAsyncCommand
79 getattr(CommandsAsync, command)(self.cmds_async, self, options) 81 commandmethod = getattr(CommandsAsync, command)
82 needcache = getattr( commandmethod, "needcache" )
83 if needcache and self.cooker.cookerState != bb.cooker.cookerParsed:
84 self.cooker.updateCache()
85 return True
86 else:
87 commandmethod(self.cmds_async, self, options)
88 return False
89 else:
90 return False
80 except: 91 except:
81 import traceback 92 import traceback
82 self.finishAsyncCommand(traceback.format_exc()) 93 self.finishAsyncCommand(traceback.format_exc())
94 return False
83 95
84 def finishAsyncCommand(self, error = None): 96 def finishAsyncCommand(self, error = None):
85 if error: 97 if error:
@@ -149,6 +161,7 @@ class CommandsAsync:
149 task = params[1] 161 task = params[1]
150 162
151 command.cooker.buildFile(bfile, task) 163 command.cooker.buildFile(bfile, task)
164 buildFile.needcache = False
152 165
153 def buildTargets(self, command, params): 166 def buildTargets(self, command, params):
154 """ 167 """
@@ -158,6 +171,7 @@ class CommandsAsync:
158 task = params[1] 171 task = params[1]
159 172
160 command.cooker.buildTargets(pkgs_to_build, task) 173 command.cooker.buildTargets(pkgs_to_build, task)
174 buildTargets.needcache = True
161 175
162 def generateDepTreeEvent(self, command, params): 176 def generateDepTreeEvent(self, command, params):
163 """ 177 """
@@ -168,6 +182,7 @@ class CommandsAsync:
168 182
169 command.cooker.generateDepTreeEvent(pkgs_to_build, task) 183 command.cooker.generateDepTreeEvent(pkgs_to_build, task)
170 command.finishAsyncCommand() 184 command.finishAsyncCommand()
185 generateDepTreeEvent.needcache = True
171 186
172 def generateDotGraph(self, command, params): 187 def generateDotGraph(self, command, params):
173 """ 188 """
@@ -178,6 +193,7 @@ class CommandsAsync:
178 193
179 command.cooker.generateDotGraphFiles(pkgs_to_build, task) 194 command.cooker.generateDotGraphFiles(pkgs_to_build, task)
180 command.finishAsyncCommand() 195 command.finishAsyncCommand()
196 generateDotGraph.needcache = True
181 197
182 def showVersions(self, command, params): 198 def showVersions(self, command, params):
183 """ 199 """
@@ -185,6 +201,7 @@ class CommandsAsync:
185 """ 201 """
186 command.cooker.showVersions() 202 command.cooker.showVersions()
187 command.finishAsyncCommand() 203 command.finishAsyncCommand()
204 showVersions.needcache = True
188 205
189 def showEnvironment(self, command, params): 206 def showEnvironment(self, command, params):
190 """ 207 """
@@ -195,6 +212,7 @@ class CommandsAsync:
195 212
196 command.cooker.showEnvironment(bfile, pkg) 213 command.cooker.showEnvironment(bfile, pkg)
197 command.finishAsyncCommand() 214 command.finishAsyncCommand()
215 showEnvironment.needcache = True
198 216
199 def parseFiles(self, command, params): 217 def parseFiles(self, command, params):
200 """ 218 """
@@ -202,6 +220,7 @@ class CommandsAsync:
202 """ 220 """
203 command.cooker.updateCache() 221 command.cooker.updateCache()
204 command.finishAsyncCommand() 222 command.finishAsyncCommand()
223 parseFiles.needcache = True
205 224
206# 225#
207# Events 226# Events
diff --git a/bitbake-dev/lib/bb/cooker.py b/bitbake-dev/lib/bb/cooker.py
index b12dc13b62..c4d65ddad5 100644
--- a/bitbake-dev/lib/bb/cooker.py
+++ b/bitbake-dev/lib/bb/cooker.py
@@ -46,7 +46,8 @@ class NothingToBuild(Exception):
46 46
47# Different states cooker can be in 47# Different states cooker can be in
48cookerClean = 1 48cookerClean = 1
49cookerParsed = 2 49cookerParsing = 2
50cookerParsed = 3
50 51
51# Different action states the cooker can be in 52# Different action states the cooker can be in
52cookerRun = 1 # Cooker is running normally 53cookerRun = 1 # Cooker is running normally
@@ -116,10 +117,8 @@ class BBCooker:
116 termios.tcsetattr(fd, termios.TCSANOW, tcattr) 117 termios.tcsetattr(fd, termios.TCSANOW, tcattr)
117 118
118 self.command = bb.command.Command(self) 119 self.command = bb.command.Command(self)
119 self.cookerIdle = True
120 self.cookerState = cookerClean 120 self.cookerState = cookerClean
121 self.cookerAction = cookerRun 121 self.cookerAction = cookerRun
122 self.server.register_idle_function(self.runCommands, self)
123 122
124 def parseConfiguration(self): 123 def parseConfiguration(self):
125 124
@@ -172,11 +171,8 @@ class BBCooker:
172 This is done by the idle handler so it runs in true context rather than 171 This is done by the idle handler so it runs in true context rather than
173 tied to any UI. 172 tied to any UI.
174 """ 173 """
175 if self.cookerIdle and not abort:
176 self.command.runAsyncCommand()
177 174
178 # Always reschedule 175 return self.command.runAsyncCommand()
179 return True
180 176
181 def tryBuildPackage(self, fn, item, task, the_data): 177 def tryBuildPackage(self, fn, item, task, the_data):
182 """ 178 """
@@ -675,12 +671,11 @@ class BBCooker:
675 failures = failures + 1 671 failures = failures + 1
676 retval = False 672 retval = False
677 if not retval: 673 if not retval:
678 self.cookerIdle = True
679 self.command.finishAsyncCommand() 674 self.command.finishAsyncCommand()
680 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures)) 675 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
681 return retval 676 return False
677 return 0.5
682 678
683 self.cookerIdle = False
684 self.server.register_idle_function(buildFileIdle, rq) 679 self.server.register_idle_function(buildFileIdle, rq)
685 680
686 def buildTargets(self, targets, task): 681 def buildTargets(self, targets, task):
@@ -712,10 +707,10 @@ class BBCooker:
712 failures = failures + 1 707 failures = failures + 1
713 retval = False 708 retval = False
714 if not retval: 709 if not retval:
715 self.cookerIdle = True
716 self.command.finishAsyncCommand() 710 self.command.finishAsyncCommand()
717 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures)) 711 bb.event.fire(bb.event.BuildCompleted(buildname, targets, self.configuration.event_data, failures))
718 return retval 712 return None
713 return 0.5
719 714
720 self.buildSetVars() 715 self.buildSetVars()
721 716
@@ -736,47 +731,54 @@ class BBCooker:
736 731
737 rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist) 732 rq = bb.runqueue.RunQueue(self, self.configuration.data, self.status, taskdata, runlist)
738 733
739 self.cookerIdle = False
740 self.server.register_idle_function(buildTargetsIdle, rq) 734 self.server.register_idle_function(buildTargetsIdle, rq)
741 735
742 def updateCache(self): 736 def updateCache(self):
743 737
744 self.parseConfiguration ()
745 if self.cookerState == cookerParsed: 738 if self.cookerState == cookerParsed:
746 return 739 return
747 740
748 # Import Psyco if available and not disabled 741 if self.cookerState != cookerParsing:
749 import platform
750 if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
751 if not self.configuration.disable_psyco:
752 try:
753 import psyco
754 except ImportError:
755 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
756 else:
757 psyco.bind( self.parse_bbfiles )
758 else:
759 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
760 742
761 self.status = bb.cache.CacheData() 743 self.parseConfiguration ()
762 744
763 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or "" 745 # Import Psyco if available and not disabled
764 self.status.ignored_dependencies = set(ignore.split()) 746 import platform
747 if platform.machine() in ['i386', 'i486', 'i586', 'i686']:
748 if not self.configuration.disable_psyco:
749 try:
750 import psyco
751 except ImportError:
752 bb.msg.note(1, bb.msg.domain.Collection, "Psyco JIT Compiler (http://psyco.sf.net) not available. Install it to increase performance.")
753 else:
754 psyco.bind( CookerParser.parse_next )
755 else:
756 bb.msg.note(1, bb.msg.domain.Collection, "You have disabled Psyco. This decreases performance.")
757
758 self.status = bb.cache.CacheData()
765 759
766 for dep in self.configuration.extra_assume_provided: 760 ignore = bb.data.getVar("ASSUME_PROVIDED", self.configuration.data, 1) or ""
767 self.status.ignored_dependencies.add(dep) 761 self.status.ignored_dependencies = set(ignore.split())
762
763 for dep in self.configuration.extra_assume_provided:
764 self.status.ignored_dependencies.add(dep)
765
766 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) )
768 767
769 self.handleCollections( bb.data.getVar("BBFILE_COLLECTIONS", self.configuration.data, 1) ) 768 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files")
769 (filelist, masked) = self.collect_bbfiles()
770 bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
770 771
771 bb.msg.debug(1, bb.msg.domain.Collection, "collecting .bb files") 772 self.parser = CookerParser(self, filelist, masked)
772 (filelist, masked) = self.collect_bbfiles() 773 self.cookerState = cookerParsing
773 bb.data.renameVar("__depends", "__base_depends", self.configuration.data)
774 self.parse_bbfiles(filelist, masked)
775 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
776 774
777 self.buildDepgraph() 775 if not self.parser.parse_next():
776 bb.msg.debug(1, bb.msg.domain.Collection, "parsing complete")
777 self.buildDepgraph()
778 self.cookerState = cookerParsed
779 return None
778 780
779 self.cookerState = cookerParsed 781 return 0.00001
780 782
781 def checkPackages(self, pkgs_to_build): 783 def checkPackages(self, pkgs_to_build):
782 784
@@ -861,57 +863,6 @@ class BBCooker:
861 863
862 return (finalfiles, masked) 864 return (finalfiles, masked)
863 865
864 def parse_bbfiles(self, filelist, masked):
865 parsed, cached, skipped, error, total = 0, 0, 0, 0, len(filelist)
866 for i in xrange(total):
867 f = filelist[i]
868
869 #bb.msg.debug(1, bb.msg.domain.Collection, "parsing %s" % f)
870
871 # read a file's metadata
872 try:
873 fromCache, skip = self.bb_cache.loadData(f, self.configuration.data, self.status)
874 if skip:
875 skipped += 1
876 bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
877 self.bb_cache.skip(f)
878 continue
879 elif fromCache: cached += 1
880 else: parsed += 1
881
882 # Disabled by RP as was no longer functional
883 # allow metadata files to add items to BBFILES
884 #data.update_data(self.pkgdata[f])
885 #addbbfiles = self.bb_cache.getVar('BBFILES', f, False) or None
886 #if addbbfiles:
887 # for aof in addbbfiles.split():
888 # if not files.count(aof):
889 # if not os.path.isabs(aof):
890 # aof = os.path.join(os.path.dirname(f),aof)
891 # files.append(aof)
892
893 except IOError, e:
894 error += 1
895 self.bb_cache.remove(f)
896 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
897 pass
898 except KeyboardInterrupt:
899 self.bb_cache.sync()
900 raise
901 except Exception, e:
902 error += 1
903 self.bb_cache.remove(f)
904 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
905 except:
906 self.bb_cache.remove(f)
907 raise
908 finally:
909 bb.event.fire(bb.event.ParseProgress(self.configuration.event_data, cached, parsed, skipped, masked, error, total))
910
911 self.bb_cache.sync()
912 if error > 0:
913 raise ParsingErrorsFound
914
915 def serve(self): 866 def serve(self):
916 867
917 # Empty the environment. The environment will be populated as 868 # Empty the environment. The environment will be populated as
@@ -955,3 +906,63 @@ class CookerExit(bb.event.Event):
955 def __init__(self, d): 906 def __init__(self, d):
956 bb.event.Event.__init__(self, d) 907 bb.event.Event.__init__(self, d)
957 908
909
910class CookerParser:
911 def __init__(self, cooker, filelist, masked):
912 # Internal data
913 self.filelist = filelist
914 self.cooker = cooker
915
916 # Accounting statistics
917 self.parsed = 0
918 self.cached = 0
919 self.skipped = 0
920 self.error = 0
921 self.masked = masked
922 self.total = len(filelist)
923
924 # Pointer to the next file to parse
925 self.pointer = 0
926
927 def parse_next(self):
928 print "Pointer %d" % self.pointer
929 f = self.filelist[self.pointer]
930 cooker = self.cooker
931
932 try:
933 fromCache, skip = cooker.bb_cache.loadData(f, cooker.configuration.data, cooker.status)
934 if skip:
935 self.skipped += 1
936 bb.msg.debug(2, bb.msg.domain.Collection, "skipping %s" % f)
937 cooker.bb_cache.skip(f)
938 elif fromCache: self.cached += 1
939 else: self.parsed += 1
940
941 except IOError, e:
942 self.error += 1
943 cooker.bb_cache.remove(f)
944 bb.msg.error(bb.msg.domain.Collection, "opening %s: %s" % (f, e))
945 pass
946 except KeyboardInterrupt:
947 cooker.bb_cache.remove(f)
948 cooker.bb_cache.sync()
949 raise
950 except Exception, e:
951 self.error += 1
952 cooker.bb_cache.remove(f)
953 bb.msg.error(bb.msg.domain.Collection, "%s while parsing %s" % (e, f))
954 except:
955 cooker.bb_cache.remove(f)
956 raise
957 finally:
958 bb.event.fire(bb.event.ParseProgress(cooker.configuration.event_data, self.cached, self.parsed, self.skipped, self.masked, self.error, self.total))
959
960 self.pointer += 1
961
962 if self.pointer >= self.total:
963 cooker.bb_cache.sync()
964 if self.error > 0:
965 raise ParsingErrorsFound
966 return False
967 return True
968