summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobert Yang <liezhi.yang@windriver.com>2012-02-26 16:48:15 +0800
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-02-26 11:05:27 +0000
commit9674ea28ffebd75f117531238e517df41ed7dafb (patch)
tree75983bf7dca3de0a78fe6a2acf1a398c74c78998
parentaac45679fb17c720f511a30261d34eb64e6a4c70 (diff)
downloadpoky-9674ea28ffebd75f117531238e517df41ed7dafb.tar.gz
V5 Disk space monitoring
Monitor disk availability and take action when the free disk space or amount of free inode is running low, it is enabled when BB_DISKMON_DIRS is set. * Variable meanings(from meta-yocto/conf/local.conf.sample): # Set the directories to monitor for disk usage, if more than one # directories are mounted in the same device, then only one directory # would be monitored since the monitor is based on the device. # The format is: # "action,directory,minimum_space,minimum_free_inode" # # The "action" must be set and should be one of: # ABORT: Immediately abort # STOPTASKS: The new tasks can't be executed any more, will stop the build # when the running tasks have been done. # WARN: show warnings (see BB_DISKMON_WARNINTERVAL for more information) # # The "directory" must be set, any directory is OK. # # Either "minimum_space" or "minimum_free_inode" (or both of them) # should be set, otherwise the monitor would not be enabled, # the unit can be G, M, K or none, but do NOT use GB, MB or KB # (B is not needed). #BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K" # # Set disk space and inode interval (only works when the action is "WARN", # the unit can be G, M, or K, but do NOT use the GB, MB or KB # (B is not needed), the format is: # "disk_space_interval, disk_inode_interval", the default value is # "50M,5K" which means that it would warn when the free space is # lower than the minimum space(or inode), and would repeat the action # when the disk space reduces 50M (or the amount of inode reduces 5k) # again. #BB_DISKMON_WARNINTERVAL = "50M,5K" [YOCTO #1589] (Bitbake rev: 4d173d441d2beb8e6492b6b1842682f8cf32e6cc) Signed-off-by: Robert Yang <liezhi.yang@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/monitordisk.py237
-rw-r--r--bitbake/lib/bb/runqueue.py7
2 files changed, 244 insertions, 0 deletions
diff --git a/bitbake/lib/bb/monitordisk.py b/bitbake/lib/bb/monitordisk.py
new file mode 100644
index 0000000000..04f090cbe6
--- /dev/null
+++ b/bitbake/lib/bb/monitordisk.py
@@ -0,0 +1,237 @@
1#!/usr/bin/env python
2# ex:ts=4:sw=4:sts=4:et
3# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
4#
5# Copyright (C) 2012 Robert Yang
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 2 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License along
17# with this program; if not, write to the Free Software Foundation, Inc.,
18# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
19
20import os, logging, re, sys
21import bb
22logger = logging.getLogger("BitBake.Monitor")
23
24def printErr(info):
25 logger.error("%s\n Disk space monitor will NOT be enabled" % info)
26
27def convertGMK(unit):
28
29 """ Convert the space unit G, M, K, the unit is case-insensitive """
30
31 unitG = re.match('([1-9][0-9]*)[gG]\s?$', unit)
32 if unitG:
33 return int(unitG.group(1)) * (1024 ** 3)
34 unitM = re.match('([1-9][0-9]*)[mM]\s?$', unit)
35 if unitM:
36 return int(unitM.group(1)) * (1024 ** 2)
37 unitK = re.match('([1-9][0-9]*)[kK]\s?$', unit)
38 if unitK:
39 return int(unitK.group(1)) * 1024
40 unitN = re.match('([1-9][0-9]*)\s?$', unit)
41 if unitN:
42 return int(unitN.group(1))
43 else:
44 return None
45
46def getMountedDev(path):
47
48 """ Get the device mounted at the path, uses /proc/mounts """
49
50 # Get the mount point of the filesystem containing path
51 # st_dev is the ID of device containing file
52 parentDev = os.stat(path).st_dev
53 currentDev = parentDev
54 # When the current directory's device is different from the
55 # parrent's, then the current directory is a mount point
56 while parentDev == currentDev:
57 mountPoint = path
58 # Use dirname to get the parrent's directory
59 path = os.path.dirname(path)
60 # Reach the "/"
61 if path == mountPoint:
62 break
63 parentDev= os.stat(path).st_dev
64
65 try:
66 with open("/proc/mounts", "r") as ifp:
67 for line in ifp:
68 procLines = line.rstrip('\n').split()
69 if procLines[1] == mountPoint:
70 return procLines[0]
71 except EnvironmentError:
72 pass
73 return None
74
75def getDiskData(BBDirs, configuration):
76
77 """Prepare disk data for disk space monitor"""
78
79 # Save the device IDs, need the ID to be unique (the dictionary's key is
80 # unique), so that when more than one directories are located in the same
81 # device, we just monitor it once
82 devDict = {}
83 for pathSpaceInode in BBDirs.split():
84 # The input format is: "dir,space,inode", dir is a must, space
85 # and inode are optional
86 pathSpaceInodeRe = re.match('([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
87 if not pathSpaceInodeRe:
88 printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
89 return None
90
91 action = pathSpaceInodeRe.group(1)
92 if action not in ("ABORT", "STOPTASKS", "WARN"):
93 printErr("Unknown disk space monitor action: %s" % action)
94 return None
95
96 path = os.path.realpath(pathSpaceInodeRe.group(2))
97 if not path:
98 printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
99 return None
100
101 # The disk space or inode is optional, but it should have a correct
102 # value once it is specified
103 minSpace = pathSpaceInodeRe.group(3)
104 if minSpace:
105 minSpace = convertGMK(minSpace)
106 if not minSpace:
107 printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
108 return None
109 else:
110 # 0 means that it is not specified
111 minSpace = None
112
113 minInode = pathSpaceInodeRe.group(4)
114 if minInode:
115 minInode = convertGMK(minInode)
116 if not minInode:
117 printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
118 return None
119 else:
120 # 0 means that it is not specified
121 minInode = None
122
123 if minSpace is None and minInode is None:
124 printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
125 return None
126 # mkdir for the directory since it may not exist, for example the
127 # DL_DIR may not exist at the very beginning
128 if not os.path.exists(path):
129 bb.utils.mkdirhier(path)
130 mountedDev = getMountedDev(path)
131 devDict[mountedDev] = action, path, minSpace, minInode
132
133 return devDict
134
135def getInterval(configuration):
136
137 """ Get the disk space interval """
138
139 interval = configuration.getVar("BB_DISKMON_WARNINTERVAL", 1)
140 if not interval:
141 # The default value is 50M and 5K.
142 return 50 * 1024 * 1024, 5 * 1024
143 else:
144 # The disk space or inode interval is optional, but it should
145 # have a correct value once it is specified
146 intervalRe = re.match('([^,]*),?\s*(.*)', interval)
147 if intervalRe:
148 intervalSpace = intervalRe.group(1)
149 if intervalSpace:
150 intervalSpace = convertGMK(intervalSpace)
151 if not intervalSpace:
152 printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
153 return None, None
154 intervalInode = intervalRe.group(2)
155 if intervalInode:
156 intervalInode = convertGMK(intervalInode)
157 if not intervalInode:
158 printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
159 return None, None
160 return intervalSpace, intervalInode
161 else:
162 printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
163 return None, None
164
165class diskMonitor:
166
167 """Prepare the disk space monitor data"""
168
169 def __init__(self, configuration):
170
171 self.enableMonitor = False
172
173 BBDirs = configuration.getVar("BB_DISKMON_DIRS", 1) or None
174 if BBDirs:
175 self.devDict = getDiskData(BBDirs, configuration)
176 if self.devDict:
177 self.spaceInterval, self.inodeInterval = getInterval(configuration)
178 if self.spaceInterval and self.inodeInterval:
179 self.enableMonitor = True
180 # These are for saving the previous disk free space and inode, we
181 # use them to avoid print too many warning messages
182 self.preFreeS = {}
183 self.preFreeI = {}
184 # This is for STOPTASKS and ABORT, to avoid print the message repeatly
185 # during waiting the tasks to finish
186 self.checked = {}
187 for dev in self.devDict:
188 self.preFreeS[dev] = 0
189 self.preFreeI[dev] = 0
190 self.checked[dev] = False
191 if self.spaceInterval is None and self.inodeInterval is None:
192 self.enableMonitor = False
193
194 def check(self, rq):
195
196 """ Take action for the monitor """
197
198 if self.enableMonitor:
199 for dev in self.devDict:
200 st = os.statvfs(self.devDict[dev][1])
201
202 # The free space, float point number
203 freeSpace = st.f_bavail * st.f_frsize
204
205 if self.devDict[dev][2] and freeSpace < self.devDict[dev][2]:
206 # Always show warning, the self.checked would always be False if the action is WARN
207 if self.preFreeS[dev] == 0 or self.preFreeS[dev] - freeSpace > self.spaceInterval and not self.checked[dev]:
208 logger.warn("The free space of %s is running low (%.3fGB left)" % (dev, freeSpace / 1024 / 1024 / 1024.0))
209 self.preFreeS[dev] = freeSpace
210
211 if self.devDict[dev][0] == "STOPTASKS" and not self.checked[dev]:
212 logger.error("No new tasks can be excuted since the disk space monitor action is \"STOPTASKS\"!")
213 self.checked[dev] = True
214 rq.finish_runqueue(False)
215 elif self.devDict[dev][0] == "ABORT" and not self.checked[dev]:
216 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
217 self.checked[dev] = True
218 rq.finish_runqueue(True)
219
220 # The free inodes, float point number
221 freeInode = st.f_favail
222
223 if self.devDict[dev][3] and freeInode < self.devDict[dev][3]:
224 # Always show warning, the self.checked would always be False if the action is WARN
225 if self.preFreeI[dev] == 0 or self.preFreeI[dev] - freeInode > self.inodeInterval and not self.checked[dev]:
226 logger.warn("The free inode of %s is running low (%.3fK left)" % (dev, freeInode / 1024.0))
227 self.preFreeI[dev] = freeInode
228
229 if self.devDict[dev][0] == "STOPTASKS" and not self.checked[dev]:
230 logger.error("No new tasks can be excuted since the disk space monitor action is \"STOPTASKS\"!")
231 self.checked[dev] = True
232 rq.finish_runqueue(False)
233 elif self.devDict[dev][0] == "ABORT" and not self.checked[dev]:
234 logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
235 self.checked[dev] = True
236 rq.finish_runqueue(True)
237 return
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 1c3187d462..f08f93aef4 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -31,6 +31,7 @@ import fcntl
31import logging 31import logging
32import bb 32import bb
33from bb import msg, data, event 33from bb import msg, data, event
34from bb import monitordisk
34 35
35bblogger = logging.getLogger("BitBake") 36bblogger = logging.getLogger("BitBake")
36logger = logging.getLogger("BitBake.RunQueue") 37logger = logging.getLogger("BitBake.RunQueue")
@@ -775,6 +776,9 @@ class RunQueue:
775 776
776 self.state = runQueuePrepare 777 self.state = runQueuePrepare
777 778
779 # For disk space monitor
780 self.dm = monitordisk.diskMonitor(cfgData)
781
778 def check_stamps(self): 782 def check_stamps(self):
779 unchecked = {} 783 unchecked = {}
780 current = [] 784 current = []
@@ -949,6 +953,9 @@ class RunQueue:
949 else: 953 else:
950 self.rqexe = RunQueueExecuteScenequeue(self) 954 self.rqexe = RunQueueExecuteScenequeue(self)
951 955
956 if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
957 self.dm.check(self)
958
952 if self.state is runQueueSceneRun: 959 if self.state is runQueueSceneRun:
953 retval = self.rqexe.execute() 960 retval = self.rqexe.execute()
954 961