1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
|
#
# Copyright (C) 2012 Robert Yang
#
# SPDX-License-Identifier: GPL-2.0-only
#
import os, logging, re
import bb
logger = logging.getLogger("BitBake.Monitor")
def printErr(info):
logger.error("%s\n Disk space monitor will NOT be enabled" % info)
def convertGMK(unit):
""" Convert the space unit G, M, K, the unit is case-insensitive """
unitG = re.match(r'([1-9][0-9]*)[gG]\s?$', unit)
if unitG:
return int(unitG.group(1)) * (1024 ** 3)
unitM = re.match(r'([1-9][0-9]*)[mM]\s?$', unit)
if unitM:
return int(unitM.group(1)) * (1024 ** 2)
unitK = re.match(r'([1-9][0-9]*)[kK]\s?$', unit)
if unitK:
return int(unitK.group(1)) * 1024
unitN = re.match(r'([1-9][0-9]*)\s?$', unit)
if unitN:
return int(unitN.group(1))
else:
return None
def getMountedDev(path):
""" Get the device mounted at the path, uses /proc/mounts """
# Get the mount point of the filesystem containing path
# st_dev is the ID of device containing file
parentDev = os.stat(path).st_dev
currentDev = parentDev
# When the current directory's device is different from the
# parent's, then the current directory is a mount point
while parentDev == currentDev:
mountPoint = path
# Use dirname to get the parent's directory
path = os.path.dirname(path)
# Reach the "/"
if path == mountPoint:
break
parentDev= os.stat(path).st_dev
try:
with open("/proc/mounts", "r") as ifp:
for line in ifp:
procLines = line.rstrip('\n').split()
if procLines[1] == mountPoint:
return procLines[0]
except EnvironmentError:
pass
return None
def getDiskData(BBDirs):
"""Prepare disk data for disk space monitor"""
# Save the device IDs, need the ID to be unique (the dictionary's key is
# unique), so that when more than one directory is located on the same
# device, we just monitor it once
devDict = {}
for pathSpaceInode in BBDirs.split():
# The input format is: "dir,space,inode", dir is a must, space
# and inode are optional
pathSpaceInodeRe = re.match(r'([^,]*),([^,]*),([^,]*),?(.*)', pathSpaceInode)
if not pathSpaceInodeRe:
printErr("Invalid value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
return None
action = pathSpaceInodeRe.group(1)
if action == "ABORT":
# Emit a deprecation warning
logger.warnonce("The BB_DISKMON_DIRS \"ABORT\" action has been renamed to \"HALT\", update configuration")
action = "HALT"
if action not in ("HALT", "STOPTASKS", "WARN"):
printErr("Unknown disk space monitor action: %s" % action)
return None
path = os.path.realpath(pathSpaceInodeRe.group(2))
if not path:
printErr("Invalid path value in BB_DISKMON_DIRS: %s" % pathSpaceInode)
return None
# The disk space or inode is optional, but it should have a correct
# value once it is specified
minSpace = pathSpaceInodeRe.group(3)
if minSpace:
minSpace = convertGMK(minSpace)
if not minSpace:
printErr("Invalid disk space value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(3))
return None
else:
# None means that it is not specified
minSpace = None
minInode = pathSpaceInodeRe.group(4)
if minInode:
minInode = convertGMK(minInode)
if not minInode:
printErr("Invalid inode value in BB_DISKMON_DIRS: %s" % pathSpaceInodeRe.group(4))
return None
else:
# None means that it is not specified
minInode = None
if minSpace is None and minInode is None:
printErr("No disk space or inode value in found BB_DISKMON_DIRS: %s" % pathSpaceInode)
return None
# mkdir for the directory since it may not exist, for example the
# DL_DIR may not exist at the very beginning
if not os.path.exists(path):
bb.utils.mkdirhier(path)
dev = getMountedDev(path)
# Use path/action as the key
devDict[(path, action)] = [dev, minSpace, minInode]
return devDict
def getInterval(configuration):
""" Get the disk space interval """
# The default value is 50M and 5K.
spaceDefault = 50 * 1024 * 1024
inodeDefault = 5 * 1024
interval = configuration.getVar("BB_DISKMON_WARNINTERVAL")
if not interval:
return spaceDefault, inodeDefault
else:
# The disk space or inode interval is optional, but it should
# have a correct value once it is specified
intervalRe = re.match(r'([^,]*),?\s*(.*)', interval)
if intervalRe:
intervalSpace = intervalRe.group(1)
if intervalSpace:
intervalSpace = convertGMK(intervalSpace)
if not intervalSpace:
printErr("Invalid disk space interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(1))
return None, None
else:
intervalSpace = spaceDefault
intervalInode = intervalRe.group(2)
if intervalInode:
intervalInode = convertGMK(intervalInode)
if not intervalInode:
printErr("Invalid disk inode interval value in BB_DISKMON_WARNINTERVAL: %s" % intervalRe.group(2))
return None, None
else:
intervalInode = inodeDefault
return intervalSpace, intervalInode
else:
printErr("Invalid interval value in BB_DISKMON_WARNINTERVAL: %s" % interval)
return None, None
class diskMonitor:
"""Prepare the disk space monitor data"""
def __init__(self, configuration):
self.enableMonitor = False
self.configuration = configuration
BBDirs = configuration.getVar("BB_DISKMON_DIRS") or None
if BBDirs:
self.devDict = getDiskData(BBDirs)
if self.devDict:
self.spaceInterval, self.inodeInterval = getInterval(configuration)
if self.spaceInterval and self.inodeInterval:
self.enableMonitor = True
# These are for saving the previous disk free space and inode, we
# use them to avoid printing too many warning messages
self.preFreeS = {}
self.preFreeI = {}
# This is for STOPTASKS and HALT, to avoid printing the message
# repeatedly while waiting for the tasks to finish
self.checked = {}
for k in self.devDict:
self.preFreeS[k] = 0
self.preFreeI[k] = 0
self.checked[k] = False
if self.spaceInterval is None and self.inodeInterval is None:
self.enableMonitor = False
def check(self, rq):
""" Take action for the monitor """
if self.enableMonitor:
diskUsage = {}
for k, attributes in self.devDict.items():
path, action = k
dev, minSpace, minInode = attributes
st = os.statvfs(path)
# The available free space, integer number
freeSpace = st.f_bavail * st.f_frsize
# Send all relevant information in the event.
freeSpaceRoot = st.f_bfree * st.f_frsize
totalSpace = st.f_blocks * st.f_frsize
diskUsage[dev] = bb.event.DiskUsageSample(freeSpace, freeSpaceRoot, totalSpace)
if minSpace and freeSpace < minSpace:
# Always show warning, the self.checked would always be False if the action is WARN
if self.preFreeS[k] == 0 or self.preFreeS[k] - freeSpace > self.spaceInterval and not self.checked[k]:
logger.warning("The free space of %s (%s) is running low (%.3fGB left)" % \
(path, dev, freeSpace / 1024 / 1024 / 1024.0))
self.preFreeS[k] = freeSpace
if action == "STOPTASKS" and not self.checked[k]:
logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
self.checked[k] = True
rq.finish_runqueue(False)
bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
elif action == "HALT" and not self.checked[k]:
logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
self.checked[k] = True
rq.finish_runqueue(True)
bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
# The free inodes, integer number
freeInode = st.f_favail
if minInode and freeInode < minInode:
# Some filesystems use dynamic inodes so can't run out.
# This is reported by the inode count being 0 (btrfs) or the free
# inode count being -1 (cephfs).
if st.f_files == 0 or st.f_favail == -1:
self.devDict[k][2] = None
continue
# Always show warning, the self.checked would always be False if the action is WARN
if self.preFreeI[k] == 0 or self.preFreeI[k] - freeInode > self.inodeInterval and not self.checked[k]:
logger.warning("The free inode of %s (%s) is running low (%.3fK left)" % \
(path, dev, freeInode / 1024.0))
self.preFreeI[k] = freeInode
if action == "STOPTASKS" and not self.checked[k]:
logger.error("No new tasks can be executed since the disk space monitor action is \"STOPTASKS\"!")
self.checked[k] = True
rq.finish_runqueue(False)
bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
elif action == "HALT" and not self.checked[k]:
logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
self.checked[k] = True
rq.finish_runqueue(True)
bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
bb.event.fire(bb.event.MonitorDiskEvent(diskUsage), self.configuration)
return
|
