summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorAndrew Bradford <andrew.bradford@kodakalaris.com>2016-06-09 09:44:08 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-06-16 17:37:58 +0100
commit26447a0d0bf5ceaac382a78c1dbd00807dbab706 (patch)
tree20d8f2f8a0c0d6d04586089abd1f5b0f4286c829 /bitbake/lib
parent9f91785cc57ee1e2aaeb8725b1c72927f16fcce6 (diff)
downloadpoky-26447a0d0bf5ceaac382a78c1dbd00807dbab706.tar.gz
bitbake: fetch2/perforce: Rework to support SRCREV and P4CONFIG
In recipes which use the perforce fetcher, enable use of SRCREV to specify any of: ${AUTOREV}, changelist number, p4date, or label. This is more in-line with how the other fetchers work for source control systems. Allow p4 to use the P4CONFIG env variable to define the server URL, username, and password if not provided in a recipe. This does change existing perforce fetcher usage by recipes and will likely need those recipes which use the perforce fetcher to be updated. No recipes in oe-core use the perforce fetcher. References [YOCTO #6303] (Bitbake rev: 6298696bb94a127cdec7964315f6891ba92cd026) Signed-off-by: Andrew Bradford <andrew.bradford@kodakalaris.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py3
-rw-r--r--bitbake/lib/bb/fetch2/perforce.py270
2 files changed, 150 insertions, 123 deletions
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index a63498a88b..b6fcaaad55 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -803,7 +803,8 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None):
803 'GIT_SMART_HTTP', 803 'GIT_SMART_HTTP',
804 'SSH_AUTH_SOCK', 'SSH_AGENT_PID', 804 'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
805 'SOCKS5_USER', 'SOCKS5_PASSWD', 805 'SOCKS5_USER', 'SOCKS5_PASSWD',
806 'DBUS_SESSION_BUS_ADDRESS'] 806 'DBUS_SESSION_BUS_ADDRESS',
807 'P4CONFIG']
807 808
808 if not cleanup: 809 if not cleanup:
809 cleanup = [] 810 cleanup = []
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index ce3cda2670..b8169f2cc9 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -1,14 +1,12 @@
1# ex:ts=4:sw=4:sts=4:et 1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- 2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3""" 3"""
4BitBake 'Fetch' implementations 4BitBake 'Fetch' implementation for perforce
5
6Classes for obtaining upstream sources for the
7BitBake build tools.
8 5
9""" 6"""
10 7
11# Copyright (C) 2003, 2004 Chris Larson 8# Copyright (C) 2003, 2004 Chris Larson
9# Copyright (C) 2016 Kodak Alaris, Inc.
12# 10#
13# This program is free software; you can redistribute it and/or modify 11# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as 12# it under the terms of the GNU General Public License version 2 as
@@ -26,7 +24,6 @@ BitBake build tools.
26# Based on functions from the base bb module, Copyright 2003 Holger Schurig 24# Based on functions from the base bb module, Copyright 2003 Holger Schurig
27 25
28import os 26import os
29import subprocess
30import logging 27import logging
31import bb 28import bb
32from bb import data 29from bb import data
@@ -36,151 +33,180 @@ from bb.fetch2 import logger
36from bb.fetch2 import runfetchcmd 33from bb.fetch2 import runfetchcmd
37 34
38class Perforce(FetchMethod): 35class Perforce(FetchMethod):
36 """ Class to fetch from perforce repositories """
39 def supports(self, ud, d): 37 def supports(self, ud, d):
38 """ Check to see if a given url can be fetched with perforce. """
40 return ud.type in ['p4'] 39 return ud.type in ['p4']
41 40
42 def doparse(url, d): 41 def urldata_init(self, ud, d):
43 parm = {} 42 """
44 path = url.split("://")[1] 43 Initialize perforce specific variables within url data. If P4CONFIG is
45 delim = path.find("@"); 44 provided by the env, use it. If P4PORT is specified by the recipe, use
45 its values, which may override the settings in P4CONFIG.
46 """
47 ud.basecmd = d.getVar('FETCHCMD_p4', True)
48 if not ud.basecmd:
49 ud.basecmd = "/usr/bin/env p4"
50
51 ud.dldir = d.getVar('P4DIR', True)
52 if not ud.dldir:
53 ud.dldir = '%s/%s' % (d.getVar('DL_DIR', True), 'p4')
54
55 path = ud.url.split('://')[1]
56 path = path.split(';')[0]
57 delim = path.find('@');
46 if delim != -1: 58 if delim != -1:
47 (user, pswd, host, port) = path.split('@')[0].split(":") 59 (ud.user, ud.pswd) = path.split('@')[0].split(':')
48 path = path.split('@')[1] 60 ud.path = path.split('@')[1]
49 else: 61 else:
50 (host, port) = d.getVar('P4PORT', False).split(':') 62 ud.path = path
51 user = ""
52 pswd = ""
53
54 if path.find(";") != -1:
55 keys=[]
56 values=[]
57 plist = path.split(';')
58 for item in plist:
59 if item.count('='):
60 (key, value) = item.split('=')
61 keys.append(key)
62 values.append(value)
63
64 parm = dict(list(zip(keys, values)))
65 path = "//" + path.split(';')[0]
66 host += ":%s" % (port)
67 parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
68
69 return host, path, user, pswd, parm
70 doparse = staticmethod(doparse)
71
72 def getcset(d, depot, host, user, pswd, parm):
73 p4opt = ""
74 if "cset" in parm:
75 return parm["cset"];
76 if user:
77 p4opt += " -u %s" % (user)
78 if pswd:
79 p4opt += " -P %s" % (pswd)
80 if host:
81 p4opt += " -p %s" % (host)
82
83 p4date = d.getVar("P4DATE", True)
84 if "revision" in parm:
85 depot += "#%s" % (parm["revision"])
86 elif "label" in parm:
87 depot += "@%s" % (parm["label"])
88 elif p4date:
89 depot += "@%s" % (p4date)
90
91 p4cmd = d.getVar('FETCHCMD_p4', True) or "p4"
92 logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)
93 p4file, errors = bb.process.run("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
94 cset = p4file.strip()
95 logger.debug(1, "READ %s", cset)
96 if not cset:
97 return -1
98
99 return cset.split(' ')[1]
100 getcset = staticmethod(getcset)
101 63
102 def urldata_init(self, ud, d): 64 ud.usingp4config = False
103 (host, path, user, pswd, parm) = Perforce.doparse(ud.url, d) 65 p4port = d.getVar('P4PORT', True)
104 66
105 base_path = path.replace('/...', '') 67 if p4port:
106 base_path = self._strip_leading_slashes(base_path) 68 logger.debug(1, 'Using recipe provided P4PORT: %s' % p4port)
107 69 ud.host = p4port
108 if "label" in parm: 70 else:
109 version = parm["label"] 71 logger.debug(1, 'Trying to use P4CONFIG to automatically set P4PORT...')
72 ud.usingp4config = True
73 p4cmd = '%s info | grep "Server address"' % ud.basecmd
74 bb.fetch2.check_network_access(d, p4cmd)
75 ud.host = runfetchcmd(p4cmd, d, True)
76 ud.host = ud.host.split(': ')[1].strip()
77 logger.debug(1, 'Determined P4PORT to be: %s' % ud.host)
78 if not ud.host:
79 raise FetchError('Could not determine P4PORT from P4CONFIG')
80
81 if ud.path.find('/...') >= 0:
82 ud.pathisdir = True
110 else: 83 else:
111 version = Perforce.getcset(d, path, host, user, pswd, parm) 84 ud.pathisdir = False
112 85
113 ud.localfile = data.expand('%s+%s+%s.tar.gz' % (host, base_path.replace('/', '.'), version), d) 86 cleanedpath = ud.path.replace('/...', '').replace('/', '.')
87 cleanedhost = ud.host.replace(':', '.')
88 ud.pkgdir = os.path.join(ud.dldir, cleanedhost, cleanedpath)
114 89
115 def download(self, ud, d): 90 ud.setup_revisons(d)
91
92 ud.localfile = data.expand('%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, ud.revision), d)
93
94 def _buildp4command(self, ud, d, command, depot_filename=None):
116 """ 95 """
117 Fetch urls 96 Build a p4 commandline. Valid commands are "changes", "print", and
97 "files". depot_filename is the full path to the file in the depot
98 including the trailing '#rev' value.
118 """ 99 """
100 p4opt = ""
101
102 if ud.user:
103 p4opt += ' -u "%s"' % (ud.user)
119 104
120 (host, depot, user, pswd, parm) = Perforce.doparse(ud.url, d) 105 if ud.pswd:
106 p4opt += ' -P "%s"' % (ud.pswd)
121 107
122 if depot.find('/...') != -1: 108 if ud.host and not ud.usingp4config:
123 path = depot[:depot.find('/...')] 109 p4opt += ' -p %s' % (ud.host)
110
111 if hasattr(ud, 'revision') and ud.revision:
112 pathnrev = '%s@%s' % (ud.path, ud.revision)
113 else:
114 pathnrev = '%s' % (ud.path)
115
116 if depot_filename:
117 if ud.pathisdir: # Remove leading path to obtain filename
118 filename = depot_filename[len(ud.path)-1:]
119 else:
120 filename = depot_filename[depot_filename.rfind('/'):]
121 filename = filename[:filename.find('#')] # Remove trailing '#rev'
122
123 if command == 'changes':
124 p4cmd = '%s%s changes -m 1 //%s' % (ud.basecmd, p4opt, pathnrev)
125 elif command == 'print':
126 if depot_filename != None:
127 p4cmd = '%s%s print -o "p4/%s" "%s"' % (ud.basecmd, p4opt, filename, depot_filename)
128 else:
129 raise FetchError('No depot file name provided to p4 %s' % command, ud.url)
130 elif command == 'files':
131 p4cmd = '%s%s files //%s' % (ud.basecmd, p4opt, pathnrev)
124 else: 132 else:
125 path = depot[:depot.rfind('/')] 133 raise FetchError('Invalid p4 command %s' % command, ud.url)
126 134
127 module = parm.get('module', os.path.basename(path)) 135 return p4cmd
128 136
129 # Get the p4 command 137 def _p4listfiles(self, ud, d):
130 p4opt = "" 138 """
131 if user: 139 Return a list of the file names which are present in the depot using the
132 p4opt += " -u %s" % (user) 140 'p4 files' command, including trailing '#rev' file revision indicator
141 """
142 p4cmd = self._buildp4command(ud, d, 'files')
143 bb.fetch2.check_network_access(d, p4cmd)
144 p4fileslist = runfetchcmd(p4cmd, d, True)
145 p4fileslist = [f.rstrip() for f in p4fileslist.splitlines()]
146
147 if not p4fileslist:
148 raise FetchError('Unable to fetch listing of p4 files from %s@%s' % (ud.host, ud.path))
149
150 count = 0
151 filelist = []
133 152
134 if pswd: 153 for filename in p4fileslist:
135 p4opt += " -P %s" % (pswd) 154 item = filename.split(' - ')
155 lastaction = item[1].split()
156 logger.debug(1, 'File: %s Last Action: %s' % (item[0], lastaction[0]))
157 if lastaction[0] == 'delete':
158 continue
159 filelist.append(item[0])
136 160
137 if host: 161 return filelist
138 p4opt += " -p %s" % (host)
139 162
140 p4cmd = d.getVar('FETCHCMD_p4', True) or "p4" 163 def download(self, ud, d):
164 """ Get the list of files, fetch each one """
165 filelist = self._p4listfiles(ud, d)
166 if not filelist:
167 raise FetchError('No files found in depot %s@%s' % (ud.host, ud.path))
141 168
142 # create temp directory 169 bb.utils.remove(ud.pkgdir, True)
143 logger.debug(2, "Fetch: creating temporary directory") 170 bb.utils.mkdirhier(ud.pkgdir)
144 bb.utils.mkdirhier(d.expand('${WORKDIR}')) 171 os.chdir(ud.pkgdir)
145 mktemp = d.getVar("FETCHCMD_p4mktemp", True) or d.expand("mktemp -d -q '${WORKDIR}/oep4.XXXXXX'")
146 tmpfile, errors = bb.process.run(mktemp)
147 tmpfile = tmpfile.strip()
148 if not tmpfile:
149 raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", ud.url)
150 172
151 if "label" in parm: 173 for afile in filelist:
152 depot = "%s@%s" % (depot, parm["label"]) 174 p4fetchcmd = self._buildp4command(ud, d, 'print', afile)
153 else: 175 bb.fetch2.check_network_access(d, p4fetchcmd)
154 cset = Perforce.getcset(d, depot, host, user, pswd, parm) 176 runfetchcmd(p4fetchcmd, d)
155 depot = "%s@%s" % (depot, cset)
156 177
157 os.chdir(tmpfile) 178 os.chdir(ud.pkgdir)
158 logger.info("Fetch " + ud.url) 179 runfetchcmd('tar -czf %s p4' % (ud.localpath), d, cleanup = [ud.localpath])
159 logger.info("%s%s files %s", p4cmd, p4opt, depot)
160 p4file, errors = bb.process.run("%s%s files %s" % (p4cmd, p4opt, depot))
161 p4file = [f.rstrip() for f in p4file.splitlines()]
162 180
163 if not p4file: 181 def clean(self, ud, d):
164 raise FetchError("Fetch: unable to get the P4 files from %s" % depot, ud.url) 182 """ Cleanup p4 specific files and dirs"""
183 bb.utils.remove(ud.localpath)
184 bb.utils.remove(ud.pkgdir, True)
165 185
166 count = 0 186 def supports_srcrev(self):
187 return True
167 188
168 for file in p4file: 189 def _revision_key(self, ud, d, name):
169 list = file.split() 190 """ Return a unique key for the url """
191 return 'p4:%s' % ud.pkgdir
170 192
171 if list[2] == "delete": 193 def _latest_revision(self, ud, d, name):
172 continue 194 """ Return the latest upstream scm revision number """
195 p4cmd = self._buildp4command(ud, d, "changes")
196 bb.fetch2.check_network_access(d, p4cmd)
197 tip = runfetchcmd(p4cmd, d, True)
198
199 if not tip:
200 raise FetchError('Could not determine the latest perforce changelist')
173 201
174 dest = list[0][len(path)+1:] 202 tipcset = tip.split(' ')[1]
175 where = dest.find("#") 203 logger.debug(1, 'p4 tip found to be changelist %s' % tipcset)
204 return tipcset
176 205
177 subprocess.call("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0]), shell=True) 206 def sortable_revision(self, ud, d, name):
178 count = count + 1 207 """ Return a sortable revision number """
208 return False, self._build_revision(ud, d)
179 209
180 if count == 0: 210 def _build_revision(self, ud, d):
181 logger.error() 211 return ud.revision
182 raise FetchError("Fetch: No files gathered from the P4 fetch", ud.url)
183 212
184 runfetchcmd("tar -czf %s %s" % (ud.localpath, module), d, cleanup = [ud.localpath])
185 # cleanup
186 bb.utils.prunedir(tmpfile)