diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/az.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/az.py | 93 |
1 files changed, 93 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py new file mode 100644 index 0000000000..3ccc594c22 --- /dev/null +++ b/bitbake/lib/bb/fetch2/az.py | |||
@@ -0,0 +1,93 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' Azure Storage implementation | ||
3 | |||
4 | """ | ||
5 | |||
6 | # Copyright (C) 2021 Alejandro Hernandez Samaniego | ||
7 | # | ||
8 | # Based on bb.fetch2.wget: | ||
9 | # Copyright (C) 2003, 2004 Chris Larson | ||
10 | # | ||
11 | # SPDX-License-Identifier: GPL-2.0-only | ||
12 | # | ||
13 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
14 | |||
15 | import shlex | ||
16 | import os | ||
17 | import bb | ||
18 | from bb.fetch2 import FetchError | ||
19 | from bb.fetch2 import logger | ||
20 | from bb.fetch2.wget import Wget | ||
21 | |||
22 | |||
23 | class Az(Wget): | ||
24 | |||
25 | def supports(self, ud, d): | ||
26 | """ | ||
27 | Check to see if a given url can be fetched from Azure Storage | ||
28 | """ | ||
29 | return ud.type in ['az'] | ||
30 | |||
31 | |||
32 | def checkstatus(self, fetch, ud, d, try_again=True): | ||
33 | |||
34 | # checkstatus discards parameters either way, we need to do this before adding the SAS | ||
35 | ud.url = ud.url.replace('az://','https://').split(';')[0] | ||
36 | |||
37 | az_sas = d.getVar('AZ_SAS') | ||
38 | if az_sas and az_sas not in ud.url: | ||
39 | ud.url += az_sas | ||
40 | |||
41 | return Wget.checkstatus(self, fetch, ud, d, try_again) | ||
42 | |||
43 | # Override download method, include retries | ||
44 | def download(self, ud, d, retries=3): | ||
45 | """Fetch urls""" | ||
46 | |||
47 | # If were reaching the account transaction limit we might be refused a connection, | ||
48 | # retrying allows us to avoid false negatives since the limit changes over time | ||
49 | fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' | ||
50 | |||
51 | # We need to provide a localpath to avoid wget using the SAS | ||
52 | # ud.localfile either has the downloadfilename or ud.path | ||
53 | localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) | ||
54 | bb.utils.mkdirhier(os.path.dirname(localpath)) | ||
55 | fetchcmd += " -O %s" % shlex.quote(localpath) | ||
56 | |||
57 | |||
58 | if ud.user and ud.pswd: | ||
59 | fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) | ||
60 | |||
61 | # Check if a Shared Access Signature was given and use it | ||
62 | az_sas = d.getVar('AZ_SAS') | ||
63 | |||
64 | if az_sas: | ||
65 | azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) | ||
66 | else: | ||
67 | azuri = '%s%s%s' % ('https://', ud.host, ud.path) | ||
68 | |||
69 | if os.path.exists(ud.localpath): | ||
70 | # file exists, but we didnt complete it.. trying again. | ||
71 | fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri) | ||
72 | else: | ||
73 | fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri) | ||
74 | |||
75 | try: | ||
76 | self._runwget(ud, d, fetchcmd, False) | ||
77 | except FetchError as e: | ||
78 | # Azure fails on handshake sometimes when using wget after some stress, producing a | ||
79 | # FetchError from the fetcher, if the artifact exists retyring should succeed | ||
80 | if 'Unable to establish SSL connection' in str(e): | ||
81 | logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) | ||
82 | self.download(ud, d, retries -1) | ||
83 | |||
84 | # Sanity check since wget can pretend it succeed when it didn't | ||
85 | # Also, this used to happen if sourceforge sent us to the mirror page | ||
86 | if not os.path.exists(ud.localpath): | ||
87 | raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) | ||
88 | |||
89 | if os.path.getsize(ud.localpath) == 0: | ||
90 | os.remove(ud.localpath) | ||
91 | raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) | ||
92 | |||
93 | return True | ||