From 0f84d24df8b83877b45b5d6aa3eca401b059071e Mon Sep 17 00:00:00 2001 From: Alejandro Hernandez Samaniego Date: Wed, 24 Feb 2021 10:26:32 -0700 Subject: bitbake: bitbake: Add Azure Storage fetcher implementation Allows bitbake to fetch from an Azure Storage account. The fetcher submodule is compatible with the az:// URI protocol, its functionality is based on bitbakes wget fetcher, superior in performance to using a propietary tool like azcopy which can handle cloud storage account operations with more functionality (that we dont need in a fetcher) but less compatibility. A sample URI uses can be defined in the following way: SRC_URI = "az://.blob.core.windows.net//foo.tar.xz" This fetcher can easily be used with PREMIRRORS and SSTATE_MIRRORS, e.g.: SSTATE_MIRRORS = "file://.* az://.blob.core.windows.net/sstate-cache/PATH;downloadfilename=PATH \n" PREMIRRORS_prepend = "\ git://.*/.* az://.blob.core.windows.net/downloads/ \n \ ftp://.*/.* az://.blob.core.windows.net/downloads/ \n \ http://.*/.* az://.blob.core.windows.net/downloads/ \n \ https://.*/.* az://.blob.core.windows.net/downloads/ \n \ " Can also be used with non-public access Azure Storage accounts/containers via a Shared Access Signature by declaring the AZ_SAS variable which will be automatically used by the fetcher: AZ_SAS="?sv=2000-01-01&ss=...&sig=somesignature" (Bitbake rev: b103b02f2ce2f8f5079f17ec1a854f904c2110a4) Signed-off-by: Alejandro Enedino Hernandez Samaniego Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 4 +- bitbake/lib/bb/fetch2/az.py | 93 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 bitbake/lib/bb/fetch2/az.py (limited to 'bitbake/lib') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 19169d780f..cf0201c490 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -1243,7 +1243,7 @@ class FetchData(object): if checksum_name in self.parm: checksum_expected = self.parm[checksum_name] - elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]: + elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az"]: checksum_expected = None else: checksum_expected = d.getVarFlag("SRC_URI", checksum_name) @@ -1908,6 +1908,7 @@ from . import repo from . import clearcase from . import npm from . import npmsw +from . import az methods.append(local.Local()) methods.append(wget.Wget()) @@ -1927,3 +1928,4 @@ methods.append(repo.Repo()) methods.append(clearcase.ClearCase()) methods.append(npm.Npm()) methods.append(npmsw.NpmShrinkWrap()) +methods.append(az.Az()) diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py new file mode 100644 index 0000000000..3ccc594c22 --- /dev/null +++ b/bitbake/lib/bb/fetch2/az.py @@ -0,0 +1,93 @@ +""" +BitBake 'Fetch' Azure Storage implementation + +""" + +# Copyright (C) 2021 Alejandro Hernandez Samaniego +# +# Based on bb.fetch2.wget: +# Copyright (C) 2003, 2004 Chris Larson +# +# SPDX-License-Identifier: GPL-2.0-only +# +# Based on functions from the base bb module, Copyright 2003 Holger Schurig + +import shlex +import os +import bb +from bb.fetch2 import FetchError +from bb.fetch2 import logger +from bb.fetch2.wget import Wget + + +class Az(Wget): + + def supports(self, ud, d): + """ + Check to see if a given url can be fetched from Azure Storage + """ + return ud.type in ['az'] + + + def checkstatus(self, fetch, ud, d, try_again=True): + + # checkstatus discards parameters either way, we need to do this before adding the SAS + ud.url = ud.url.replace('az://','https://').split(';')[0] + + az_sas = d.getVar('AZ_SAS') + if az_sas and az_sas not in ud.url: + ud.url += az_sas + + return Wget.checkstatus(self, fetch, ud, d, try_again) + + # Override download method, include retries + def download(self, ud, d, retries=3): + """Fetch urls""" + + # If were reaching the account transaction limit we might be refused a connection, + # retrying allows us to avoid false negatives since the limit changes over time + fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' + + # We need to provide a localpath to avoid wget using the SAS + # ud.localfile either has the downloadfilename or ud.path + localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) + bb.utils.mkdirhier(os.path.dirname(localpath)) + fetchcmd += " -O %s" % shlex.quote(localpath) + + + if ud.user and ud.pswd: + fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd) + + # Check if a Shared Access Signature was given and use it + az_sas = d.getVar('AZ_SAS') + + if az_sas: + azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) + else: + azuri = '%s%s%s' % ('https://', ud.host, ud.path) + + if os.path.exists(ud.localpath): + # file exists, but we didnt complete it.. trying again. + fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri) + else: + fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri) + + try: + self._runwget(ud, d, fetchcmd, False) + except FetchError as e: + # Azure fails on handshake sometimes when using wget after some stress, producing a + # FetchError from the fetcher, if the artifact exists retyring should succeed + if 'Unable to establish SSL connection' in str(e): + logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) + self.download(ud, d, retries -1) + + # Sanity check since wget can pretend it succeed when it didn't + # Also, this used to happen if sourceforge sent us to the mirror page + if not os.path.exists(ud.localpath): + raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) + + if os.path.getsize(ud.localpath) == 0: + os.remove(ud.localpath) + raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) + + return True -- cgit v1.2.3-54-g00ecf