diff options
Diffstat (limited to 'bitbake/lib/bb/fetch2/gcp.py')
-rw-r--r-- | bitbake/lib/bb/fetch2/gcp.py | 102 |
1 files changed, 102 insertions, 0 deletions
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py new file mode 100644 index 0000000000..eb3e0c6a6b --- /dev/null +++ b/bitbake/lib/bb/fetch2/gcp.py | |||
@@ -0,0 +1,102 @@ | |||
1 | """ | ||
2 | BitBake 'Fetch' implementation for Google Cloup Platform Storage. | ||
3 | |||
4 | Class for fetching files from Google Cloud Storage using the | ||
5 | Google Cloud Storage Python Client. The GCS Python Client must | ||
6 | be correctly installed, configured and authenticated prior to use. | ||
7 | Additionally, gsutil must also be installed. | ||
8 | |||
9 | """ | ||
10 | |||
11 | # Copyright (C) 2023, Snap Inc. | ||
12 | # | ||
13 | # Based in part on bb.fetch2.s3: | ||
14 | # Copyright (C) 2017 Andre McCurdy | ||
15 | # | ||
16 | # SPDX-License-Identifier: GPL-2.0-only | ||
17 | # | ||
18 | # Based on functions from the base bb module, Copyright 2003 Holger Schurig | ||
19 | |||
20 | import os | ||
21 | import bb | ||
22 | import urllib.parse, urllib.error | ||
23 | from bb.fetch2 import FetchMethod | ||
24 | from bb.fetch2 import FetchError | ||
25 | from bb.fetch2 import logger | ||
26 | from bb.fetch2 import runfetchcmd | ||
27 | |||
28 | class GCP(FetchMethod): | ||
29 | """ | ||
30 | Class to fetch urls via GCP's Python API. | ||
31 | """ | ||
32 | def __init__(self): | ||
33 | self.gcp_client = None | ||
34 | |||
35 | def supports(self, ud, d): | ||
36 | """ | ||
37 | Check to see if a given url can be fetched with GCP. | ||
38 | """ | ||
39 | return ud.type in ['gs'] | ||
40 | |||
41 | def recommends_checksum(self, urldata): | ||
42 | return True | ||
43 | |||
44 | def urldata_init(self, ud, d): | ||
45 | if 'downloadfilename' in ud.parm: | ||
46 | ud.basename = ud.parm['downloadfilename'] | ||
47 | else: | ||
48 | ud.basename = os.path.basename(ud.path) | ||
49 | |||
50 | ud.localfile = d.expand(urllib.parse.unquote(ud.basename)) | ||
51 | ud.basecmd = "gsutil stat" | ||
52 | |||
53 | def get_gcp_client(self): | ||
54 | from google.cloud import storage | ||
55 | self.gcp_client = storage.Client(project=None) | ||
56 | |||
57 | def download(self, ud, d): | ||
58 | """ | ||
59 | Fetch urls using the GCP API. | ||
60 | Assumes localpath was called first. | ||
61 | """ | ||
62 | logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}") | ||
63 | if self.gcp_client is None: | ||
64 | self.get_gcp_client() | ||
65 | |||
66 | bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}") | ||
67 | runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d) | ||
68 | |||
69 | # Path sometimes has leading slash, so strip it | ||
70 | path = ud.path.lstrip("/") | ||
71 | blob = self.gcp_client.bucket(ud.host).blob(path) | ||
72 | blob.download_to_filename(ud.localpath) | ||
73 | |||
74 | # Additional sanity checks copied from the wget class (although there | ||
75 | # are no known issues which mean these are required, treat the GCP API | ||
76 | # tool with a little healthy suspicion). | ||
77 | if not os.path.exists(ud.localpath): | ||
78 | raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!") | ||
79 | |||
80 | if os.path.getsize(ud.localpath) == 0: | ||
81 | os.remove(ud.localpath) | ||
82 | raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.") | ||
83 | |||
84 | return True | ||
85 | |||
86 | def checkstatus(self, fetch, ud, d): | ||
87 | """ | ||
88 | Check the status of a URL. | ||
89 | """ | ||
90 | logger.debug2(f"Checking status of gs://{ud.host}{ud.path}") | ||
91 | if self.gcp_client is None: | ||
92 | self.get_gcp_client() | ||
93 | |||
94 | bb.fetch2.check_network_access(d, ud.basecmd, f"gs://{ud.host}{ud.path}") | ||
95 | runfetchcmd("%s %s" % (ud.basecmd, f"gs://{ud.host}{ud.path}"), d) | ||
96 | |||
97 | # Path sometimes has leading slash, so strip it | ||
98 | path = ud.path.lstrip("/") | ||
99 | if self.gcp_client.bucket(ud.host).blob(path).exists() == False: | ||
100 | raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist") | ||
101 | else: | ||
102 | return True | ||