From 8b8be74ed21b878b2fe30d5b76ff0648e6e48c18 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Wed, 23 May 2012 00:23:32 +0100 Subject: bitbake: implement checksums for local files in SRC_URI Gathers a list of paths to have checksums calculated at parse time, and processes these when calculating task hashes. Checksums are cached with the file's current mtime. Thus, changing any local file in SRC_URI will now cause the do_fetch taskhash to change, thus forcing a rebuild. This change adds very roughly about an 8% increase in parse time (a few seconds) and maybe a few seconds during runqueue generation, so a fairly moderate performance hit. Note that since paths are resolved at parse time, this will not force a rebuild when files are introduced which would cause that resolved path to be different - for example, where a machine-specific version of a file was added without otherwise changing the recipe. This will need to be handled in a future update. Code to hook this into the signature generator was courtesy of Richard Purdie . Implements [YOCTO #2044]. (Bitbake rev: c993b7c457f8b7776e8a5dff253bfa0724bc2cae) Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- bitbake/lib/bb/fetch2/__init__.py | 85 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) (limited to 'bitbake/lib/bb/fetch2/__init__.py') diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 0b976c4079..d4b6c3ec39 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -8,6 +8,7 @@ BitBake build tools. """ # Copyright (C) 2003, 2004 Chris Larson +# Copyright (C) 2012 Intel Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 as @@ -30,9 +31,11 @@ import os, re import logging import urllib import bb.persist_data, bb.utils +import bb.checksum from bb import data __version__ = "2" +_checksum_cache = bb.checksum.FileChecksumCache() logger = logging.getLogger("BitBake.Fetcher") @@ -233,10 +236,18 @@ def fetcher_init(d): else: raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy) + _checksum_cache.init_cache(d) + for m in methods: if hasattr(m, "init"): m.init(d) +def fetcher_parse_save(d): + _checksum_cache.save_extras(d) + +def fetcher_parse_done(d): + _checksum_cache.save_merge(d) + def fetcher_compare_revisions(d): """ Compare the revisions in the persistant cache with current values and @@ -553,6 +564,80 @@ def srcrev_internal_helper(ud, d, name): return rev + +def get_checksum_file_list(d): + """ Get a list of files checksum in SRC_URI + + Returns the all resolved local path of all local file entries in + SRC_URI as a space-separated string + """ + fetch = Fetch([], d) + + dl_dir = d.getVar('DL_DIR', True) + filelist = [] + for u in fetch.urls: + ud = fetch.ud[u] + + if isinstance(ud.method, local.Local): + ud.setup_localpath(d) + f = ud.localpath + if f.startswith(dl_dir): + # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else + if os.path.exists(f): + bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f))) + else: + bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f))) + continue + filelist.append(f) + + return " ".join(filelist) + + +def get_file_checksums(filelist, pn): + """Get a list of the checksums for a list of local files + + Returns the checksums for a list of local files, caching the results as + it proceeds + + """ + + def checksum_file(f): + try: + checksum = _checksum_cache.get_checksum(f) + except OSError as e: + import traceback + bb.warn("Unable to get checksum for %s SRC_URI entry %s: %s" % (pn, os.path.basename(f), e)) + return None + return checksum + + checksums = [] + for pth in filelist.split(): + checksum = None + if '*' in pth: + # Handle globs + import glob + for f in glob.glob(pth): + checksum = checksum_file(f) + if checksum: + checksums.append((f, checksum)) + elif os.path.isdir(pth): + # Handle directories + for root, dirs, files in os.walk(pth): + for name in files: + fullpth = os.path.join(root, name) + checksum = checksum_file(fullpth) + if checksum: + checksums.append((fullpth, checksum)) + else: + checksum = checksum_file(pth) + + if checksum: + checksums.append((pth, checksum)) + + checksums.sort() + return checksums + + class FetchData(object): """ A class which represents the fetcher state for a given URI. -- cgit v1.2.3-54-g00ecf