diff options
Diffstat (limited to 'scripts/lib/devtool/standard.py')
-rw-r--r-- | scripts/lib/devtool/standard.py | 231 |
1 files changed, 87 insertions, 144 deletions
diff --git a/scripts/lib/devtool/standard.py b/scripts/lib/devtool/standard.py index 6674e67267..1d0fe13788 100644 --- a/scripts/lib/devtool/standard.py +++ b/scripts/lib/devtool/standard.py | |||
@@ -387,6 +387,19 @@ def _git_ls_tree(repodir, treeish='HEAD', recursive=False): | |||
387 | ret[split[3]] = split[0:3] | 387 | ret[split[3]] = split[0:3] |
388 | return ret | 388 | return ret |
389 | 389 | ||
390 | def _git_modified(repodir): | ||
391 | """List the difference between HEAD and the index""" | ||
392 | import bb | ||
393 | cmd = ['git', 'status', '--porcelain'] | ||
394 | out, _ = bb.process.run(cmd, cwd=repodir) | ||
395 | ret = [] | ||
396 | if out: | ||
397 | for line in out.split("\n"): | ||
398 | if line and not line.startswith('??'): | ||
399 | ret.append(line[3:]) | ||
400 | return ret | ||
401 | |||
402 | |||
390 | def _git_exclude_path(srctree, path): | 403 | def _git_exclude_path(srctree, path): |
391 | """Return pathspec (list of paths) that excludes certain path""" | 404 | """Return pathspec (list of paths) that excludes certain path""" |
392 | # NOTE: "Filtering out" files/paths in this way is not entirely reliable - | 405 | # NOTE: "Filtering out" files/paths in this way is not entirely reliable - |
@@ -460,32 +473,6 @@ def sync(args, config, basepath, workspace): | |||
460 | finally: | 473 | finally: |
461 | tinfoil.shutdown() | 474 | tinfoil.shutdown() |
462 | 475 | ||
463 | def symlink_oelocal_files_srctree(rd, srctree): | ||
464 | import oe.patch | ||
465 | if os.path.abspath(rd.getVar('S')) == os.path.abspath(rd.getVar('WORKDIR')): | ||
466 | # If recipe extracts to ${WORKDIR}, symlink the files into the srctree | ||
467 | # (otherwise the recipe won't build as expected) | ||
468 | local_files_dir = os.path.join(srctree, 'oe-local-files') | ||
469 | addfiles = [] | ||
470 | for root, _, files in os.walk(local_files_dir): | ||
471 | relpth = os.path.relpath(root, local_files_dir) | ||
472 | if relpth != '.': | ||
473 | bb.utils.mkdirhier(os.path.join(srctree, relpth)) | ||
474 | for fn in files: | ||
475 | if fn == '.gitignore': | ||
476 | continue | ||
477 | destpth = os.path.join(srctree, relpth, fn) | ||
478 | if os.path.exists(destpth): | ||
479 | os.unlink(destpth) | ||
480 | if relpth != '.': | ||
481 | back_relpth = os.path.relpath(local_files_dir, root) | ||
482 | os.symlink('%s/oe-local-files/%s/%s' % (back_relpth, relpth, fn), destpth) | ||
483 | else: | ||
484 | os.symlink('oe-local-files/%s' % fn, destpth) | ||
485 | addfiles.append(os.path.join(relpth, fn)) | ||
486 | if addfiles: | ||
487 | oe.patch.GitApplyTree.commitIgnored("Add local file symlinks", dir=srctree, files=addfiles, d=rd) | ||
488 | |||
489 | def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): | 476 | def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, workspace, fixed_setup, d, tinfoil, no_overrides=False): |
490 | """Extract sources of a recipe""" | 477 | """Extract sources of a recipe""" |
491 | import oe.recipeutils | 478 | import oe.recipeutils |
@@ -657,35 +644,22 @@ def _extract_source(srctree, keep_temp, devbranch, sync, config, basepath, works | |||
657 | elif not os.path.exists(workshareddir): | 644 | elif not os.path.exists(workshareddir): |
658 | oe.path.copyhardlinktree(srcsubdir, workshareddir) | 645 | oe.path.copyhardlinktree(srcsubdir, workshareddir) |
659 | 646 | ||
660 | tempdir_localdir = os.path.join(tempdir, 'oe-local-files') | ||
661 | srctree_localdir = os.path.join(srctree, 'oe-local-files') | ||
662 | |||
663 | if sync: | 647 | if sync: |
664 | bb.process.run('git fetch file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) | 648 | try: |
665 | 649 | logger.info('Backing up current %s branch as branch: %s.bak' % (devbranch, devbranch)) | |
666 | # Move the oe-local-files directory to srctree. | 650 | bb.process.run('git branch -f ' + devbranch + '.bak', cwd=srctree) |
667 | # As oe-local-files is not part of the constructed git tree, | 651 | |
668 | # removing it directly during the synchronization might surprise | 652 | # Use git fetch to update the source with the current recipe |
669 | # the user. Instead, we move it to oe-local-files.bak and remind | 653 | # To be able to update the currently checked out branch with |
670 | # the user in the log message. | 654 | # possibly new history (no fast-forward) git needs to be told |
671 | if os.path.exists(srctree_localdir + '.bak'): | 655 | # that's ok |
672 | shutil.rmtree(srctree_localdir + '.bak') | 656 | logger.info('Syncing source files including patches to git branch: %s' % devbranch) |
673 | 657 | bb.process.run('git fetch --update-head-ok --force file://' + srcsubdir + ' ' + devbranch + ':' + devbranch, cwd=srctree) | |
674 | if os.path.exists(srctree_localdir): | 658 | except bb.process.ExecutionError as e: |
675 | logger.info('Backing up current local file directory %s' % srctree_localdir) | 659 | raise DevtoolError("Error when syncing source files to local checkout: %s" % str(e)) |
676 | shutil.move(srctree_localdir, srctree_localdir + '.bak') | ||
677 | |||
678 | if os.path.exists(tempdir_localdir): | ||
679 | logger.info('Syncing local source files to srctree...') | ||
680 | shutil.copytree(tempdir_localdir, srctree_localdir) | ||
681 | else: | ||
682 | # Move oe-local-files directory to srctree | ||
683 | if os.path.exists(tempdir_localdir): | ||
684 | logger.info('Adding local source files to srctree...') | ||
685 | shutil.move(tempdir_localdir, srcsubdir) | ||
686 | 660 | ||
661 | else: | ||
687 | shutil.move(srcsubdir, srctree) | 662 | shutil.move(srcsubdir, srctree) |
688 | symlink_oelocal_files_srctree(d, srctree) | ||
689 | 663 | ||
690 | if is_kernel_yocto: | 664 | if is_kernel_yocto: |
691 | logger.info('Copying kernel config to srctree') | 665 | logger.info('Copying kernel config to srctree') |
@@ -841,34 +815,22 @@ def modify(args, config, basepath, workspace): | |||
841 | if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): | 815 | if (os.path.exists(srcdir) and os.listdir(srcdir)) and (kernelVersion in staging_kerVer and staging_kbranch == kbranch): |
842 | oe.path.copyhardlinktree(srcdir, srctree) | 816 | oe.path.copyhardlinktree(srcdir, srctree) |
843 | workdir = rd.getVar('WORKDIR') | 817 | workdir = rd.getVar('WORKDIR') |
818 | unpackdir = rd.getVar('UNPACKDIR') | ||
844 | srcsubdir = rd.getVar('S') | 819 | srcsubdir = rd.getVar('S') |
845 | localfilesdir = os.path.join(srctree, 'oe-local-files') | 820 | localfilesdir = os.path.join(srctree, 'oe-local-files') |
846 | # Move local source files into separate subdir | ||
847 | recipe_patches = [os.path.basename(patch) for patch in oe.recipeutils.get_recipe_patches(rd)] | ||
848 | local_files = oe.recipeutils.get_recipe_local_files(rd) | ||
849 | 821 | ||
850 | for key in local_files.copy(): | 822 | # Add locally copied files to gitignore as we add back to the metadata directly |
851 | if key.endswith('scc'): | 823 | local_files = oe.recipeutils.get_recipe_local_files(rd) |
852 | sccfile = open(local_files[key], 'r') | ||
853 | for l in sccfile: | ||
854 | line = l.split() | ||
855 | if line and line[0] in ('kconf', 'patch'): | ||
856 | cfg = os.path.join(os.path.dirname(local_files[key]), line[-1]) | ||
857 | if not cfg in local_files.values(): | ||
858 | local_files[line[-1]] = cfg | ||
859 | shutil.copy2(cfg, workdir) | ||
860 | sccfile.close() | ||
861 | |||
862 | # Ignore local files with subdir={BP} | ||
863 | srcabspath = os.path.abspath(srcsubdir) | 824 | srcabspath = os.path.abspath(srcsubdir) |
864 | local_files = [fname for fname in local_files if os.path.exists(os.path.join(workdir, fname)) and (srcabspath == workdir or not os.path.join(workdir, fname).startswith(srcabspath + os.sep))] | 825 | local_files = [fname for fname in local_files if |
826 | os.path.exists(os.path.join(unpackdir, fname)) and | ||
827 | srcabspath == unpackdir] | ||
865 | if local_files: | 828 | if local_files: |
866 | for fname in local_files: | 829 | with open(os.path.join(srctree, '.gitignore'), 'a+') as f: |
867 | _move_file(os.path.join(workdir, fname), os.path.join(srctree, 'oe-local-files', fname)) | 830 | f.write('# Ignore local files, by default. Remove following lines' |
868 | with open(os.path.join(srctree, 'oe-local-files', '.gitignore'), 'w') as f: | 831 | 'if you want to commit the directory to Git\n') |
869 | f.write('# Ignore local files, by default. Remove this file if you want to commit the directory to Git\n*\n') | 832 | for fname in local_files: |
870 | 833 | f.write('%s\n' % fname) | |
871 | symlink_oelocal_files_srctree(rd, srctree) | ||
872 | 834 | ||
873 | task = 'do_configure' | 835 | task = 'do_configure' |
874 | res = tinfoil.build_targets(pn, task, handle_events=True) | 836 | res = tinfoil.build_targets(pn, task, handle_events=True) |
@@ -893,7 +855,10 @@ def modify(args, config, basepath, workspace): | |||
893 | (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) | 855 | (stdout, _) = bb.process.run('git rev-list --reverse %s..HEAD' % initial_revs["."], cwd=srctree) |
894 | commits["."] = stdout.split() | 856 | commits["."] = stdout.split() |
895 | check_commits = True | 857 | check_commits = True |
896 | (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) | 858 | try: |
859 | (stdout, _) = bb.process.run('git submodule --quiet foreach --recursive \'echo `git rev-parse devtool-base` $PWD\'', cwd=srctree) | ||
860 | except bb.process.ExecutionError: | ||
861 | stdout = "" | ||
897 | for line in stdout.splitlines(): | 862 | for line in stdout.splitlines(): |
898 | (rev, submodule_path) = line.split() | 863 | (rev, submodule_path) = line.split() |
899 | submodule = os.path.relpath(submodule_path, srctree) | 864 | submodule = os.path.relpath(submodule_path, srctree) |
@@ -1452,8 +1417,10 @@ def _export_local_files(srctree, rd, destdir, srctreebase): | |||
1452 | 1. updated - files that already exist in SRCURI | 1417 | 1. updated - files that already exist in SRCURI |
1453 | 2. added - new files files that don't exist in SRCURI | 1418 | 2. added - new files files that don't exist in SRCURI |
1454 | 3 removed - files that exist in SRCURI but not in exported files | 1419 | 3 removed - files that exist in SRCURI but not in exported files |
1455 | In each dict the key is the 'basepath' of the URI and value is the | 1420 | In each dict the key is the 'basepath' of the URI and value is: |
1456 | absolute path to the existing file in recipe space (if any). | 1421 | - for updated and added dicts, a dict with 1 optionnal key: |
1422 | - 'path': the absolute path to the existing file in recipe space (if any) | ||
1423 | - for removed dict, the absolute path to the existing file in recipe space | ||
1457 | """ | 1424 | """ |
1458 | import oe.recipeutils | 1425 | import oe.recipeutils |
1459 | 1426 | ||
@@ -1462,6 +1429,7 @@ def _export_local_files(srctree, rd, destdir, srctreebase): | |||
1462 | # Instead they are directly copied over the original source files (in | 1429 | # Instead they are directly copied over the original source files (in |
1463 | # recipe space). | 1430 | # recipe space). |
1464 | existing_files = oe.recipeutils.get_recipe_local_files(rd) | 1431 | existing_files = oe.recipeutils.get_recipe_local_files(rd) |
1432 | |||
1465 | new_set = None | 1433 | new_set = None |
1466 | updated = OrderedDict() | 1434 | updated = OrderedDict() |
1467 | added = OrderedDict() | 1435 | added = OrderedDict() |
@@ -1478,24 +1446,28 @@ def _export_local_files(srctree, rd, destdir, srctreebase): | |||
1478 | if branchname.startswith(override_branch_prefix): | 1446 | if branchname.startswith(override_branch_prefix): |
1479 | return (updated, added, removed) | 1447 | return (updated, added, removed) |
1480 | 1448 | ||
1481 | local_files_dir = os.path.join(srctreebase, 'oe-local-files') | 1449 | files = _git_modified(srctree) |
1482 | git_files = _git_ls_tree(srctree) | 1450 | #if not files: |
1483 | if 'oe-local-files' in git_files: | 1451 | # files = _ls_tree(srctree) |
1484 | # If tracked by Git, take the files from srctree HEAD. First get | 1452 | for f in files: |
1485 | # the tree object of the directory | 1453 | fullfile = os.path.join(srctree, f) |
1486 | tmp_index = os.path.join(srctree, '.git', 'index.tmp.devtool') | 1454 | if os.path.exists(os.path.join(fullfile, ".git")): |
1487 | tree = git_files['oe-local-files'][2] | 1455 | # submodules handled elsewhere |
1488 | bb.process.run(['git', 'checkout', tree, '--', '.'], cwd=srctree, | 1456 | continue |
1489 | env=dict(os.environ, GIT_WORK_TREE=destdir, | 1457 | if f not in existing_files: |
1490 | GIT_INDEX_FILE=tmp_index)) | 1458 | added[f] = {} |
1491 | new_set = list(_git_ls_tree(srctree, tree, True).keys()) | 1459 | if os.path.isdir(os.path.join(srctree, f)): |
1492 | elif os.path.isdir(local_files_dir): | 1460 | shutil.copytree(fullfile, os.path.join(destdir, f)) |
1493 | # If not tracked by Git, just copy from working copy | 1461 | else: |
1494 | new_set = _ls_tree(local_files_dir) | 1462 | shutil.copy2(fullfile, os.path.join(destdir, f)) |
1495 | bb.process.run(['cp', '-ax', | 1463 | elif not os.path.exists(fullfile): |
1496 | os.path.join(local_files_dir, '.'), destdir]) | 1464 | removed[f] = existing_files[f] |
1497 | else: | 1465 | elif f in existing_files: |
1498 | new_set = [] | 1466 | updated[f] = {'path' : existing_files[f]} |
1467 | if os.path.isdir(os.path.join(srctree, f)): | ||
1468 | shutil.copytree(fullfile, os.path.join(destdir, f)) | ||
1469 | else: | ||
1470 | shutil.copy2(fullfile, os.path.join(destdir, f)) | ||
1499 | 1471 | ||
1500 | # Special handling for kernel config | 1472 | # Special handling for kernel config |
1501 | if bb.data.inherits_class('kernel-yocto', rd): | 1473 | if bb.data.inherits_class('kernel-yocto', rd): |
@@ -1503,17 +1475,14 @@ def _export_local_files(srctree, rd, destdir, srctreebase): | |||
1503 | fragment_path = os.path.join(destdir, fragment_fn) | 1475 | fragment_path = os.path.join(destdir, fragment_fn) |
1504 | if _create_kconfig_diff(srctree, rd, fragment_path): | 1476 | if _create_kconfig_diff(srctree, rd, fragment_path): |
1505 | if os.path.exists(fragment_path): | 1477 | if os.path.exists(fragment_path): |
1506 | if fragment_fn not in new_set: | 1478 | if fragment_fn in removed: |
1507 | new_set.append(fragment_fn) | 1479 | del removed[fragment_fn] |
1508 | # Copy fragment to local-files | 1480 | if fragment_fn not in updated and fragment_fn not in added: |
1509 | if os.path.isdir(local_files_dir): | 1481 | added[fragment_fn] = {} |
1510 | shutil.copy2(fragment_path, local_files_dir) | ||
1511 | else: | 1482 | else: |
1512 | if fragment_fn in new_set: | 1483 | if fragment_fn in updated: |
1513 | new_set.remove(fragment_fn) | 1484 | revoved[fragment_fn] = updated[fragment_fn] |
1514 | # Remove fragment from local-files | 1485 | del updated[fragment_fn] |
1515 | if os.path.exists(os.path.join(local_files_dir, fragment_fn)): | ||
1516 | os.unlink(os.path.join(local_files_dir, fragment_fn)) | ||
1517 | 1486 | ||
1518 | # Special handling for cml1, ccmake, etc bbclasses that generated | 1487 | # Special handling for cml1, ccmake, etc bbclasses that generated |
1519 | # configuration fragment files that are consumed as source files | 1488 | # configuration fragment files that are consumed as source files |
@@ -1521,42 +1490,13 @@ def _export_local_files(srctree, rd, destdir, srctreebase): | |||
1521 | if bb.data.inherits_class(frag_class, rd): | 1490 | if bb.data.inherits_class(frag_class, rd): |
1522 | srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) | 1491 | srcpath = os.path.join(rd.getVar('WORKDIR'), frag_name) |
1523 | if os.path.exists(srcpath): | 1492 | if os.path.exists(srcpath): |
1524 | if frag_name not in new_set: | 1493 | if frag_name in removed: |
1525 | new_set.append(frag_name) | 1494 | del removed[frag_name] |
1495 | if frag_name not in updated: | ||
1496 | added[frag_name] = {} | ||
1526 | # copy fragment into destdir | 1497 | # copy fragment into destdir |
1527 | shutil.copy2(srcpath, destdir) | 1498 | shutil.copy2(srcpath, destdir) |
1528 | # copy fragment into local files if exists | 1499 | |
1529 | if os.path.isdir(local_files_dir): | ||
1530 | shutil.copy2(srcpath, local_files_dir) | ||
1531 | |||
1532 | if new_set is not None: | ||
1533 | for fname in new_set: | ||
1534 | if fname in existing_files: | ||
1535 | origpath = existing_files.pop(fname) | ||
1536 | workpath = os.path.join(local_files_dir, fname) | ||
1537 | if not filecmp.cmp(origpath, workpath): | ||
1538 | updated[fname] = origpath | ||
1539 | elif fname != '.gitignore': | ||
1540 | added[fname] = None | ||
1541 | |||
1542 | workdir = rd.getVar('WORKDIR') | ||
1543 | s = rd.getVar('S') | ||
1544 | if not s.endswith(os.sep): | ||
1545 | s += os.sep | ||
1546 | |||
1547 | if workdir != s: | ||
1548 | # Handle files where subdir= was specified | ||
1549 | for fname in list(existing_files.keys()): | ||
1550 | # FIXME handle both subdir starting with BP and not? | ||
1551 | fworkpath = os.path.join(workdir, fname) | ||
1552 | if fworkpath.startswith(s): | ||
1553 | fpath = os.path.join(srctree, os.path.relpath(fworkpath, s)) | ||
1554 | if os.path.exists(fpath): | ||
1555 | origpath = existing_files.pop(fname) | ||
1556 | if not filecmp.cmp(origpath, fpath): | ||
1557 | updated[fpath] = origpath | ||
1558 | |||
1559 | removed = existing_files | ||
1560 | return (updated, added, removed) | 1500 | return (updated, added, removed) |
1561 | 1501 | ||
1562 | 1502 | ||
@@ -1640,7 +1580,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi | |||
1640 | redirect_output=dry_run_outdir) | 1580 | redirect_output=dry_run_outdir) |
1641 | else: | 1581 | else: |
1642 | files_dir = _determine_files_dir(rd) | 1582 | files_dir = _determine_files_dir(rd) |
1643 | for basepath, path in upd_f.items(): | 1583 | for basepath, param in upd_f.items(): |
1584 | path = param['path'] | ||
1644 | logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) | 1585 | logger.info('Updating file %s%s' % (basepath, dry_run_suffix)) |
1645 | if os.path.isabs(basepath): | 1586 | if os.path.isabs(basepath): |
1646 | # Original file (probably with subdir pointing inside source tree) | 1587 | # Original file (probably with subdir pointing inside source tree) |
@@ -1650,7 +1591,8 @@ def _update_recipe_srcrev(recipename, workspace, srctree, rd, appendlayerdir, wi | |||
1650 | _move_file(os.path.join(local_files_dir, basepath), path, | 1591 | _move_file(os.path.join(local_files_dir, basepath), path, |
1651 | dry_run_outdir=dry_run_outdir, base_outdir=recipedir) | 1592 | dry_run_outdir=dry_run_outdir, base_outdir=recipedir) |
1652 | update_srcuri= True | 1593 | update_srcuri= True |
1653 | for basepath, path in new_f.items(): | 1594 | for basepath, param in new_f.items(): |
1595 | path = param['path'] | ||
1654 | logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) | 1596 | logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) |
1655 | _move_file(os.path.join(local_files_dir, basepath), | 1597 | _move_file(os.path.join(local_files_dir, basepath), |
1656 | os.path.join(files_dir, basepath), | 1598 | os.path.join(files_dir, basepath), |
@@ -1772,7 +1714,8 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil | |||
1772 | else: | 1714 | else: |
1773 | # Update existing files | 1715 | # Update existing files |
1774 | files_dir = _determine_files_dir(rd) | 1716 | files_dir = _determine_files_dir(rd) |
1775 | for basepath, path in upd_f.items(): | 1717 | for basepath, param in upd_f.items(): |
1718 | path = param['path'] | ||
1776 | logger.info('Updating file %s' % basepath) | 1719 | logger.info('Updating file %s' % basepath) |
1777 | if os.path.isabs(basepath): | 1720 | if os.path.isabs(basepath): |
1778 | # Original file (probably with subdir pointing inside source tree) | 1721 | # Original file (probably with subdir pointing inside source tree) |
@@ -1806,7 +1749,7 @@ def _update_recipe_patch(recipename, workspace, srctree, rd, appendlayerdir, wil | |||
1806 | dry_run_outdir=dry_run_outdir, base_outdir=recipedir) | 1749 | dry_run_outdir=dry_run_outdir, base_outdir=recipedir) |
1807 | updatefiles = True | 1750 | updatefiles = True |
1808 | # Add any new files | 1751 | # Add any new files |
1809 | for basepath, path in new_f.items(): | 1752 | for basepath, param in new_f.items(): |
1810 | logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) | 1753 | logger.info('Adding new file %s%s' % (basepath, dry_run_suffix)) |
1811 | _move_file(os.path.join(local_files_dir, basepath), | 1754 | _move_file(os.path.join(local_files_dir, basepath), |
1812 | os.path.join(files_dir, basepath), | 1755 | os.path.join(files_dir, basepath), |