From 8cf7030eefd0902418406c0f308140310d695dc8 Mon Sep 17 00:00:00 2001 From: Peter Hurley Date: Mon, 10 Oct 2022 12:24:32 -0700 Subject: [PATCH 1/2] externalsrc: fix git submodule--helper list unsupported harder oe-core commit 6d9364e5f353 "externalsrc: git submodule--helper list unsupported" caused build errors and does the wrong thing because it: 1. checks for .gitmodules in bitbake's cwd which could be anywhere and unrelated to the srcdir/EXTERNALSRC 2. enumerates inactive submodules which won't have initialized repos oe-core commit 66ff3d1f65cd "externalsrc: fix lookup for .gitmodules" fixed 1 but not 2. Instead, still attempt to use list subcommand to enumerate active submodules, and if unsuccessful (because list is unsupported) [1], enumerate active submodule paths via the more modern "foreach" subcommand [2]. Avoiding the git submodule porcelain is still necessary for the reasons detailed in oe-core commit 2055718fdd19 "Revert "externalsrc: Detect code changes in submodules". Both subcommands output the submodule path relative to the immediate superproject; parse last column of either output [3][4] as submodule path. [1] 31955475d1c2 "submodule--helper: remove unused "list" helper" [2] foreach subcommand added to git submodule--helper in git v2.19.0 fc1b9243cd5d submodule: port submodule subcommand 'foreach' from shell to C and --quiet option fixed in git v2.22.0 a282f5a90613 submodule foreach: fix " --quiet" not being respected [3] example output of 'git submodule--helper list' $ git submodule--helper list 160000 48de465976bffb76853b2e17c1c9b65bada30d2a 0 path/to/submodule_A 160000 4c34ba86f96944153b6c6566a1bceaeace25aa07 0 deeper/path/to/submodule_B [4] example output of "git submodule--helper foreach 'echo $path'" path/to/submodule_A deeper/path/to/submodule_B Change-Id: I67915f54df7c802d1659b8f132426df60e24480f Signed-off-by: Peter Hurley --- meta/classes/externalsrc.bbclass | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index 0deb5dbf5fc..474311edae5 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass @@ -230,16 +230,18 @@ def srctree_hash_files(d, srcdir=None): env['GIT_INDEX_FILE'] = tmp_index.name subprocess.check_output(['git', 'add', '-A', '.'], cwd=s_dir, env=env) git_sha1 = subprocess.check_output(['git', 'write-tree'], cwd=s_dir, env=env).decode("utf-8") - if os.path.exists(os.path.join(s_dir, ".gitmodules")): - submodule_helper = subprocess.check_output(["git", "config", "--file", ".gitmodules", "--get-regexp", "path"], cwd=s_dir, env=env).decode("utf-8") - for line in submodule_helper.splitlines(): - module_dir = os.path.join(s_dir, line.rsplit(maxsplit=1)[1]) - if os.path.isdir(module_dir): - proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - proc.communicate() - proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - stdout, _ = proc.communicate() - git_sha1 += stdout.decode("utf-8") + try: + submodule_helper = subprocess.check_output(['git', 'submodule--helper', 'list'], cwd=s_dir, env=env).decode("utf-8") + except subprocess.CalledProcessError: + submodule_helper = subprocess.check_output("git submodule--helper foreach --quiet 'echo $path'", cwd=s_dir, env=env, shell=True).decode("utf-8") + for line in submodule_helper.splitlines(): + module_dir = os.path.join(s_dir, line.split()[-1]) + if os.path.isdir(module_dir): + proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + proc.communicate() + proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + stdout, _ = proc.communicate() + git_sha1 += stdout.decode("utf-8") sha1 = hashlib.sha1(git_sha1.encode("utf-8")).hexdigest() with open(oe_hash_file, 'w') as fobj: fobj.write(sha1) From 00b9fcc2e57bf6636478b0e93f99c8e16b65858d Mon Sep 17 00:00:00 2001 From: Peter Hurley Date: Sun, 9 Oct 2022 20:20:07 -0700 Subject: [PATCH 2/2] externalsrc: fix submodule path formation Submodule paths output by git submodule--helper subcommands are relative to the containing superproject top-level directory, whereas scrdir/EXTERNALSRC may be a subdirectory _within_ a superproject and not necessarily equivalent to the top-level directory. Form submodule paths from top-level directory [1] and only process submodules where srcdir/EXTERNALSRC is a parent path of the submodule path [2]. [1] which may not be equivalent to git_dir in a git worktree [2] uses os.path.commonpath() available since python 3.5 Change-Id: I94d40074586469242228a63ef66089f9f97d1159 Signed-off-by: Peter Hurley --- meta/classes/externalsrc.bbclass | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/meta/classes/externalsrc.bbclass b/meta/classes/externalsrc.bbclass index 474311edae5..1aa5e2bbd44 100644 --- a/meta/classes/externalsrc.bbclass +++ b/meta/classes/externalsrc.bbclass @@ -234,9 +234,10 @@ def srctree_hash_files(d, srcdir=None): submodule_helper = subprocess.check_output(['git', 'submodule--helper', 'list'], cwd=s_dir, env=env).decode("utf-8") except subprocess.CalledProcessError: submodule_helper = subprocess.check_output("git submodule--helper foreach --quiet 'echo $path'", cwd=s_dir, env=env, shell=True).decode("utf-8") + toplevel = subprocess.check_output(['git', '-C', s_dir, 'rev-parse', '--show-toplevel'], cwd=s_dir, env=env).decode("utf-8") for line in submodule_helper.splitlines(): - module_dir = os.path.join(s_dir, line.split()[-1]) - if os.path.isdir(module_dir): + module_dir = os.path.join(toplevel.strip(), line.split()[-1]) + if os.path.isdir(module_dir) and os.path.commonpath([s_dir]) == os.path.commonpath([s_dir, module_dir]): proc = subprocess.Popen(['git', 'add', '-A', '.'], cwd=module_dir, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) proc.communicate() proc = subprocess.Popen(['git', 'write-tree'], cwd=module_dir, env=env, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)