From 5a4200efb5f7655fce242d479fdd348f0d9a8ef0 Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 15:51:11 +0100 Subject: [PATCH 1/6] repo.py: use os.path.exists() to check for existing repo For reasons currently not known os.path.isdir() returns False even for an existing .git directory in some cases. Let's switch to os.path.exists(). --- scripts/lib/repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index 522cbbf..9d98c45 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -110,7 +110,7 @@ class Module(): ## @brief check if path contains already a git repo ## @return True/False def path_contains_repo(self): - return os.path.isdir("{}/.git".format(self.__path)) + return os.path.exists("{}/.git".format(self.__path)) ## @brief check that the given ref exists def rev_exists(self, rev): -- GitLab From 2d2f317789c2f221d2bf50a3cf7fdaf165413706 Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 15:55:13 +0100 Subject: [PATCH 2/6] repo.py: Improve getGitServer() and return the full core repo URI For creating a Module instance for the core repo, we need the core repo's full URI. Also using a dictionary with named keys is easier to read than using a simple array as return type of gitGetServer(). --- scripts/lib/repo.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index 9d98c45..3453ff9 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -252,17 +252,21 @@ class Module(): def getGitServer(): p = git_cmd(['config', 'remote.origin.url']) - uri = p.stdout.strip() - if re.match('^[a-z]*://', uri): # http://server/path or (ssh, git ...) - clone_uri = '/'.join(uri.split('/')[:3]) + core_uri = p.stdout.strip() + if re.match('^[a-z]*://', core_uri): # http://server/path or (ssh, git ...) + clone_uri = '/'.join(core_uri.split('/')[:3]) git_uri = 'git@' + clone_uri.split('://')[1] else: # git@server:path scheme - clone_uri = uri.split(':', 1)[0] + clone_uri = core_uri.split(':', 1)[0] git_uri = clone_uri - return [clone_uri, git_uri] + return { + 'clone_uri': clone_uri, + 'git_uri': git_uri, + 'core_uri': core_uri + } def getCoreModule(): - return Module(getGitServer()[0], os.getcwd()) + return Module(getGitServer()['core_uri'], os.getcwd()) def loadBuildModule(f, b, remote_repo=None, offline=False): if not os.path.exists(f): @@ -270,8 +274,8 @@ def loadBuildModule(f, b, remote_repo=None, offline=False): config = configparser.ConfigParser() config.read(f) - os.environ['GIT_SERVER'] = getGitServer()[0] - os.environ['GIT_SERVER_SSH'] = getGitServer()[1] + os.environ['GIT_SERVER'] = getGitServer()['clone_uri'] + os.environ['GIT_SERVER_SSH'] = getGitServer()['git_uri'] local_b = os.path.basename(b) if not b: @@ -334,8 +338,8 @@ def loadModules(f, b=None): config = configparser.ConfigParser() config.read(f) - os.environ['GIT_SERVER'] = getGitServer()[0] - os.environ['GIT_SERVER_SSH'] = getGitServer()[1] + os.environ['GIT_SERVER'] = getGitServer()['clone_uri'] + os.environ['GIT_SERVER_SSH'] = getGitServer()['git_uri'] modules = {} for name, uri in config.items("modules"): -- GitLab From 15e6321010266704e9ce003d56c960a0a596e83a Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 16:09:44 +0100 Subject: [PATCH 3/6] repo.py: Fix meta layer URIs for SSH access on KED server In case our core repos remote address is using SSH access, we might need to replace the slash in the URIs from repo.conf with a colon for the correct URI. --- scripts/lib/repo.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index 3453ff9..af6b5ba 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -347,6 +347,9 @@ def loadModules(f, b=None): if uri[0].startswith("{GIT_SERVER}") and \ os.environ['GIT_SERVER'].startswith("http"): uri[0] = uri[0].replace("}:", "}/") + elif uri[0].startswith("{GIT_SERVER}") and \ + os.environ['GIT_SERVER'].startswith("git@"): + uri[0] = uri[0].replace("}/", "}:") if ("{GIT_SERVER}" in uri[0] or \ "{GIT_SERVER_SSH}" in uri[0]) and \ not uri[0].endswith(".git"): -- GitLab From 8c9113989b21edf8b8b571b9bf78df043f844168 Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 16:13:06 +0100 Subject: [PATCH 4/6] repo.py: Add non-interactive mode for CI builds In case the init-env and meta-update script is called via automatic build pipelines, we might want to handle some things differently. To detect this we read the value of the environment variable INITENV_INTERACTIVE. If it is '0', 'disabled', 'false' or 'no', the non-interactive mode is enabled, otherwise the default interactive mode is used. This commit only provides the function to read the env variable and does not alter the behavior. --- scripts/lib/repo.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index af6b5ba..b93c922 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -265,6 +265,13 @@ def getGitServer(): 'core_uri': core_uri } +def getInteractive(): + interactive_str = os.environ.get('INITENV_INTERACTIVE') + if (interactive_str and interactive_str.casefold() in + map(str.casefold, ['false', 'no', 'disabled', '0'])): + return False + return True + def getCoreModule(): return Module(getGitServer()['core_uri'], os.getcwd()) -- GitLab From 3c65174a883535f82c175dd2c4908604a1b96d6a Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 16:22:42 +0100 Subject: [PATCH 5/6] repo.py: Reset branch if fast forward fails in non-interactive mode If the local and remote branch have diverged and it is not possible to do a fast forward, we might want to fall back to reset the local branch to match the remote. As this rewrites your local history, we only do this in non-interactive mode where we typically only have a temporary clone and not any ongoing work (e.g. in CI builds). --- scripts/lib/repo.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index b93c922..db46ad9 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -189,7 +189,7 @@ class Module(): git_cmd(["checkout", branch], display=True, cwd=self.__path) # if checked out branch head does not match rev, or pull was requested, try to merge remote branch if self.rev_parse(rev) != self.rev_parse('HEAD') or pull: - git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True, cwd=self.__path) + self.forward(branch) # if checked out branch still does not match (e.g. because local branch has unpushed commits) # then check out specific rev if self.rev_parse(rev) != self.rev_parse('HEAD'): @@ -206,11 +206,24 @@ class Module(): git_cmd(["checkout", "-b", branch, "%s/%s" % (remote, branch)], display=True, cwd=self.__path) else: - git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True, cwd=self.__path) + self.forward(branch) except Exception as e: self.__err('Failed to checkout %s branch' % branch, e) return self.rev_parse('HEAD') + def forward(self, branch): + remote = self.params.get('remote', 'origin') + try: + git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True, + cwd=self.__path) + except Exception as e: + if getInteractive(): + self.__err('Failed to fast forward local branch: %s' % branch, e) + else: + vinfo("Resetting local branch {} to match remote {}".format(branch, remote)) + git_cmd(["reset", "--hard", "%s/%s" % (remote, branch)], display=True, + cwd=self.__path) + def setRev(self, rev): self.params['rev'] = rev branch = self.rev_is_branch_head(rev) -- GitLab From f9f831cb4cf5fc7e9e9b5af57581ffb375bcdd7d Mon Sep 17 00:00:00 2001 From: Frieder Schrempf <frieder.schrempf@kontron.de> Date: Tue, 15 Mar 2022 16:30:11 +0100 Subject: [PATCH 6/6] repo.py: Try shallow cloning of meta layers in non-interactive mode We can save some time and bandwidth in automatic builds if we do shallow clones of the meta layers when we don't need the full history. As some servers don't support shallow cloning via HTTP, let's fall back to normal clones in case of errors. Also in case we need to update an already existing shallow clone, the clone will be unshallowed and remaining objects will be fetched from the remote. --- scripts/lib/repo.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py index db46ad9..aad473b 100644 --- a/scripts/lib/repo.py +++ b/scripts/lib/repo.py @@ -21,7 +21,7 @@ def remove_prefix(prefix_list, text): ## @brief repo object manipulator ## class Module(): - def __init__(self, uri, path, params=None): + def __init__(self, uri, path, params=None, shallow=False): self.params = {} if params is None else params self.__path = path if not uri: @@ -32,6 +32,7 @@ class Module(): except Exception as e: return None self.__uri = uri + self.shallow = shallow def is_valid_uri(self): try: @@ -72,9 +73,15 @@ class Module(): if not self.path_contains_repo(): try: print("Cloning ...") - git_cmd(["clone", self.uri(), self.__path], display=True, - cwd=os.getcwd()) - + if self.shallow and self.branch(): + try: + git_cmd(["clone", "--depth", "1", "--branch", self.branch(), self.uri(), + self.__path], display=True, cwd=os.getcwd()) + return + except: + self.shallow = False + pass + git_cmd(["clone", self.uri(), self.__path], display=True, cwd=os.getcwd()) except Exception as e: self.__err('Failed to clone repo: %s' % self.uri(), e) else: @@ -180,6 +187,11 @@ class Module(): # Checking out a specific rev if not self.rev_exists(rev): self.update() + if not self.rev_exists(rev) and self.shallow and self.branch(): + remote = self.params.get('remote', 'origin') + git_cmd(["fetch", "--unshallow", remote, self.branch()], display=True, + cwd=self.__path) + self.shallow = False if not self.rev_exists(rev): self.__err('Could not find git rev: %s in repo %s' % (rev, self.__path)) @@ -382,7 +394,7 @@ def loadModules(f, b=None): else: params[p] = None uri = uri[0] - modules[name] = Module(uri, name, params) + modules[name] = Module(uri, name, params, shallow=not getInteractive()) return modules -- GitLab