From 5a4200efb5f7655fce242d479fdd348f0d9a8ef0 Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 15:51:11 +0100
Subject: [PATCH 1/6] repo.py: use os.path.exists() to check for existing repo

For reasons currently not known os.path.isdir() returns False
even for an existing .git directory in some cases. Let's switch
to os.path.exists().
---
 scripts/lib/repo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index 522cbbf..9d98c45 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -110,7 +110,7 @@ class Module():
     ## @brief check if path contains already a git repo
     ## @return True/False
     def path_contains_repo(self):
-        return os.path.isdir("{}/.git".format(self.__path))
+        return os.path.exists("{}/.git".format(self.__path))
 
     ## @brief check that the given ref exists
     def rev_exists(self, rev):
-- 
GitLab


From 2d2f317789c2f221d2bf50a3cf7fdaf165413706 Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 15:55:13 +0100
Subject: [PATCH 2/6] repo.py: Improve getGitServer() and return the full core
 repo URI

For creating a Module instance for the core repo, we need the core repo's
full URI. Also using a dictionary with named keys is easier to read than
using a simple array as return type of gitGetServer().
---
 scripts/lib/repo.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index 9d98c45..3453ff9 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -252,17 +252,21 @@ class Module():
 
 def getGitServer():
     p = git_cmd(['config', 'remote.origin.url'])
-    uri = p.stdout.strip()
-    if re.match('^[a-z]*://', uri): # http://server/path or (ssh, git ...)
-        clone_uri = '/'.join(uri.split('/')[:3])
+    core_uri = p.stdout.strip()
+    if re.match('^[a-z]*://', core_uri): # http://server/path or (ssh, git ...)
+        clone_uri = '/'.join(core_uri.split('/')[:3])
         git_uri = 'git@' + clone_uri.split('://')[1]
     else: # git@server:path scheme
-        clone_uri = uri.split(':', 1)[0]
+        clone_uri = core_uri.split(':', 1)[0]
         git_uri = clone_uri
-    return [clone_uri, git_uri]
+    return {
+        'clone_uri': clone_uri,
+        'git_uri': git_uri,
+        'core_uri': core_uri
+    }
 
 def getCoreModule():
-    return Module(getGitServer()[0], os.getcwd())
+    return Module(getGitServer()['core_uri'], os.getcwd())
 
 def loadBuildModule(f, b, remote_repo=None, offline=False):
     if not os.path.exists(f):
@@ -270,8 +274,8 @@ def loadBuildModule(f, b, remote_repo=None, offline=False):
 
     config = configparser.ConfigParser()
     config.read(f)
-    os.environ['GIT_SERVER'] = getGitServer()[0]
-    os.environ['GIT_SERVER_SSH'] = getGitServer()[1]
+    os.environ['GIT_SERVER'] = getGitServer()['clone_uri']
+    os.environ['GIT_SERVER_SSH'] = getGitServer()['git_uri']
     local_b = os.path.basename(b)
 
     if not b:
@@ -334,8 +338,8 @@ def loadModules(f, b=None):
 
     config = configparser.ConfigParser()
     config.read(f)
-    os.environ['GIT_SERVER'] = getGitServer()[0]
-    os.environ['GIT_SERVER_SSH'] = getGitServer()[1]
+    os.environ['GIT_SERVER'] = getGitServer()['clone_uri']
+    os.environ['GIT_SERVER_SSH'] = getGitServer()['git_uri']
 
     modules = {}
     for name, uri in config.items("modules"):
-- 
GitLab


From 15e6321010266704e9ce003d56c960a0a596e83a Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 16:09:44 +0100
Subject: [PATCH 3/6] repo.py: Fix meta layer URIs for SSH access on KED server

In case our core repos remote address is using SSH access, we might need
to replace the slash in the URIs from repo.conf with a colon for the correct
URI.
---
 scripts/lib/repo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index 3453ff9..af6b5ba 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -347,6 +347,9 @@ def loadModules(f, b=None):
         if uri[0].startswith("{GIT_SERVER}") and \
           os.environ['GIT_SERVER'].startswith("http"):
             uri[0] = uri[0].replace("}:", "}/")
+        elif uri[0].startswith("{GIT_SERVER}") and \
+          os.environ['GIT_SERVER'].startswith("git@"):
+            uri[0] = uri[0].replace("}/", "}:")
         if ("{GIT_SERVER}" in uri[0] or \
             "{GIT_SERVER_SSH}" in uri[0]) and \
             not uri[0].endswith(".git"):
-- 
GitLab


From 8c9113989b21edf8b8b571b9bf78df043f844168 Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 16:13:06 +0100
Subject: [PATCH 4/6] repo.py: Add non-interactive mode for CI builds

In case the init-env and meta-update script is called via automatic
build pipelines, we might want to handle some things differently.
To detect this we read the value of the environment variable
INITENV_INTERACTIVE. If it is '0', 'disabled', 'false' or 'no',
the non-interactive mode is enabled, otherwise the default interactive
mode is used.

This commit only provides the function to read the env variable and
does not alter the behavior.
---
 scripts/lib/repo.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index af6b5ba..b93c922 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -265,6 +265,13 @@ def getGitServer():
         'core_uri': core_uri
     }
 
+def getInteractive():
+    interactive_str = os.environ.get('INITENV_INTERACTIVE')
+    if (interactive_str and interactive_str.casefold() in
+            map(str.casefold, ['false', 'no', 'disabled', '0'])):
+        return False
+    return True
+
 def getCoreModule():
     return Module(getGitServer()['core_uri'], os.getcwd())
 
-- 
GitLab


From 3c65174a883535f82c175dd2c4908604a1b96d6a Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 16:22:42 +0100
Subject: [PATCH 5/6] repo.py: Reset branch if fast forward fails in
 non-interactive mode

If the local and remote branch have diverged and it is not possible to do
a fast forward, we might want to fall back to reset the local branch to
match the remote. As this rewrites your local history, we only do this
in non-interactive mode where we typically only have a temporary clone
and not any ongoing work (e.g. in CI builds).
---
 scripts/lib/repo.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index b93c922..db46ad9 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -189,7 +189,7 @@ class Module():
                 git_cmd(["checkout", branch], display=True, cwd=self.__path)
                 # if checked out branch head does not match rev, or pull was requested, try to merge remote branch
                 if self.rev_parse(rev) != self.rev_parse('HEAD') or pull:
-                    git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True, cwd=self.__path)
+                    self.forward(branch)
                 # if checked out branch still does not match (e.g. because local branch has unpushed commits)
                 # then check out specific rev
                 if self.rev_parse(rev) != self.rev_parse('HEAD'):
@@ -206,11 +206,24 @@ class Module():
                     git_cmd(["checkout", "-b", branch, "%s/%s" %
                         (remote, branch)], display=True, cwd=self.__path)
                 else:
-                    git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True, cwd=self.__path)
+                    self.forward(branch)
             except Exception as e:
                 self.__err('Failed to checkout %s branch' % branch, e)
         return self.rev_parse('HEAD')
 
+    def forward(self, branch):
+        remote = self.params.get('remote', 'origin')
+        try:
+            git_cmd(["merge", "--ff-only", "%s/%s" % (remote, branch)], display=True,
+                    cwd=self.__path)
+        except Exception as e:
+            if getInteractive():
+                self.__err('Failed to fast forward local branch: %s' % branch, e)
+            else:
+                vinfo("Resetting local branch {} to match remote {}".format(branch, remote))
+                git_cmd(["reset", "--hard", "%s/%s" % (remote, branch)], display=True,
+                        cwd=self.__path)
+
     def setRev(self, rev):
         self.params['rev'] = rev
         branch = self.rev_is_branch_head(rev)
-- 
GitLab


From f9f831cb4cf5fc7e9e9b5af57581ffb375bcdd7d Mon Sep 17 00:00:00 2001
From: Frieder Schrempf <frieder.schrempf@kontron.de>
Date: Tue, 15 Mar 2022 16:30:11 +0100
Subject: [PATCH 6/6] repo.py: Try shallow cloning of meta layers in
 non-interactive mode

We can save some time and bandwidth in automatic builds if we do shallow
clones of the meta layers when we don't need the full history.

As some servers don't support shallow cloning via HTTP, let's fall back
to normal clones in case of errors.

Also in case we need to update an already existing shallow clone, the
clone will be unshallowed and remaining objects will be fetched from
the remote.
---
 scripts/lib/repo.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/scripts/lib/repo.py b/scripts/lib/repo.py
index db46ad9..aad473b 100644
--- a/scripts/lib/repo.py
+++ b/scripts/lib/repo.py
@@ -21,7 +21,7 @@ def remove_prefix(prefix_list, text):
 ## @brief repo object manipulator
 ##
 class Module():
-    def __init__(self, uri, path, params=None):
+    def __init__(self, uri, path, params=None, shallow=False):
         self.params = {} if params is None else params
         self.__path = path
         if not uri:
@@ -32,6 +32,7 @@ class Module():
                 except Exception as e:
                     return None
         self.__uri = uri
+        self.shallow = shallow
 
     def is_valid_uri(self):
         try:
@@ -72,9 +73,15 @@ class Module():
         if not self.path_contains_repo():
             try:
                 print("Cloning ...")
-                git_cmd(["clone", self.uri(), self.__path], display=True,
-                    cwd=os.getcwd())
-
+                if self.shallow and self.branch():
+                    try:
+                        git_cmd(["clone", "--depth", "1", "--branch", self.branch(), self.uri(),
+                                 self.__path], display=True, cwd=os.getcwd())
+                        return
+                    except:
+                        self.shallow = False
+                        pass
+                git_cmd(["clone", self.uri(), self.__path], display=True, cwd=os.getcwd())
             except Exception as e:
                 self.__err('Failed to clone repo: %s' % self.uri(), e)
         else:
@@ -180,6 +187,11 @@ class Module():
             # Checking out a specific rev
             if not self.rev_exists(rev):
                 self.update()
+            if not self.rev_exists(rev) and self.shallow and self.branch():
+                remote = self.params.get('remote', 'origin')
+                git_cmd(["fetch", "--unshallow", remote, self.branch()], display=True,
+                         cwd=self.__path)
+                self.shallow = False
             if not self.rev_exists(rev):
                 self.__err('Could not find git rev: %s in repo %s' %
                     (rev, self.__path))
@@ -382,7 +394,7 @@ def loadModules(f, b=None):
             else:
                 params[p] = None
         uri = uri[0]
-        modules[name] = Module(uri, name, params)
+        modules[name] = Module(uri, name, params, shallow=not getInteractive())
     return modules
 
 
-- 
GitLab