Skip to content

Commit

Permalink
Merge pull request #20 from ESMCI/cleanup030124
Browse files Browse the repository at this point in the history 
Cleanup030124
  • Loading branch information
@jedwards4b
jedwards4b authored Mar 14, 2024
2 parents ff3fcb9 + c31e17f commit 78b3154
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 58 deletions.
2 changes: 2 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ Git-fleximod is a Python-based tool that extends Git's submodule and sparse chec
- AlwaysOptional: Always optional (checked out with --optional flag).
fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
fxurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
**NOTE** the fxurl variable is only used to identify the official project repository and should not be
changed by users. Use the url variable to change to a fork if desired.

## Sparse Checkouts

Expand Down
98 changes: 85 additions & 13 deletions git_fleximod/git_fleximod.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,24 @@ def commandline_arguments(args=None):
def submodule_sparse_checkout(
root_dir, name, url, path, sparsefile, tag="master"
):
"""
This function performs a sparse checkout of a git submodule. It does so by first creating the .git/info/sparse-checkout fileq
in the submodule and then checking out the desired tag. If the submodule is already checked out, it will not be checked out again.
Creating the sparse-checkout file first prevents the entire submodule from being checked out and then removed. This is important
because the submodule may have a large number of files and checking out the entire submodule and then removing it would be time
and disk space consuming.
Parameters:
root_dir (str): The root directory for the git operation.
name (str): The name of the submodule.
url (str): The URL of the submodule.
path (str): The path to the submodule.
sparsefile (str): The sparse file for the submodule.
tag (str, optional): The tag to checkout. Defaults to "master".
Returns:
None
"""
logger.info("Called sparse_checkout for {}".format(name))
rgit = GitInterface(root_dir, logger)
superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
Expand Down Expand Up @@ -140,8 +158,24 @@ def submodule_sparse_checkout(
rgit.config_set_value(f'submodule "{name}"',"url",url)

def single_submodule_checkout(
root, name, path, url=None, tag=None, force=False, optional=False
root, name, path, url=None, tag=None, force=False, optional=False
):
"""
This function checks out a single git submodule.
Parameters:
root (str): The root directory for the git operation.
name (str): The name of the submodule.
path (str): The path to the submodule.
url (str, optional): The URL of the submodule. Defaults to None.
tag (str, optional): The tag to checkout. Defaults to None.
force (bool, optional): If set to True, forces the checkout operation. Defaults to False.
optional (bool, optional): If set to True, the submodule is considered optional. Defaults to False.
Returns:
None
"""
# function implementation...
git = GitInterface(root, logger)
repodir = os.path.join(root, path)
logger.info("Checkout {} into {}/{}".format(name,root,path))
Expand Down Expand Up @@ -184,14 +218,14 @@ def single_submodule_checkout(

if os.path.exists(os.path.join(repodir, ".gitmodules")):
# recursively handle this checkout
print(f"Recursively checking out submodules of {name} {repodir} {url}")
print(f"Recursively checking out submodules of {name}")
gitmodules = GitModules(logger, confpath=repodir)
requiredlist = ["AlwaysRequired"]
if optional:
requiredlist.append("AlwaysOptional")
submodules_checkout(gitmodules, repodir, requiredlist, force=force)
if os.path.exists(os.path.join(repodir, ".git")):
print(f"Successfully checked out {name} {repodir}")
print(f"Successfully checked out {name:>20}")
else:
utils.fatal_error(f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}")

Expand All @@ -201,12 +235,15 @@ def single_submodule_checkout(
return


def submodules_status(gitmodules, root_dir):
def submodules_status(gitmodules, root_dir, toplevel=False):
testfails = 0
localmods = 0
needsupdate = 0
for name in gitmodules.sections():
path = gitmodules.get(name, "path")
tag = gitmodules.get(name, "fxtag")
required = gitmodules.get(name, "fxrequired")
level = required and "Toplevel" in required
if not path:
utils.fatal_error("No path found in .gitmodules for {}".format(name))
newpath = os.path.join(root_dir, path)
Expand All @@ -217,6 +254,9 @@ def submodules_status(gitmodules, root_dir):
url = gitmodules.get(name, "url")
tags = rootgit.git_operation("ls-remote", "--tags", url)
atag = None
needsupdate += 1
if not toplevel and level:
continue
for htag in tags.split("\n"):
if tag and tag in htag:
atag = (htag.split()[1])[10:]
Expand Down Expand Up @@ -248,6 +288,7 @@ def submodules_status(gitmodules, root_dir):
elif tag:
print(f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}")
testfails += 1
needsupdate += 1
else:
print(
f"e {name:>20} has no fxtag defined in .gitmodules, module at {atag}"
Expand All @@ -259,23 +300,26 @@ def submodules_status(gitmodules, root_dir):
localmods = localmods + 1
print("M" + textwrap.indent(status, " "))

return testfails, localmods
return testfails, localmods, needsupdate


def submodules_update(gitmodules, root_dir, requiredlist, force):
_, localmods = submodules_status(gitmodules, root_dir)
print("")
_, localmods, needsupdate = submodules_status(gitmodules, root_dir)

if localmods and not force:
print(
"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
)
return
if needsupdate == 0:
return

for name in gitmodules.sections():
fxtag = gitmodules.get(name, "fxtag")
path = gitmodules.get(name, "path")
url = gitmodules.get(name, "url")
logger.info("name={} path={} url={} fxtag={} requiredlist={}".format(name,os.path.join(root_dir, path), url, fxtag, requiredlist))
# if not os.path.exists(os.path.join(root_dir,path, ".git")):
# if not os.path.exists(os.path.join(root_dir,path, ".git")):
fxrequired = gitmodules.get(name, "fxrequired")
assert(fxrequired in fxrequired_allowed_values())
rgit = GitInterface(root_dir, logger)
Expand Down Expand Up @@ -343,13 +387,28 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):

# checkout is done by update if required so this function may be depricated
def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
_, localmods = submodules_status(gitmodules, root_dir)
"""
This function checks out all git submodules based on the provided parameters.
Parameters:
gitmodules (ConfigParser): The gitmodules configuration.
root_dir (str): The root directory for the git operation.
requiredlist (list): The list of required modules.
force (bool, optional): If set to True, forces the checkout operation. Defaults to False.
Returns:
None
"""
# function implementation...
print("")
_, localmods, needsupdate = submodules_status(gitmodules, root_dir)
if localmods and not force:
print(
"Repository has local mods, cowardly refusing to continue, fix issues or use --force to override"
)
return
if not needsupdate:
return
for name in gitmodules.sections():
fxrequired = gitmodules.get(name, "fxrequired")
fxsparse = gitmodules.get(name, "fxsparse")
Expand Down Expand Up @@ -380,10 +439,23 @@ def submodules_checkout(gitmodules, root_dir, requiredlist, force=False):
optional = "AlwaysOptional" in requiredlist
)


def submodules_test(gitmodules, root_dir):
"""
This function tests the git submodules based on the provided parameters.
It first checks that fxtags are present and in sync with submodule hashes.
Then it ensures that urls are consistent with fxurls (not forks and not ssh)
and that sparse checkout files exist.
Parameters:
gitmodules (ConfigParser): The gitmodules configuration.
root_dir (str): The root directory for the git operation.
Returns:
int: The number of test failures.
"""
# First check that fxtags are present and in sync with submodule hashes
testfails, localmods = submodules_status(gitmodules, root_dir)
testfails, localmods, needsupdate = submodules_status(gitmodules, root_dir)
print("")
# Then make sure that urls are consistant with fxurls (not forks and not ssh)
# and that sparse checkout files exist
Expand All @@ -398,7 +470,7 @@ def submodules_test(gitmodules, root_dir):
if fxsparse and not os.path.isfile(os.path.join(root_dir, path, fxsparse)):
print(f"{name:>20} sparse checkout file {fxsparse} not found")
testfails += 1
return testfails + localmods
return testfails + localmods + needsupdate


def main():
Expand Down Expand Up @@ -440,7 +512,7 @@ def main():
if action == "update":
submodules_update(gitmodules, root_dir, fxrequired, force)
elif action == "status":
submodules_status(gitmodules, root_dir)
submodules_status(gitmodules, root_dir, toplevel=True)
elif action == "test":
retval = submodules_test(gitmodules, root_dir)
else:
Expand Down
84 changes: 39 additions & 45 deletions git_fleximod/metoflexi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,53 +150,47 @@ def translate_single_repo(self, section, tag, url, path, efile, hash_, sparse, p
if sparse:
self.gitmodules.set(section, "fxsparse", sparse)
self.gitmodules.set(section, "fxrequired", "ToplevelRequired")

return

newpath = (self.rootpath / Path(path))
if newpath.exists():
shutil.rmtree(newpath)
logger.info("Creating directory {}".format(newpath))
newpath.mkdir(parents=True)
if tag:
logger.info("cloning {}".format(section))
try:
self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path)
except:
else:
newpath = (self.rootpath / Path(path))
if newpath.exists():
shutil.rmtree(newpath)
logger.info("Creating directory {}".format(newpath))
newpath.mkdir(parents=True)
if tag:
logger.info("cloning {}".format(section))
try:
self.git.git_operation("clone", "-b", tag, "--depth", "1", url, path)
except:
self.git.git_operation("clone", url, path)
with utils.pushd(newpath):
ngit = GitInterface(newpath, logger)
ngit.git_operation("checkout", tag)
if hash_:
self.git.git_operation("clone", url, path)
with utils.pushd(newpath):
ngit = GitInterface(newpath, logger)
ngit.git_operation("checkout", tag)

# if (newpath / ".gitignore").exists():
# logger.info("Moving .gitignore file in {}".format(newpath))
# (newpath / ".gitignore").rename((newpath / "save.gitignore"))

if hash_:
self.git.git_operation("clone", url, path)
git = GitInterface(newpath, logger)
git.git_operation("fetch", "origin")
git.git_operation("checkout", hash_)
if sparse:
print("setting as sparse submodule {}".format(section))
sparsefile = (newpath / Path(sparse))
newfile = (newpath / ".git" / "info" / "sparse-checkout")
print(f"sparsefile {sparsefile} newfile {newfile}")
shutil.copy(sparsefile, newfile)
logger.info("adding submodule {}".format(section))
self.gitmodules.save()
self.git.git_operation("submodule", "add", "-f", "--name", section, url, path)
self.git.git_operation("submodule","absorbgitdirs")
self.gitmodules.reload()
if tag:
self.gitmodules.set(section, "fxtag", tag)
if hash_:
self.gitmodules.set(section, "fxtag", hash_)
git = GitInterface(newpath, logger)
git.git_operation("fetch", "origin")
git.git_operation("checkout", hash_)
if sparse:
print("setting as sparse submodule {}".format(section))
sparsefile = (newpath / Path(sparse))
newfile = (newpath / ".git" / "info" / "sparse-checkout")
print(f"sparsefile {sparsefile} newfile {newfile}")
shutil.copy(sparsefile, newfile)

logger.info("adding submodule {}".format(section))
self.gitmodules.save()
self.git.git_operation("submodule", "add", "-f", "--name", section, url, path)
self.git.git_operation("submodule","absorbgitdirs")
self.gitmodules.reload()
if tag:
self.gitmodules.set(section, "fxtag", tag)
if hash_:
self.gitmodules.set(section, "fxtag", hash_)

self.gitmodules.set(section, "fxurl", url)
if sparse:
self.gitmodules.set(section, "fxsparse", sparse)
self.gitmodules.set(section, "fxrequired", "ToplevelRequired")
self.gitmodules.set(section, "fxurl", url)
if sparse:
self.gitmodules.set(section, "fxsparse", sparse)
self.gitmodules.set(section, "fxrequired", "ToplevelRequired")


def translate_repo(self):
Expand Down

0 comments on commit 78b3154

Please sign in to comment.