Merge pull request #18 from caracal-pipeline/qc

QC -- do not merge yet

cultcargo/builder/build_cargo.py

@@ -43,7 +43,7 @@ class Metadata(object):
         GITHUB_REPOSITORY: str = ""
 
     metadata: Metadata
-    assign: Dict[str, Any] 
+    assign: Dict[str, Any]
     images: Dict[str, ImageInfo]
 
 
@@ -61,25 +61,26 @@ def run(command, cwd=None, input=None):
 
 
 @click.command()
-@click.option('-m', '--manifest', type=click.Path(exists=True), 
-                default=DEFAULT_MANIFEST, 
+@click.option('-m', '--manifest', type=click.Path(exists=True),
+                default=DEFAULT_MANIFEST,
                 help=f'Cargo manifest. Default is {DEFAULT_MANIFEST}.')
 @click.option('-l', '--list', 'do_list', is_flag=True, help='List only, do not push or build. Returns error if images are missing.')
 @click.option('-b', '--build', is_flag=True, help='Build only, do not push.')
 @click.option('-p', '--push', is_flag=True, help='Push only, do not build.')
 @click.option('-r', '--rebuild', is_flag=True, help='Ignore docker image caches (i.e. rebuild).')
 @click.option('-a', '--all', is_flag=True, help='Build and/or push all images in manifest.')
 @click.option('-v', '--verbose', is_flag=True, help='Be verbose.')
+@click.option('-b', '--boring', is_flag=True, help='Be boring -- no progress bar.')
 @click.argument('imagenames', type=str, nargs=-1)
-def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False, rebuild=False, verbose=False, imagenames: List[str] = []):
+def build_cargo(manifest: str, do_list=False, build=False, push=False, all=False, rebuild=False, boring=False, verbose=False, imagenames: List[str] = []):
     if not (build or push or do_list):
         build = push = True
 
     with Progress(
             TimeElapsedColumn(),
             SpinnerColumn(),
             "{task.description}",
-            console=console) as progress:
+            console=console, disable=boring) as progress:
         print = progress.console.print
 
         progress_task = progress.add_task("loading manifest")
@@ -144,16 +145,16 @@ def resolve_config_reference(value):
                     raise KeyError(f"{comps[2]} not found in {comps[1]}")
                 return container
             return value
-                
+
         conf.metadata.REGISTRY = resolve_config_reference(conf.metadata.REGISTRY)
         conf.metadata.BUNDLE_VERSION = resolve_config_reference(conf.metadata.BUNDLE_VERSION)
         print(f"Registry is {conf.metadata.REGISTRY}, bundle is '{conf.metadata.BUNDLE_VERSION}', prefix '{conf.metadata.BUNDLE_VERSION_PREFIX}'")
         if not conf.metadata.BUNDLE_VERSION.startswith(conf.metadata.BUNDLE_VERSION_PREFIX):
             print("Inconsistent manifest metadata: BUNDLE_VERSION must start with BUNDLE_VERSION_PREFIX")
             return 1
-        
+
         unprefixed_image_version = conf.metadata.BUNDLE_VERSION[len(conf.metadata.BUNDLE_VERSION_PREFIX):]
-        
+
         if '::' in conf.metadata.BASE_IMAGE_PATH:
             modname, path = conf.metadata.BASE_IMAGE_PATH.split('::', 1)
             pkg_path = os.path.dirname(importlib.import_module(modname).__file__)
@@ -203,7 +204,7 @@ def resolve_config_reference(value):
         if not imagenames:
             print(f"Nothing to be done. Please specify some image names, or run with -a/-all.")
             return 0
-        
+
         for image in imagenames:
             version = None
             if ':' in image:
@@ -214,7 +215,7 @@ def resolve_config_reference(value):
             if version is not None and version not in conf.images[image].versions:
                 print(f"Unknown image '{image}:{version}'")
                 return 1
-            
+
         remote_images_exist = {}
 
         for i_image,image in enumerate(imagenames):
@@ -238,19 +239,19 @@ def resolve_config_reference(value):
                 progress.update(progress_task, description=
                     f"image [bold]{image}[/bold] [{i_image}/{len(imagenames)}]: "
                     f"version [bold]{version}[/bold] [{i_version}/{len(versions)}]")
-                
+
                 if version == "latest":
                     image_version = BUNDLE_VERSION
-                else: 
+                else:
                     version = version.format(**image_vars)
                     image_version = f"{version}-{BUNDLE_VERSION}"
-                
+
                 version_info = image_info.versions[version]
                 version_vars = image_vars.copy()
                 version_vars.update(**version_info)
                 version_vars["VERSION"] = version
                 version_vars["IMAGE_VERSION"] = image_version
-                
+
                 dockerfile = version_info.get('dockerfile') or image_info.dockerfile or 'Dockerfile'
                 dockerfile = dockerfile.format(**version_vars)
                 full_image = f"{registry}/{image}:{image_version}"
@@ -274,7 +275,7 @@ def resolve_config_reference(value):
                         subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True)
                         print(f"  Manifest returned for {full_image}")
                     except subprocess.CalledProcessError as e:
-                        output = e.stderr.strip() 
+                        output = e.stderr.strip()
                         if "no such manifest" in output or "was deleted" in output:
                             print(f"  {output}")
                             print(f"  [green]No manifest returned for {full_image}[/green]")
@@ -296,7 +297,7 @@ def resolve_config_reference(value):
                         print(f"{content}", style="dim", highlight=True)
                     run(f"docker build {no_cache} -t {full_image} -", cwd=build_dir, input=content)
                     # is this the latest version that needs to be tagged
-                    if image_version == tag_latest[image]:
+                    if image_version == tag_latest.get(image):
                         run(f"docker tag {registry}/{image}:{image_version} {registry}/{image}:{BUNDLE_VERSION}")
 
                 # go push
@@ -318,9 +319,8 @@ def resolve_config_reference(value):
                         else:
                             print(f"  Image exists, but package unreleased, ok push.")
                     run(f"docker push {full_image}", cwd=path)
-                    if image_version == tag_latest[image]:
+                    if image_version == tag_latest.get(image):
                         run(f"docker push {registry}/{image}:{BUNDLE_VERSION}")
-
             progress.update(progress_task, description=
                 f"image [bold]{image}[/bold] [{i_image}/{len(imagenames)}]: tagging latest version")
 

cultcargo/builder/cargo-manifest.yml

@@ -1,11 +1,11 @@
 metadata:
-  # name of python package 
+  # name of python package
   PACKAGE: cult-cargo
   # its repository
   GITHUB_REPOSITORY: caracal-pipeline/cult-cargo
-  # registry to use. Use module::filename.yml::variable format to pull from a config file inside a python module 
+  # registry to use. Use module::filename.yml::variable format to pull from a config file inside a python module
   REGISTRY: cultcargo.genesis::cult-cargo-base.yml::vars.cult-cargo.images.registry
-  # image bundle ersion. Use module::filename.yml::variable format to pull from a config file inside a python module 
+  # image bundle ersion. Use module::filename.yml::variable format to pull from a config file inside a python module
   BUNDLE_VERSION: cultcargo.genesis::cult-cargo-base.yml::vars.cult-cargo.images.version
   # prefix to be removed from image version when comparing to python package version
   BUNDLE_VERSION_PREFIX: cc
@@ -17,7 +17,7 @@ assign:
   pre_install: ''
   post_install: ''
   extra_deps: ''
-  
+
   # base image for generic Python-based packages
   base_python_image: python-astro:3.9
   # corresponding python binary
@@ -30,7 +30,7 @@ images:
 
   wsclean:
     versions:
-      kern7: 
+      kern7:
         dockerfile: Dockerfile.kern7
       ## 2.9 won't build with this recipe, figure it out later
       # '2.9':
@@ -66,20 +66,20 @@ images:
       '4.7.2':
         tarball: casa-release-4.7.2-el7.tar.gz
         entrypoint: /casa-release-4.7.2-el7/bin/casa
-      '5.8.0': 
+      '5.8.0':
         tarball: casa-release-5.8.0-109.el7.tar.gz
         entrypoint: /casa-release-5.8.0-109.el7/bin/casa
 
   casa6:
     versions:
-      '6.5': 
+      '6.5':
           wheel_version: 6.5.5.20
 
   breizorro:
     versions:
       '0.1.1':
         package: breizorro==0.1.1
-      latest: 
+      latest:
         package: breizorro
       master:
         package: git+https://github.com/ratt-ru/breizorro
@@ -88,16 +88,16 @@ images:
     versions:
       '0.5.3':
         package: shadems==0.5.3
-      latest: 
+      latest:
         package: shadems
       master:
         package: git+https://github.com/ratt-ru/shadeMS
-      
+
   crystalball:
     versions:
       # '0.3.0':
       #   package: crystalball==0.3.0
-#      latest: 
+#      latest:
 #         package: crystalball  # doesn't work?
       master:
         package: git+https://github.com/caracal-pipeline/crystalball
@@ -109,7 +109,7 @@ images:
     versions:
       '0.1.8.1':
         package: tricolour==0.1.8
-      latest: 
+      latest:
         package: tricolour
       master:
         package: git+https://github.com/ratt-ru/tricolour
@@ -118,7 +118,7 @@ images:
     versions:
       '0.1.7':
         package: smops==0.1.7
-      latest: 
+      latest:
         package: smops
       master:
         package: git+https://github.com/Mulan-94/smops
@@ -127,16 +127,16 @@ images:
     versions:
       '1.2.0':
         package: msutils==1.2.0
-      latest: 
+      latest:
         package: msutils
       master:
         package: git+https://github.com/SpheMakh/msutils
 
   quartical:
     versions:
-      '0.2.2': 
+      '0.2.2':
         package: quartical==0.2.2
-      latest: 
+      latest:
         package: quartical
       master:
         package: git+https://github.com/ratt-ru/QuartiCal
@@ -150,6 +150,11 @@ images:
       master:
         package: git+https://github.com/ratt-ru/CubiCal
 
+  pfb-clean:
+    versions:
+      latest:
+        package: git+https://github.com/ratt-ru/pfb-clean@awskube
+
   mosaic-queen:
     assign:
       pre_install: RUN apt update && apt install -y montage && rm -rf /var/lib/apt/lists/* && rm -rf /var/cache/apt/archives

cultcargo/genesis/pfb-clean/latest/cgopts.yml

@@ -0,0 +1,30 @@
+cg-tol:
+  dtype: float
+  default: 1e-3
+  info: Tolreance of conjugate gradient algorithm
+  abbreviation: cgtol
+cg-maxit:
+  dtype: int
+  default: 150
+  info: Maximum iterations for conjugate gradient algorithm
+  abbreviation: cgmaxit
+cg-minit:
+  dtype: int
+  default: 10
+  info: Minimum iterations for conjugate gradient algorithm
+  abbreviation: cgminit
+cg-verbose:
+  dtype: int
+  default: 1
+  info: Verbosity of conjugate gradient algorithm. Set to > 1 for \ debugging, 0 for
+    silence
+  abbreviation: cgverb
+cg-report-freq:
+  dtype: int
+  default: 10
+  info: Report frequency of conjugate gradient algorithm
+  abbreviation: cgrf
+backtrack:
+  dtype: bool
+  default: true
+  info: Ensure residual decreases at every iteration