add support to customize flux submit command (#207)

* add support to customize flux submit command for older container that still require flux mini submit (or another derivative) this is needed. Signed-off-by: vsoch <[email protected]>
flux-framework · Sep 10, 2023 · 76c4754 · 76c4754
1 parent 3175f68
commit 76c4754
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 4 deletions.
diff --git a/api/v1alpha1/minicluster_types.go b/api/v1alpha1/minicluster_types.go
@@ -354,6 +354,10 @@ type FluxSpec struct {
 	// +default="/usr"
 	InstallRoot string `json:"installRoot,omitempty"`
 
+	// Modify flux submit to be something else
+	// +optional
+	SubmitCommand string `json:"submitCommand,omitempty"`
+
 	// Commands for flux start --wrap
 	// +optional
 	Wrap string `json:"wrap,omitempty"`

diff --git a/api/v1alpha1/swagger.json b/api/v1alpha1/swagger.json
@@ -258,6 +258,10 @@
           "default": {},
           "$ref": "#/definitions/FluxScheduler"
         },
+        "submitCommand": {
+          "description": "Modify flux submit to be something else",
+          "type": "string"
+        },
         "wrap": {
           "description": "Commands for flux start --wrap",
           "type": "string"

diff --git a/api/v1alpha1/zz_generated.openapi.go b/api/v1alpha1/zz_generated.openapi.go
diff --git a/chart/templates/minicluster-crd.yaml b/chart/templates/minicluster-crd.yaml
@@ -381,6 +381,9 @@ spec:
                           also be "easy"
                         type: string
                     type: object
+                  submitCommand:
+                    description: Modify flux submit to be something else
+                    type: string
                   wrap:
                     description: Commands for flux start --wrap
                     type: string

diff --git a/config/crd/bases/flux-framework.org_miniclusters.yaml b/config/crd/bases/flux-framework.org_miniclusters.yaml
@@ -384,6 +384,9 @@ spec:
                           also be "easy"
                         type: string
                     type: object
+                  submitCommand:
+                    description: Modify flux submit to be something else
+                    type: string
                   wrap:
                     description: Commands for flux start --wrap
                     type: string

diff --git a/controllers/flux/templates/wait.sh b/controllers/flux/templates/wait.sh
@@ -294,7 +294,7 @@ else
         echo "#!/bin/bash
 {{ if .Container.BatchRaw }}{{range $index, $line := .Batch}}{{ if $line }}{{$line}}{{ end }}
 {{ end }}
-{{ else }}{{range $index, $line := .Batch}}{{ if $line }}flux submit --flags waitable --error=${FLUX_OUTPUT_DIR}/job-{{$index}}.err --output=${FLUX_OUTPUT_DIR}/job-{{$index}}.out {{$line}}{{ end }}
+{{ else }}{{range $index, $line := .Batch}}{{ if $line }}{{ if .Spec.Flux.SubmitCommand }}{{ .Spec.Flux.SubmitCommand }}{{ else }}flux submit {{ end }} --flags waitable --error=${FLUX_OUTPUT_DIR}/job-{{$index}}.err --output=${FLUX_OUTPUT_DIR}/job-{{$index}}.out {{$line}}{{ end }}
 {{ end }}
 flux queue idle
 flux jobs -a{{ end }}
@@ -335,13 +335,13 @@ flux jobs -a{{ end }}
             {{ if .Container.Launcher }}
             printf "\n🌀 Launcher Mode: flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }}-o --config /etc/flux/config ${brokerOptions} {{.Container.Commands.Prefix}} $@\n"
             {{ else }}
-            printf "\n🌀 Submit Mode: flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }}-o --config /etc/flux/config ${brokerOptions} {{.Container.Commands.Prefix}} flux submit {{ if ge .Spec.Tasks .Spec.Size }} -N {{.Spec.Size}}{{ end }} -n {{.Spec.Tasks}} --quiet {{ if .Spec.Flux.OptionFlags }}{{ .Spec.Flux.OptionFlags}}{{ end }} --watch{{ if .Spec.Logging.Debug }} -vvv{{ end }} $@\n"
+            printf "\n🌀 Submit Mode: flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }}-o --config /etc/flux/config ${brokerOptions} {{.Container.Commands.Prefix}} {{ if .Spec.Flux.SubmitCommand }}{{ .Spec.Flux.SubmitCommand }}{{ else }}flux submit {{ end }} {{ if ge .Spec.Tasks .Spec.Size }} -N {{.Spec.Size}}{{ end }} -n {{.Spec.Tasks}} --quiet {{ if .Spec.Flux.OptionFlags }}{{ .Spec.Flux.OptionFlags}}{{ end }} --watch{{ if .Spec.Logging.Debug }} -vvv{{ end }} $@\n"
             {{ end }}
 {{ end }}
             {{ if .Container.Launcher }}
             {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }}-o --config /etc/flux/config ${brokerOptions} {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxsubmit wall time %E" {{ end }} {{.Container.Commands.Prefix}} $@
             {{ else }}
-            {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }} -o --config /etc/flux/config ${brokerOptions} {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxsubmit wall time %E" {{ end }} {{.Container.Commands.Prefix}} flux submit {{ if ge .Spec.Tasks .Spec.Size }} -N {{.Spec.Size}}{{ end }} -n {{.Spec.Tasks}} --quiet {{ if .Spec.Flux.OptionFlags }}{{ .Spec.Flux.OptionFlags}}{{ end }} --watch{{ if .Spec.Logging.Debug }} -vvv{{ end }} $@
+            {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxstart wall time %E" {{ end }}${asFlux} flux start {{ if .Spec.Flux.Wrap }}--wrap={{ .Spec.Flux.Wrap }} {{ end }} -o --config /etc/flux/config ${brokerOptions} {{ if .Spec.Logging.Timed }}/usr/bin/time -f "FLUXTIME fluxsubmit wall time %E" {{ end }} {{.Container.Commands.Prefix}} {{ if .Spec.Flux.SubmitCommand }}{{ .Spec.Flux.SubmitCommand }}{{ else }}flux submit {{ end }} {{ if ge .Spec.Tasks .Spec.Size }} -N {{.Spec.Size}}{{ end }} -n {{.Spec.Tasks}} --quiet {{ if .Spec.Flux.OptionFlags }}{{ .Spec.Flux.OptionFlags}}{{ end }} --watch{{ if .Spec.Logging.Debug }} -vvv{{ end }} $@
             {{ end }} # end if container.launcher
             {{ end }} # end if container.batch
         fi

diff --git a/docs/getting_started/custom-resource-definition.md b/docs/getting_started/custom-resource-definition.md
@@ -304,6 +304,16 @@ network:
 
 Settings under the Flux directive typically refer to flux options, e.g., for the broker or similar.
 
+#### submitCommand
+
+If you need to use a container with a different version of flux (or an older one)
+you might want to edit the submit command. You can do that as follows:
+
+```yaml
+flux:
+   submitCommand: "flux mini submit"
+```
+
 #### optionFlags
 
 Often when you run flux, you need to provide an option flag. E.g.,:

diff --git a/examples/dist/flux-operator-arm.yaml b/examples/dist/flux-operator-arm.yaml
@@ -390,6 +390,9 @@ spec:
                           also be "easy"
                         type: string
                     type: object
+                  submitCommand:
+                    description: Modify flux submit to be something else
+                    type: string
                   wrap:
                     description: Commands for flux start --wrap
                     type: string

diff --git a/examples/dist/flux-operator.yaml b/examples/dist/flux-operator.yaml
@@ -390,6 +390,9 @@ spec:
                           also be "easy"
                         type: string
                     type: object
+                  submitCommand:
+                    description: Modify flux submit to be something else
+                    type: string
                   wrap:
                     description: Commands for flux start --wrap
                     type: string

diff --git a/sdk/python/v1alpha1/docs/FluxSpec.md b/sdk/python/v1alpha1/docs/FluxSpec.md
@@ -15,6 +15,7 @@ Name | Type | Description | Notes
 **munge_secret** | **str** | Expect a secret (named according to this string) for a munge key. This is intended for bursting. Assumed to be at /etc/munge/munge.key This is binary data. | [optional] [default to '']
 **option_flags** | **str** | Flux option flags, usually provided with -o optional - if needed, default option flags for the server These can also be set in the user interface to override here. This is only valid for a FluxRunner \&quot;runFlux\&quot; true | [optional] [default to '']
 **scheduler** | [**FluxScheduler**](FluxScheduler.md) |  | [optional] 
+**submit_command** | **str** | Modify flux submit to be something else | [optional] 
 **wrap** | **str** | Commands for flux start --wrap | [optional] 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

diff --git a/sdk/python/v1alpha1/fluxoperator/models/flux_spec.py b/sdk/python/v1alpha1/fluxoperator/models/flux_spec.py
@@ -44,6 +44,7 @@ class FluxSpec(object):
         'munge_secret': 'str',
         'option_flags': 'str',
         'scheduler': 'FluxScheduler',
+        'submit_command': 'str',
         'wrap': 'str'
     }
 
@@ -59,10 +60,11 @@ class FluxSpec(object):
         'munge_secret': 'mungeSecret',
         'option_flags': 'optionFlags',
         'scheduler': 'scheduler',
+        'submit_command': 'submitCommand',
         'wrap': 'wrap'
     }
 
-    def __init__(self, broker_config='', bursting=None, connect_timeout='5s', curve_cert='', curve_cert_secret='', install_root='/usr', log_level=6, minimal_service=False, munge_secret='', option_flags='', scheduler=None, wrap=None, local_vars_configuration=None):  # noqa: E501
+    def __init__(self, broker_config='', bursting=None, connect_timeout='5s', curve_cert='', curve_cert_secret='', install_root='/usr', log_level=6, minimal_service=False, munge_secret='', option_flags='', scheduler=None, submit_command=None, wrap=None, local_vars_configuration=None):  # noqa: E501
         """FluxSpec - a model defined in OpenAPI"""  # noqa: E501
         if local_vars_configuration is None:
             local_vars_configuration = Configuration.get_default_copy()
@@ -79,6 +81,7 @@ def __init__(self, broker_config='', bursting=None, connect_timeout='5s', curve_
         self._munge_secret = None
         self._option_flags = None
         self._scheduler = None
+        self._submit_command = None
         self._wrap = None
         self.discriminator = None
 
@@ -104,6 +107,8 @@ def __init__(self, broker_config='', bursting=None, connect_timeout='5s', curve_
             self.option_flags = option_flags
         if scheduler is not None:
             self.scheduler = scheduler
+        if submit_command is not None:
+            self.submit_command = submit_command
         if wrap is not None:
             self.wrap = wrap
 
@@ -356,6 +361,29 @@ def scheduler(self, scheduler):
 
         self._scheduler = scheduler
 
+    @property
+    def submit_command(self):
+        """Gets the submit_command of this FluxSpec.  # noqa: E501
+
+        Modify flux submit to be something else  # noqa: E501
+
+        :return: The submit_command of this FluxSpec.  # noqa: E501
+        :rtype: str
+        """
+        return self._submit_command
+
+    @submit_command.setter
+    def submit_command(self, submit_command):
+        """Sets the submit_command of this FluxSpec.
+
+        Modify flux submit to be something else  # noqa: E501
+
+        :param submit_command: The submit_command of this FluxSpec.  # noqa: E501
+        :type submit_command: str
+        """
+
+        self._submit_command = submit_command
+
     @property
     def wrap(self):
         """Gets the wrap of this FluxSpec.  # noqa: E501