From 11ff7f1bc35664120a7b2d14c9279207432797c0 Mon Sep 17 00:00:00 2001 From: bjwswang Date: Wed, 1 Nov 2023 07:32:51 +0000 Subject: [PATCH] refactor: definition of versioned dataset Signed-off-by: bjwswang --- api/v1alpha1/condition.go | 27 ++++- api/v1alpha1/versioneddataset_types.go | 23 ++--- api/v1alpha1/zz_generated.deepcopy.go | 45 +++------ ....kubeagi.k8s.com.cn_versioneddatasets.yaml | 98 +++---------------- 4 files changed, 58 insertions(+), 135 deletions(-) diff --git a/api/v1alpha1/condition.go b/api/v1alpha1/condition.go index da93fedc0..a4665ff4f 100644 --- a/api/v1alpha1/condition.go +++ b/api/v1alpha1/condition.go @@ -42,7 +42,6 @@ type ConditionReason string // Some common Condition reasons. const ( - ReasonPublished ConditionReason = "Published" ReasonAvailable ConditionReason = "Available" ReasonUnavailable ConditionReason = "Unavailable" ReasonCreating ConditionReason = "Creating" @@ -52,6 +51,32 @@ const ( ReasonReconcilePaused ConditionReason = "ReconcilePaused" ) +// Some Data related Condition Types +const ( + // Dataset have 3 phases: load -> process -> publish + // TypeLoaded resources are believed to be loaded + TypeLoaded ConditionType = "Loaded" + // TypeProcessed resources are believed to be processed + TypeProcessed ConditionType = "Processed" + // TypePublished resources are believed to be published + TypePublished ConditionType = "Published" +) + +// Some Dataset related Condition reasons +const ( + // Load data + ReasonDataLoading ConditionReason = "DataLoading" + ReasonDataLoadError ConditionReason = "DataLoadError" + ReasonDataLoadSuccess ConditionReason = "DataLoadSuccess" + // Process data + ReasonDataProcessing ConditionReason = "DataProcessing" + ReasonDataProcessError ConditionReason = "DataProcessError" + ReasonDataProcessSuccess ConditionReason = "DataProcessSuccess" + // Publish dataset + ReasonDatasetUnpublished ConditionReason = "DatasetUnpublished" + ReasonDatasetPublished ConditionReason = "DatasetPublished" +) + // A Condition that may apply to a resource. type Condition struct { // Type of this condition. At most one of each condition type may apply to diff --git a/api/v1alpha1/versioneddataset_types.go b/api/v1alpha1/versioneddataset_types.go index d669e0de1..834233b5a 100644 --- a/api/v1alpha1/versioneddataset_types.go +++ b/api/v1alpha1/versioneddataset_types.go @@ -23,18 +23,11 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type File struct { +type FileGroup struct { // From defines the datasource which provides this `File` - From *TypedObjectReference `json:"from"` - // Path defines the detail path to get this `File` - Path string `json:"path"` -} - -type FileStatus struct { - // UploadCondition records the status of file upload - UploadCondition Condition `json:"uploadCondition,omitempty"` - // ProcessCondition records the status of data processing - ProcessCondition Condition `json:"processCondition,omitempty"` + Datasource *TypedObjectReference `json:"datasource"` + // Paths defines the detail paths to get objects from above datasource + Paths []string `json:"paths"` } // VersionedDatasetSpec defines the desired state of VersionedDataset @@ -45,17 +38,15 @@ type VersionedDatasetSpec struct { // Version Version string `json:"version"` - // Files included in this `VersionedDataset` - Files []File `json:"files,omitempty"` + // FileGroups included in this `VersionedDataset` + // Grouped by Datasource + FileGroups []FileGroup `json:"fileGroups,omitempty"` } // VersionedDatasetStatus defines the observed state of VersionedDataset type VersionedDatasetStatus struct { // ConditionedStatus is the current status ConditionedStatus `json:",inline"` - - // FilesStatus contains the status to all files in VersionedDatasetSpec - FilesStatus []FileStatus `json:"filesStatus,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8572c54f1..d849be646 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -371,38 +371,26 @@ func (in *Endpoint) DeepCopy() *Endpoint { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *File) DeepCopyInto(out *File) { +func (in *FileGroup) DeepCopyInto(out *FileGroup) { *out = *in - if in.From != nil { - in, out := &in.From, &out.From + if in.Datasource != nil { + in, out := &in.Datasource, &out.Datasource *out = new(TypedObjectReference) (*in).DeepCopyInto(*out) } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new File. -func (in *File) DeepCopy() *File { - if in == nil { - return nil + if in.Paths != nil { + in, out := &in.Paths, &out.Paths + *out = make([]string, len(*in)) + copy(*out, *in) } - out := new(File) - in.DeepCopyInto(out) - return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileStatus) DeepCopyInto(out *FileStatus) { - *out = *in - in.UploadCondition.DeepCopyInto(&out.UploadCondition) - in.ProcessCondition.DeepCopyInto(&out.ProcessCondition) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileStatus. -func (in *FileStatus) DeepCopy() *FileStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileGroup. +func (in *FileGroup) DeepCopy() *FileGroup { if in == nil { return nil } - out := new(FileStatus) + out := new(FileGroup) in.DeepCopyInto(out) return out } @@ -798,9 +786,9 @@ func (in *VersionedDatasetSpec) DeepCopyInto(out *VersionedDatasetSpec) { *out = new(TypedObjectReference) (*in).DeepCopyInto(*out) } - if in.Files != nil { - in, out := &in.Files, &out.Files - *out = make([]File, len(*in)) + if in.FileGroups != nil { + in, out := &in.FileGroups, &out.FileGroups + *out = make([]FileGroup, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -821,13 +809,6 @@ func (in *VersionedDatasetSpec) DeepCopy() *VersionedDatasetSpec { func (in *VersionedDatasetStatus) DeepCopyInto(out *VersionedDatasetStatus) { *out = *in in.ConditionedStatus.DeepCopyInto(&out.ConditionedStatus) - if in.FilesStatus != nil { - in, out := &in.FilesStatus, &out.FilesStatus - *out = make([]FileStatus, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VersionedDatasetStatus. diff --git a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_versioneddatasets.yaml b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_versioneddatasets.yaml index b2c83eb0d..9f06a7ea6 100644 --- a/config/crd/bases/arcadia.kubeagi.k8s.com.cn_versioneddatasets.yaml +++ b/config/crd/bases/arcadia.kubeagi.k8s.com.cn_versioneddatasets.yaml @@ -64,11 +64,12 @@ spec: - kind - name type: object - files: - description: Files included in this `VersionedDataset` + fileGroups: + description: FileGroups included in this `VersionedDataset` Grouped + by Datasource items: properties: - from: + datasource: description: From defines the datasource which provides this `File` properties: @@ -92,12 +93,15 @@ spec: - kind - name type: object - path: - description: Path defines the detail path to get this `File` - type: string + paths: + description: Paths defines the detail paths to get objects from + above datasource + items: + type: string + type: array required: - - from - - path + - datasource + - paths type: object type: array version: @@ -148,84 +152,6 @@ spec: - type type: object type: array - filesStatus: - description: FilesStatus contains the status to all files in VersionedDatasetSpec - items: - properties: - processCondition: - description: ProcessCondition records the status of data processing - properties: - lastSuccessfulTime: - description: LastSuccessfulTime is repository Last Successful - Update Time - format: date-time - type: string - lastTransitionTime: - description: LastTransitionTime is the last time this condition - transitioned from one status to another. - format: date-time - type: string - message: - description: A Message containing details about this condition's - last transition from one status to another, if any. - type: string - reason: - description: A Reason for this condition's last transition - from one status to another. - type: string - status: - description: Status of this condition; is it currently True, - False, or Unknown - type: string - type: - description: Type of this condition. At most one of each - condition type may apply to a resource at any point in - time. - type: string - required: - - lastTransitionTime - - reason - - status - - type - type: object - uploadCondition: - description: UploadCondition records the status of file upload - properties: - lastSuccessfulTime: - description: LastSuccessfulTime is repository Last Successful - Update Time - format: date-time - type: string - lastTransitionTime: - description: LastTransitionTime is the last time this condition - transitioned from one status to another. - format: date-time - type: string - message: - description: A Message containing details about this condition's - last transition from one status to another, if any. - type: string - reason: - description: A Reason for this condition's last transition - from one status to another. - type: string - status: - description: Status of this condition; is it currently True, - False, or Unknown - type: string - type: - description: Type of this condition. At most one of each - condition type may apply to a resource at any point in - time. - type: string - required: - - lastTransitionTime - - reason - - status - - type - type: object - type: object - type: array type: object type: object served: true