Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(indexer): add to modules and implement proto fields #22544

Merged
merged 29 commits into from
Nov 28, 2024
Merged

feat(indexer): add to modules and implement proto fields #22544

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
24879d0
progress
facundomedica Nov 18, 2024
8047466
progress
facundomedica Nov 18, 2024
5a52970
progress
facundomedica Nov 18, 2024
ce8b931
progress, need to revisit enums
facundomedica Nov 19, 2024
86fc09a
progrss
facundomedica Nov 19, 2024
7e5a665
remove debug prints
facundomedica Nov 19, 2024
4ebf63f
small fix
facundomedica Nov 19, 2024
38fcf4c
add enums
facundomedica Nov 21, 2024
ee06bf2
add x/auth + CollInterfaceValue schemaCodec
facundomedica Nov 22, 2024
5381823
add some more modules just to make sure these things work
facundomedica Nov 22, 2024
748bac9
added mint
facundomedica Nov 22, 2024
3cd0277
handle nil schematype
facundomedica Nov 25, 2024
4b3b800
remove print
facundomedica Nov 25, 2024
0a73e70
merge main
facundomedica Nov 25, 2024
ebd7397
return errors instead of os.Exit
facundomedica Nov 26, 2024
5196e56
remove toolchain updates
facundomedica Nov 26, 2024
6ed03b5
add cl
facundomedica Nov 26, 2024
6c03930
lint fix
facundomedica Nov 26, 2024
f4a3aa0
fix tests
facundomedica Nov 26, 2024
5bb37a3
fixing stuff
facundomedica Nov 26, 2024
de642e1
fix more tests
facundomedica Nov 26, 2024
668838f
rollback instead
facundomedica Nov 27, 2024
f3cd58f
fix more stuff
facundomedica Nov 27, 2024
80d762e
Merge branch 'main' into facu/staking-indexer1
facundomedica Nov 27, 2024
91674b1
lint
facundomedica Nov 27, 2024
228ffb5
rollback some go mod changes
facundomedica Nov 27, 2024
e320f48
go mod
facundomedica Nov 27, 2024
d94bde6
suggestions from Julien
facundomedica Nov 28, 2024
dc20c04
Merge branch 'main' into facu/staking-indexer1
facundomedica Nov 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 12 additions & 3 deletions baseapp/abci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -903,8 +903,12 @@ func (app *BaseApp) FinalizeBlock(req *abci.FinalizeBlockRequest) (res *abci.Fin
defer func() {
// call the streaming service hooks with the FinalizeBlock messages
for _, streamingListener := range app.streamingManager.ABCIListeners {
if err := streamingListener.ListenFinalizeBlock(app.finalizeBlockState.Context(), *req, *res); err != nil {
if streamErr := streamingListener.ListenFinalizeBlock(app.finalizeBlockState.Context(), *req, *res); streamErr != nil {
app.logger.Error("ListenFinalizeBlock listening hook failed", "height", req.Height, "err", err)
if app.streamingManager.StopNodeOnErr {
// if StopNodeOnErr is set, we should return the streamErr in order to stop the node
err = streamErr
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

doesn't this get override line 937? Shouldn't we return the error as the comment says?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is inside a defer, so it will replace the one we are returning

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see yeah, so I suppose we should join it too right?

}
facundomedica marked this conversation as resolved.
Show resolved Hide resolved
}
}
}()
Expand Down Expand Up @@ -976,8 +980,6 @@ func (app *BaseApp) Commit() (*abci.CommitResponse, error) {
rms.SetCommitHeader(header)
}

app.cms.Commit()

resp := &abci.CommitResponse{
RetainHeight: retainHeight,
}
Expand All @@ -991,10 +993,17 @@ func (app *BaseApp) Commit() (*abci.CommitResponse, error) {
for _, abciListener := range abciListeners {
if err := abciListener.ListenCommit(ctx, *resp, changeSet); err != nil {
app.logger.Error("Commit listening hook failed", "height", blockHeight, "err", err)
if app.streamingManager.StopNodeOnErr {
return nil, err
}
}
}
}

// Commit after all listeners have been called, in case they error and we
// need to stop before committing.
app.cms.Commit()

// Reset the CheckTx state to the latest committed.
//
// NOTE: This is safe because CometBFT holds a lock on the mempool for
Expand Down
2 changes: 1 addition & 1 deletion client/v2/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module cosmossdk.io/client/v2

go 1.23.1
go 1.23.2

require (
cosmossdk.io/api v0.7.6
Expand Down
248 changes: 247 additions & 1 deletion codec/collections.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
package codec

import (
"encoding/json"
"fmt"
"reflect"
"strings"

"github.com/cosmos/gogoproto/proto"
gogotypes "github.com/cosmos/gogoproto/types"
"google.golang.org/protobuf/encoding/protojson"
protov2 "google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/dynamicpb"
"google.golang.org/protobuf/types/known/durationpb"
"google.golang.org/protobuf/types/known/timestamppb"

"cosmossdk.io/collections"
collcodec "cosmossdk.io/collections/codec"
"cosmossdk.io/schema"
)

// BoolValue implements a ValueCodec that saves the bool value
Expand Down Expand Up @@ -51,12 +58,17 @@ type protoMessage[T any] interface {
proto.Message
}

type protoCollValueCodec[T any] interface {
collcodec.HasSchemaCodec[T]
collcodec.ValueCodec[T]
}

// CollValue inits a collections.ValueCodec for a generic gogo protobuf message.
func CollValue[T any, PT protoMessage[T]](cdc interface {
Marshal(proto.Message) ([]byte, error)
Unmarshal([]byte, proto.Message) error
},
) collcodec.ValueCodec[T] {
) protoCollValueCodec[T] {
return &collValue[T, PT]{cdc.(Codec), proto.MessageName(PT(new(T)))}
}

Expand Down Expand Up @@ -91,6 +103,139 @@ func (c collValue[T, PT]) ValueType() string {
return "github.com/cosmos/gogoproto/" + c.messageName
}

func (c collValue[T, PT]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
t T
pt PT
)
msgName := proto.MessageName(pt)
desc, err := proto.HybridResolver.FindDescriptorByName(protoreflect.FullName(msgName))
if err != nil {
return collcodec.SchemaCodec[T]{}, fmt.Errorf("could not find descriptor for %s: %w", msgName, err)
}
schemaFields := protoCols(desc.(protoreflect.MessageDescriptor))

kind := schema.KindForGoValue(t)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: schemaFields,
ToSchemaType: func(t T) (any, error) {
values := []interface{}{}
msgDesc, ok := desc.(protoreflect.MessageDescriptor)
if !ok {
return nil, fmt.Errorf("expected message descriptor, got %T", desc)
}

nm := dynamicpb.NewMessage(msgDesc)
bz, err := c.cdc.Marshal(any(&t).(PT))
if err != nil {
return nil, err
}

err = c.cdc.Unmarshal(bz, nm)
if err != nil {
return nil, err
}

for _, field := range schemaFields {
// Find the field descriptor by the Protobuf field name
fieldDesc := msgDesc.Fields().ByName(protoreflect.Name(field.Name))
if fieldDesc == nil {
return nil, fmt.Errorf("field %q not found in message %s", field.Name, desc.FullName())
}

val := nm.ProtoReflect().Get(fieldDesc)

// if the field is a map or list, we need to convert it to a slice of values
if fieldDesc.IsList() {
repeatedVals := []interface{}{}
list := val.List()
for i := 0; i < list.Len(); i++ {
repeatedVals = append(repeatedVals, list.Get(i).Interface())
}
values = append(values, repeatedVals)
continue
}

switch fieldDesc.Kind() {
case protoreflect.BoolKind:
values = append(values, val.Bool())
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind,
protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
values = append(values, val.Int())
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind,
protoreflect.Fixed64Kind:
values = append(values, val.Uint())
case protoreflect.FloatKind, protoreflect.DoubleKind:
values = append(values, val.Float())
case protoreflect.StringKind:
values = append(values, val.String())
case protoreflect.BytesKind:
values = append(values, val.Bytes())
case protoreflect.EnumKind:
// TODO: postgres uses the enum name, not the number
values = append(values, string(fieldDesc.Enum().Values().ByNumber(val.Enum()).Name()))
case protoreflect.MessageKind:
msg := val.Interface().(*dynamicpb.Message)
msgbz, err := c.cdc.Marshal(msg)
if err != nil {
return nil, err
}

if field.Kind == schema.TimeKind {
// make it a time.Time
ts := &timestamppb.Timestamp{}
err = c.cdc.Unmarshal(msgbz, ts)
if err != nil {
return nil, fmt.Errorf("error unmarshalling timestamp: %w %x %s", err, msgbz, fieldDesc.FullName())
}
values = append(values, ts.AsTime())
} else if field.Kind == schema.DurationKind {
// make it a time.Duration
dur := &durationpb.Duration{}
err = c.cdc.Unmarshal(msgbz, dur)
if err != nil {
return nil, fmt.Errorf("error unmarshalling duration: %w", err)
}
values = append(values, dur.AsDuration())
} else {
// if not a time or duration, just keep it as a JSON object
// we might want to change this to include the entire object as separate fields
bz, err := c.cdc.MarshalJSON(msg)
if err != nil {
return nil, fmt.Errorf("error marshalling message: %w", err)
}

values = append(values, json.RawMessage(bz))
}
}

}

// if there's only one value, return it directly
if len(values) == 1 {
return values[0], nil
}
return values, nil
},
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
Comment on lines +232 to +234
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Avoid using panic in library code; return an error instead

Using panic("not implemented") in the FromSchemaType function can cause unexpected crashes in consuming applications. It's recommended to return an error to handle unimplemented functionality gracefully.

Apply this diff to replace the panic with an error return:

func(a any) (T, error) {
-    panic("not implemented")
+    return T{}, fmt.Errorf("FromSchemaType is not implemented")
},
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
FromSchemaType: func(a any) (T, error) {
return T{}, fmt.Errorf("FromSchemaType is not implemented")
},

}, nil
}
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add error handling instead of panicking in FromSchemaType

Using panic("not implemented") in library code can lead to unexpected crashes. It's better practice to return an error, allowing the caller to handle it appropriately.

Apply this diff to replace the panic with an error return:

 func (c collValue[T, PT]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
   // ...
   FromSchemaType: func(a any) (T, error) {
-    panic("not implemented")
+    var t T
+    return t, fmt.Errorf("FromSchemaType not implemented")
   },
 },
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
func (c collValue[T, PT]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
t T
pt PT
)
msgName := proto.MessageName(pt)
desc, err := proto.HybridResolver.FindDescriptorByName(protoreflect.FullName(msgName))
if err != nil {
return collcodec.SchemaCodec[T]{}, fmt.Errorf("could not find descriptor for %s: %w", msgName, err)
}
schemaFields := protoCols(desc.(protoreflect.MessageDescriptor))
kind := schema.KindForGoValue(t)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: schemaFields,
ToSchemaType: func(t T) (any, error) {
values := []interface{}{}
msgDesc, ok := desc.(protoreflect.MessageDescriptor)
if !ok {
return nil, fmt.Errorf("expected message descriptor, got %T", desc)
}
nm := dynamicpb.NewMessage(msgDesc)
bz, err := c.cdc.Marshal(any(&t).(PT))
if err != nil {
return nil, err
}
err = c.cdc.Unmarshal(bz, nm)
if err != nil {
return nil, err
}
for _, field := range schemaFields {
// Find the field descriptor by the Protobuf field name
fieldDesc := msgDesc.Fields().ByName(protoreflect.Name(field.Name))
if fieldDesc == nil {
return nil, fmt.Errorf("field %q not found in message %s", field.Name, desc.FullName())
}
val := nm.ProtoReflect().Get(fieldDesc)
// if the field is a map or list, we need to convert it to a slice of values
if fieldDesc.IsList() {
repeatedVals := []interface{}{}
list := val.List()
for i := 0; i < list.Len(); i++ {
repeatedVals = append(repeatedVals, list.Get(i).Interface())
}
values = append(values, repeatedVals)
continue
}
switch fieldDesc.Kind() {
case protoreflect.BoolKind:
values = append(values, val.Bool())
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind,
protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
values = append(values, val.Int())
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind,
protoreflect.Fixed64Kind:
values = append(values, val.Uint())
case protoreflect.FloatKind, protoreflect.DoubleKind:
values = append(values, val.Float())
case protoreflect.StringKind:
values = append(values, val.String())
case protoreflect.BytesKind:
values = append(values, val.Bytes())
case protoreflect.EnumKind:
// TODO: postgres uses the enum name, not the number
values = append(values, string(fieldDesc.Enum().Values().ByNumber(val.Enum()).Name()))
case protoreflect.MessageKind:
msg := val.Interface().(*dynamicpb.Message)
msgbz, err := c.cdc.Marshal(msg)
if err != nil {
return nil, err
}
if field.Kind == schema.TimeKind {
// make it a time.Time
ts := &timestamppb.Timestamp{}
err = c.cdc.Unmarshal(msgbz, ts)
if err != nil {
return nil, fmt.Errorf("error unmarshalling timestamp: %w %x %s", err, msgbz, fieldDesc.FullName())
}
values = append(values, ts.AsTime())
} else if field.Kind == schema.DurationKind {
// make it a time.Duration
dur := &durationpb.Duration{}
err = c.cdc.Unmarshal(msgbz, dur)
if err != nil {
return nil, fmt.Errorf("error unmarshalling duration: %w", err)
}
values = append(values, dur.AsDuration())
} else {
// if not a time or duration, just keep it as a JSON object
// we might want to change this to include the entire object as separate fields
bz, err := c.cdc.MarshalJSON(msg)
if err != nil {
return nil, fmt.Errorf("error marshalling message: %w", err)
}
values = append(values, json.RawMessage(bz))
}
}
}
// if there's only one value, return it directly
if len(values) == 1 {
return values[0], nil
}
return values, nil
},
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
}, nil
}
}
func (c collValue[T, PT]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
t T
pt PT
)
msgName := proto.MessageName(pt)
desc, err := proto.HybridResolver.FindDescriptorByName(protoreflect.FullName(msgName))
if err != nil {
return collcodec.SchemaCodec[T]{}, fmt.Errorf("could not find descriptor for %s: %w", msgName, err)
}
schemaFields := protoCols(desc.(protoreflect.MessageDescriptor))
kind := schema.KindForGoValue(t)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: schemaFields,
ToSchemaType: func(t T) (any, error) {
values := []interface{}{}
msgDesc, ok := desc.(protoreflect.MessageDescriptor)
if !ok {
return nil, fmt.Errorf("expected message descriptor, got %T", desc)
}
nm := dynamicpb.NewMessage(msgDesc)
bz, err := c.cdc.Marshal(any(&t).(PT))
if err != nil {
return nil, err
}
err = c.cdc.Unmarshal(bz, nm)
if err != nil {
return nil, err
}
for _, field := range schemaFields {
// Find the field descriptor by the Protobuf field name
fieldDesc := msgDesc.Fields().ByName(protoreflect.Name(field.Name))
if fieldDesc == nil {
return nil, fmt.Errorf("field %q not found in message %s", field.Name, desc.FullName())
}
val := nm.ProtoReflect().Get(fieldDesc)
// if the field is a map or list, we need to convert it to a slice of values
if fieldDesc.IsList() {
repeatedVals := []interface{}{}
list := val.List()
for i := 0; i < list.Len(); i++ {
repeatedVals = append(repeatedVals, list.Get(i).Interface())
}
values = append(values, repeatedVals)
continue
}
switch fieldDesc.Kind() {
case protoreflect.BoolKind:
values = append(values, val.Bool())
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind,
protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
values = append(values, val.Int())
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind, protoreflect.Uint64Kind,
protoreflect.Fixed64Kind:
values = append(values, val.Uint())
case protoreflect.FloatKind, protoreflect.DoubleKind:
values = append(values, val.Float())
case protoreflect.StringKind:
values = append(values, val.String())
case protoreflect.BytesKind:
values = append(values, val.Bytes())
case protoreflect.EnumKind:
// TODO: postgres uses the enum name, not the number
values = append(values, string(fieldDesc.Enum().Values().ByNumber(val.Enum()).Name()))
case protoreflect.MessageKind:
msg := val.Interface().(*dynamicpb.Message)
msgbz, err := c.cdc.Marshal(msg)
if err != nil {
return nil, err
}
if field.Kind == schema.TimeKind {
// make it a time.Time
ts := &timestamppb.Timestamp{}
err = c.cdc.Unmarshal(msgbz, ts)
if err != nil {
return nil, fmt.Errorf("error unmarshalling timestamp: %w %x %s", err, msgbz, fieldDesc.FullName())
}
values = append(values, ts.AsTime())
} else if field.Kind == schema.DurationKind {
// make it a time.Duration
dur := &durationpb.Duration{}
err = c.cdc.Unmarshal(msgbz, dur)
if err != nil {
return nil, fmt.Errorf("error unmarshalling duration: %w", err)
}
values = append(values, dur.AsDuration())
} else {
// if not a time or duration, just keep it as a JSON object
// we might want to change this to include the entire object as separate fields
bz, err := c.cdc.MarshalJSON(msg)
if err != nil {
return nil, fmt.Errorf("error marshalling message: %w", err)
}
values = append(values, json.RawMessage(bz))
}
}
}
// if there's only one value, return it directly
if len(values) == 1 {
return values[0], nil
}
return values, nil
},
FromSchemaType: func(a any) (T, error) {
var t T
return t, fmt.Errorf("FromSchemaType not implemented")
},
}, nil
}
}
🧰 Tools
🪛 golangci-lint (1.61.0)

217-217: marshalling is a misspelling of marshaling

(misspell)


type protoMessageV2[T any] interface {
*T
protov2.Message
Expand Down Expand Up @@ -179,3 +324,104 @@ func (c collInterfaceValue[T]) ValueType() string {
var t T
return fmt.Sprintf("%T", t)
}

// SchemaCodec returns a schema codec, which will always have a single JSON field
// as there is no way to know in advance the necessary fields for an interface.
func (c collInterfaceValue[T]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
pt T
)

kind := schema.KindForGoValue(pt)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
Name: "value",
Kind: schema.JSONKind,
}},
ToSchemaType: func(t T) (any, error) {
bz, err := c.codec.MarshalInterfaceJSON(t)
if err != nil {
return nil, err
}

return json.RawMessage(bz), nil
},
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
Comment on lines +359 to +361
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue

Avoid using panic in library code; return an error instead

The use of panic("not implemented") in the FromSchemaType function is discouraged. Returning an error provides a safer way to handle unimplemented functions.

Apply this diff to replace the panic with an error return:

func(a any) (T, error) {
-    panic("not implemented")
+    var t T
+    return t, fmt.Errorf("FromSchemaType is not implemented")
},
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
FromSchemaType: func(a any) (T, error) {
var t T
return t, fmt.Errorf("FromSchemaType is not implemented")
},

}, nil
}

}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Add error handling instead of panicking in FromSchemaType

As with the previous instance, replace panic("not implemented") with an error return to prevent unexpected crashes and improve robustness.

Modify the code as follows:

 func (c collInterfaceValue[T]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
   // ...
   FromSchemaType: func(a any) (T, error) {
-    panic("not implemented")
+    var t T
+    return t, fmt.Errorf("FromSchemaType not implemented")
   },
 },
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
// SchemaCodec returns a schema codec, which will always have a single JSON field
// as there is no way to know in advance the necessary fields for an interface.
func (c collInterfaceValue[T]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
pt T
)
kind := schema.KindForGoValue(pt)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
Name: "value",
Kind: schema.JSONKind,
}},
ToSchemaType: func(t T) (any, error) {
bz, err := c.codec.MarshalInterfaceJSON(t)
if err != nil {
return nil, err
}
return json.RawMessage(bz), nil
},
FromSchemaType: func(a any) (T, error) {
panic("not implemented")
},
}, nil
}
}
// SchemaCodec returns a schema codec, which will always have a single JSON field
// as there is no way to know in advance the necessary fields for an interface.
func (c collInterfaceValue[T]) SchemaCodec() (collcodec.SchemaCodec[T], error) {
var (
pt T
)
kind := schema.KindForGoValue(pt)
if err := kind.Validate(); err == nil {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
// we don't set any name so that this can be set to a good default by the caller
Name: "",
Kind: kind,
}},
// these can be nil because T maps directly to a schema value for this kind
ToSchemaType: nil,
FromSchemaType: nil,
}, nil
} else {
return collcodec.SchemaCodec[T]{
Fields: []schema.Field{{
Name: "value",
Kind: schema.JSONKind,
}},
ToSchemaType: func(t T) (any, error) {
bz, err := c.codec.MarshalInterfaceJSON(t)
if err != nil {
return nil, err
}
return json.RawMessage(bz), nil
},
FromSchemaType: func(a any) (T, error) {
var t T
return t, fmt.Errorf("FromSchemaType not implemented")
},
}, nil
}
}
🧰 Tools
🪛 golangci-lint (1.61.0)

331-331: File is not gofumpt-ed with -extra

(gofumpt)

366-366: File is not gofumpt-ed with -extra

(gofumpt)


func protoCols(desc protoreflect.MessageDescriptor) []schema.Field {
nFields := desc.Fields()
cols := make([]schema.Field, 0, nFields.Len())
for i := 0; i < nFields.Len(); i++ {
f := nFields.Get(i)
cols = append(cols, protoCol(f))
}
return cols
}

func protoCol(f protoreflect.FieldDescriptor) schema.Field {
col := schema.Field{Name: string(f.Name())}
if f.IsMap() || f.IsList() {
col.Kind = schema.JSONKind
col.Nullable = true
} else {
switch f.Kind() {
case protoreflect.BoolKind:
col.Kind = schema.BoolKind
case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind:
col.Kind = schema.Int32Kind
case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind:
col.Kind = schema.Int64Kind
case protoreflect.Uint32Kind, protoreflect.Fixed32Kind:
col.Kind = schema.Int64Kind
case protoreflect.Uint64Kind, protoreflect.Fixed64Kind:
col.Kind = schema.Uint64Kind
case protoreflect.FloatKind:
col.Kind = schema.Float32Kind
case protoreflect.DoubleKind:
col.Kind = schema.Float64Kind
case protoreflect.StringKind:
col.Kind = schema.StringKind
case protoreflect.BytesKind:
col.Kind = schema.BytesKind
case protoreflect.EnumKind:
// TODO: support enums
col.Kind = schema.EnumKind
// use the full name to avoid collissions
col.ReferencedType = string(f.Enum().FullName())
col.ReferencedType = strings.ReplaceAll(col.ReferencedType, ".", "_")
case protoreflect.MessageKind:
col.Nullable = true
fullName := f.Message().FullName()
if fullName == "google.protobuf.Timestamp" {
col.Kind = schema.TimeKind
} else if fullName == "google.protobuf.Duration" {
col.Kind = schema.DurationKind
} else {
col.Kind = schema.JSONKind
}
}
if f.HasPresence() {
col.Nullable = true
}
}

return col
}
8 changes: 6 additions & 2 deletions collections/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,22 @@
module cosmossdk.io/collections

go 1.23
go 1.23.2

require (
cosmossdk.io/core v1.0.0-alpha.6
cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29
cosmossdk.io/schema v0.3.0
github.com/cosmos/gogoproto v1.7.0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💡 Codebase verification

Update google.golang.org/protobuf to address security vulnerability

The codebase extensively uses protojson.Unmarshal across multiple packages, making it susceptible to the reported moderate severity vulnerability (infinite loop when unmarshaling certain forms of invalid JSON). Since this is a security concern and the function is used in critical paths including transaction processing and ORM operations, an update is recommended.

  • Update google.golang.org/protobuf to at least version v1.33.0 in go.mod
  • Test the changes thoroughly as this is a widely used dependency in the codebase
🔗 Analysis chain

Verify protobuf version compatibility

The PR adds protobuf-related dependencies which align with the implementation of proto fields. However, let's verify if we're using the latest stable version of google.golang.org/protobuf.

Also applies to: 14-14

Update google.golang.org/protobuf to v1.33.0 or later

The current version v1.35.2 is vulnerable to a moderate severity issue (infinite loop in protojson.Unmarshal) that was patched in v1.33.0. While v1.35.2 is actually the latest version available, it still contains this vulnerability according to the GitHub Security Advisory database. This suggests there might be a discrepancy in the version information, and it would be prudent to verify this further.

🏁 Scripts executed

The following scripts were executed for the analysis:

Script:

#!/bin/bash
# Check latest version of google.golang.org/protobuf
curl -s "https://proxy.golang.org/google.golang.org/protobuf/@v/list" | sort -V | tail -n 1

# Check if there are any known issues with the current version
gh api graphql -f query='
{
  securityVulnerabilities(first: 5, ecosystem: GO, package: "google.golang.org/protobuf") {
    nodes {
      advisory {
        summary
        severity
        publishedAt
      }
      vulnerableVersionRange
      firstPatchedVersion {
        identifier
      }
    }
  }
}'

Length of output: 974

Script:

#!/bin/bash
# Double check the version information from go.mod
rg "google.golang.org/protobuf" -A 1 -B 1 go.mod

# Check if protojson.Unmarshal is actually used in the codebase
rg "protojson\.Unmarshal"

Length of output: 1499

github.com/stretchr/testify v1.10.0
github.com/tidwall/btree v1.7.0
google.golang.org/protobuf v1.35.2
pgregory.net/rapid v1.1.0
)

require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
12 changes: 10 additions & 2 deletions collections/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,22 @@ cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29 h1:NxxUo0GMJUbIuVg0
cosmossdk.io/core/testing v0.0.0-20240923163230-04da382a9f29/go.mod h1:8s2tPeJtSiQuoyPmr2Ag7meikonISO4Fv4MoO8+ORrs=
cosmossdk.io/schema v0.3.0 h1:01lcaM4trhzZ1HQTfTV8z6Ma1GziOZ/YmdzBN3F720c=
cosmossdk.io/schema v0.3.0/go.mod h1:RDAhxIeNB4bYqAlF4NBJwRrgtnciMcyyg0DOKnhNZQQ=
github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro=
github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI=
github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY=
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
Expand Down
Loading
Loading