forked from knative-extensions/eventing-kafka
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[0.25] KafkaChannel to init offsets before dispatcher (knative-extens…
…ions#913) * Cherry pick 3f2a9d7 [0.24] KafkaChannel to init offsets before dispatcher (knative-extensions#886) * ./hack/update-codegen.sh
Showing
33 changed files
with
728 additions
and
2,170 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
60 changes: 0 additions & 60 deletions
60
pkg/channel/consolidated/dispatcher/subscription_endpoint.go
This file was deleted.
Oops, something went wrong.
155 changes: 0 additions & 155 deletions
155
pkg/channel/consolidated/dispatcher/subscription_endpoint_test.go
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
274 changes: 135 additions & 139 deletions
274
pkg/channel/consolidated/reconciler/controller/kafkachannel_test.go
Large diffs are not rendered by default.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,74 @@ | ||
package consumer | ||
|
||
import ( | ||
"context" | ||
"fmt" | ||
"time" | ||
|
||
"github.com/Shopify/sarama" | ||
"go.uber.org/zap" | ||
"k8s.io/apimachinery/pkg/util/wait" | ||
"knative.dev/eventing-kafka/pkg/common/kafka/offset" | ||
) | ||
|
||
const ( | ||
OffsetCheckRetryTimeout = 60 * time.Second | ||
OffsetCheckRetryInterval = 5 * time.Second | ||
) | ||
|
||
// wrapper functions for the Sarama functions, to facilitate unit testing | ||
var newSaramaClient = sarama.NewClient | ||
var newSaramaClusterAdmin = sarama.NewClusterAdmin | ||
|
||
type ConsumerGroupOffsetsChecker interface { | ||
WaitForOffsetsInitialization(ctx context.Context, groupID string, topics []string, logger *zap.SugaredLogger, addrs []string, config *sarama.Config) error | ||
} | ||
|
||
type NoopConsumerGroupOffsetsChecker struct { | ||
} | ||
|
||
func (c *NoopConsumerGroupOffsetsChecker) WaitForOffsetsInitialization(ctx context.Context, groupID string, topics []string, logger *zap.SugaredLogger, addrs []string, config *sarama.Config) error { | ||
return nil | ||
} | ||
|
||
type KafkaConsumerGroupOffsetsChecker struct { | ||
} | ||
|
||
func (k *KafkaConsumerGroupOffsetsChecker) WaitForOffsetsInitialization(ctx context.Context, groupID string, topics []string, logger *zap.SugaredLogger, addrs []string, config *sarama.Config) error { | ||
logger.Debugw("checking if all offsets are initialized", zap.Any("topics", topics), zap.Any("groupID", groupID)) | ||
|
||
client, err := newSaramaClient(addrs, config) | ||
if err != nil { | ||
logger.Errorw("unable to create Kafka client", zap.Any("topics", topics), zap.String("groupId", groupID), zap.Error(err)) | ||
return err | ||
} | ||
defer client.Close() | ||
|
||
clusterAdmin, err := newSaramaClusterAdmin(addrs, config) | ||
if err != nil { | ||
logger.Errorw("unable to create Kafka cluster admin client", zap.Any("topics", topics), zap.String("groupId", groupID), zap.Error(err)) | ||
return err | ||
} | ||
defer clusterAdmin.Close() | ||
|
||
check := func() (bool, error) { | ||
if initialized, err := offset.CheckIfAllOffsetsInitialized(client, clusterAdmin, topics, groupID); err == nil { | ||
if initialized { | ||
return true, nil | ||
} else { | ||
logger.Debugw("offsets not yet initialized, going to try again") | ||
return false, nil | ||
} | ||
} else { | ||
return false, fmt.Errorf("error checking if offsets are initialized. stopping trying. %w", err) | ||
} | ||
} | ||
pollCtx, pollCtxCancel := context.WithTimeout(ctx, OffsetCheckRetryTimeout) | ||
err = wait.PollUntil(OffsetCheckRetryInterval, check, pollCtx.Done()) | ||
defer pollCtxCancel() | ||
|
||
if err != nil { | ||
return fmt.Errorf("failed to check if offsets are initialized %w", err) | ||
} | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,113 @@ | ||
package testing | ||
|
||
import "github.com/Shopify/sarama" | ||
|
||
var _ sarama.ClusterAdmin = (*MockClusterAdmin)(nil) | ||
|
||
type MockClusterAdmin struct { | ||
MockCreateTopicFunc func(topic string, detail *sarama.TopicDetail, validateOnly bool) error | ||
MockDeleteTopicFunc func(topic string) error | ||
MockListConsumerGroupsFunc func() (map[string]string, error) | ||
} | ||
|
||
func (ca *MockClusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*sarama.PartitionReplicaReassignmentsStatus, err error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeLogDirs(brokers []int32) (map[int32][]sarama.DescribeLogDirsResponseDirMetadata, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeUserScramCredentials(users []string) ([]*sarama.DescribeUserScramCredentialsResult, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DeleteUserScramCredentials(delete []sarama.AlterUserScramCredentialsDelete) ([]*sarama.AlterUserScramCredentialsResult, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) UpsertUserScramCredentials(upsert []sarama.AlterUserScramCredentialsUpsert) ([]*sarama.AlterUserScramCredentialsResult, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) CreateTopic(topic string, detail *sarama.TopicDetail, validateOnly bool) error { | ||
if ca.MockCreateTopicFunc != nil { | ||
return ca.MockCreateTopicFunc(topic, detail, validateOnly) | ||
} | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) Close() error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DeleteTopic(topic string) error { | ||
if ca.MockDeleteTopicFunc != nil { | ||
return ca.MockDeleteTopicFunc(topic) | ||
} | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeTopics(topics []string) (metadata []*sarama.TopicMetadata, err error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) ListTopics() (map[string]sarama.TopicDetail, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeConfig(resource sarama.ConfigResource) ([]sarama.ConfigEntry, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) AlterConfig(resourceType sarama.ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) CreateACL(resource sarama.Resource, acl sarama.Acl) error { | ||
return nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) ListAcls(filter sarama.AclFilter) ([]sarama.ResourceAcls, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DeleteACL(filter sarama.AclFilter, validateOnly bool) ([]sarama.MatchingAcl, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) ListConsumerGroups() (map[string]string, error) { | ||
if ca.MockListConsumerGroupsFunc != nil { | ||
return ca.MockListConsumerGroupsFunc() | ||
} | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeConsumerGroups(groups []string) ([]*sarama.GroupDescription, error) { | ||
return nil, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*sarama.OffsetFetchResponse, error) { | ||
return &sarama.OffsetFetchResponse{}, nil | ||
} | ||
|
||
func (ca *MockClusterAdmin) DescribeCluster() (brokers []*sarama.Broker, controllerID int32, err error) { | ||
return nil, 0, nil | ||
} | ||
|
||
// Delete a consumer group. | ||
func (ca *MockClusterAdmin) DeleteConsumerGroup(group string) error { | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
201 changes: 0 additions & 201 deletions
201
third_party/VENDOR-LICENSE/knative.dev/networking/pkg/prober/LICENSE
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters