Skip to content

Commit

Permalink
Add TLS support and OpsRequests for Druid (#1314)
Browse files Browse the repository at this point in the history 
* Add druid TLS

Signed-off-by: Tapajit Chandra Paul <[email protected]>
  • Loading branch information
@tapojit047
tapojit047 authored Oct 25, 2024
1 parent 765bca2 commit 405c179
Show file tree
Hide file tree
Showing 12 changed files with 894 additions and 79 deletions.
87 changes: 63 additions & 24 deletions apis/kubedb/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -998,15 +998,16 @@ const (
DruidConfigDirRouters = "/opt/druid/conf/druid/cluster/query/router"
DruidCConfigDirMySQLMetadata = "/opt/druid/extensions/mysql-metadata-storage"

DruidVolumeOperatorConfig = "operator-config-volume"
DruidVolumeMainConfig = "main-config-volume"
DruidVolumeCustomConfig = "custom-config"
DruidMetadataTLSVolume = "metadata-tls-volume"
DruidVolumeOperatorConfig = "operator-config-volume"
DruidVolumeMainConfig = "main-config-volume"
DruidVolumeCustomConfig = "custom-config"
DruidMetadataTLSVolume = "metadata-tls-volume"
DruidMetadataTLSTempVolume = "metadata-tls-volume-temp"

DruidOperatorConfigDir = "/tmp/config/operator-config"
DruidMainConfigDir = "/opt/druid/conf"
DruidCustomConfigDir = "/tmp/config/custom-config"
DruidMetadataTLSConfigDir = "/tmp/metadata-tls"
DruidOperatorConfigDir = "/tmp/config/operator-config"
DruidMainConfigDir = "/opt/druid/conf"
DruidCustomConfigDir = "/tmp/config/custom-config"
DruidMetadataTLSTempConfigDir = "/tmp/metadata-tls"

DruidVolumeCommonConfig = "common-config-volume"
DruidCommonConfigFile = "common.runtime.properties"
Expand All @@ -1032,14 +1033,23 @@ const (
EnvDruidCoordinatorAsOverlord = "DRUID_COORDINATOR_AS_OVERLORD"
EnvDruidMetadataTLSEnable = "DRUID_METADATA_TLS_ENABLE"
EnvDruidMetadataStorageType = "DRUID_METADATA_STORAGE_TYPE"
EnvDruidKeyStorePassword = "DRUID_KEY_STORE_PASSWORD"

DruidPortCoordinators = 8081
DruidPortOverlords = 8090
DruidPortHistoricals = 8083
DruidPortMiddleManagers = 8091
DruidPortBrokers = 8082
DruidPortRouters = 8888
DruidExporterPort = 9104
DruidPlainTextPortCoordinators = 8081
DruidPlainTextPortOverlords = 8090
DruidPlainTextPortHistoricals = 8083
DruidPlainTextPortMiddleManagers = 8091
DruidPlainTextPortBrokers = 8082
DruidPlainTextPortRouters = 8888

DruidTLSPortCoordinators = 8281
DruidTLSPortOverlords = 8290
DruidTLSPortHistoricals = 8283
DruidTLSPortMiddleManagers = 8291
DruidTLSPortBrokers = 8282
DruidTLSPortRouters = 9088

DruidExporterPort = 9104

DruidMetadataStorageTypePostgres = "Postgres"

Expand All @@ -1062,24 +1072,49 @@ const (
DruidMetadataStorageConnectorPasswordEnvConfig = "{\"type\": \"environment\", \"variable\": \"DRUID_METADATA_STORAGE_PASSWORD\"}"
DruidMetadataStorageCreateTables = "druid.metadata.storage.connector.createTables"

// Druid TLS
DruidKeystorePasswordKey = "keystore_password"
DruidTrustStorePasswordKey = "truststore_password"
DruidKeystoreSecretKey = "keystore-cred"

DruidEnablePlaintextPort = "druid.enablePlaintextPort"
DruidEnableTLSPort = "druid.enableTlsPort"
DruidKeyStorePath = "druid.server.https.keyStorePath"
DruidKeyStoreType = "druid.server.https.keyStoreType"
DruidCertAlias = "druid.server.https.certAlias"
DruidKeyStorePassword = "druid.server.https.keyStorePassword"
DruidRequireClientCertificate = "druid.server.https.requireClientCertificate"
DruidTrustStoreType = "druid.server.https.trustStoreType"

DruidTrustStorePassword = "druid.client.https.trustStorePassword"
DruidTrustStorePath = "druid.client.https.trustStorePath"
DruidClientTrustStoreType = "druid.client.https.trustStoreType"
DruidClientValidateHostNames = "druid.client.https.validateHostnames"

DruidKeyStoreTypeJKS = "jks"
DruidKeyStorePasswordEnvConfig = "{\"type\": \"environment\", \"variable\": \"DRUID_KEY_STORE_PASSWORD\"}"

DruidValueTrue = "true"
DruidValueFalse = "false"

DruidCertDir = "/opt/druid/ssl"
DruidCertMetadataSubDir = "metadata"

// MySQL TLS
DruidMetadataMySQLUseSSL = "druid.metadata.mysql.ssl.useSSL"
DruidMetadataMySQLClientCertKeyStoreURL = "druid.metadata.mysql.ssl.clientCertificateKeyStoreUrl"
DruidMetadataMySQLClientCertKeyStorePath = "/opt/druid/conf/tls/metadatakeystore.jks"
DruidMetadataMySQLClientCertKeyStoreType = "druid.metadata.mysql.ssl.clientCertificateKeyStoreType"
DruidMetadataMySQLClientCertKeyStoreTypeJKS = "JKS"
DruidMetadataMySQLClientCertKeyStorePassword = "druid.metadata.mysql.ssl.clientCertificateKeyStorePassword"
DruidMetadataMySQLClientCertKeyStorePasswordValue = "password"

// Postgres TLS
DruidMetadataPostgresUseSSL = "druid.metadata.postgres.ssl.useSSL"
DruidMetadataPGUseSSLMode = "druid.metadata.postgres.ssl.sslMode"
DruidMetadataPGSSLCert = "druid.metadata.postgres.ssl.sslCert"
DruidMetadataPGSSLCertPath = "/opt/druid/conf/tls/tls.crt"
DruidMetadataPGSSLKey = "druid.metadata.postgres.ssl.sslKey"
DruidMetadataPGSSLKeyPath = "/opt/druid/conf/tls/tls.key"
DruidMetadataPGSSLRootCert = "druid.metadata.postgres.ssl.sslRootCert"
DruidMetadataPGSSLRootCertPath = "/opt/druid/conf/tls/ca.cert"
DruidMetadataPostgresUseSSL = "druid.metadata.postgres.ssl.useSSL"
DruidMetadataPGUseSSLMode = "druid.metadata.postgres.ssl.sslMode"
DruidMetadataPGUseSSLModeVerifyFull = "verify-full"
DruidMetadataPGSSLCert = "druid.metadata.postgres.ssl.sslCert"
DruidMetadataPGSSLKey = "druid.metadata.postgres.ssl.sslKey"
DruidMetadataPGSSLRootCert = "druid.metadata.postgres.ssl.sslRootCert"

// Deep Storage
DruidDeepStorageTypeKey = "druid.storage.type"
Expand Down Expand Up @@ -1134,6 +1169,7 @@ const (
DruidExtensionBasicSecurity = "druid-basic-security"
DruidExtensionMultiStageQuery = "druid-multi-stage-query"
DruidExtensionPrometheusEmitter = "prometheus-emitter"
DruidExtensionSSLContext = "simple-client-sslcontext"
DruidService = "druid.service"

// Monitoring Configurations
Expand All @@ -1154,6 +1190,9 @@ const (
DruidMonitoringTaskCountStatsMonitor = "org.apache.druid.server.metrics.TaskCountStatsMonitor"
DruidMonitoringSysMonitor = "org.apache.druid.java.util.metrics.SysMonitor"

DruidDimensionMapDir = "/opt/druid/conf/metrics.json"
DruidEmitterPrometheusStrategyValue = "exporter"

/// Coordinators Configurations
DruidCoordinatorStartDelay = "druid.coordinator.startDelay"
DruidCoordinatorPeriod = "druid.coordinator.period"
Expand Down
141 changes: 110 additions & 31 deletions apis/kubedb/v1alpha2/druid_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package v1alpha2
import (
"context"
"fmt"
"path/filepath"
"strconv"
"strings"

Expand Down Expand Up @@ -125,14 +126,18 @@ func (d *Druid) ServiceLabels(alias ServiceAlias, extraLabels ...map[string]stri
return d.offShootLabels(meta_util.OverwriteKeys(d.OffShootSelectors(), extraLabels...), svcTemplate.Labels)
}

func (r *Druid) Finalizer() string {
return fmt.Sprintf("%s/%s", apis.Finalizer, r.ResourceSingular())
func (d *Druid) Finalizer() string {
return fmt.Sprintf("%s/%s", apis.Finalizer, d.ResourceSingular())
}

func (d *Druid) DefaultUserCredSecretName(username string) string {
return meta_util.NameWithSuffix(d.Name, strings.ReplaceAll(fmt.Sprintf("%s-cred", username), "_", "-"))
}

func (d *Druid) DruidSecretName(suffix string) string {
return strings.Join([]string{d.Name, suffix}, "-")
}

type DruidStatsService struct {
*Druid
}
Expand Down Expand Up @@ -204,21 +209,23 @@ func (d *Druid) PodControllerLabels(nodeType DruidNodeRoleType, extraLabels ...m
}

func (d *Druid) GetNodeSpec(nodeType DruidNodeRoleType) (*DruidNode, *DruidDataNode) {
if nodeType == DruidNodeRoleCoordinators {
switch nodeType {
case DruidNodeRoleCoordinators:
return d.Spec.Topology.Coordinators, nil
} else if nodeType == DruidNodeRoleOverlords {
case DruidNodeRoleOverlords:
return d.Spec.Topology.Overlords, nil
} else if nodeType == DruidNodeRoleMiddleManagers {
case DruidNodeRoleMiddleManagers:
return nil, d.Spec.Topology.MiddleManagers
} else if nodeType == DruidNodeRoleHistoricals {
case DruidNodeRoleHistoricals:
return nil, d.Spec.Topology.Historicals
} else if nodeType == DruidNodeRoleBrokers {
case DruidNodeRoleBrokers:
return d.Spec.Topology.Brokers, nil
} else if nodeType == DruidNodeRoleRouters {
case DruidNodeRoleRouters:
return d.Spec.Topology.Routers, nil
default:
klog.Errorf("unknown druid node role %s\n", nodeType)
return nil, nil
}

panic("Node role name does not match any known types")
}

func (d *Druid) ServiceAccountName() string {
Expand All @@ -235,19 +242,42 @@ func (d *Druid) DruidNodeRoleStringSingular(nodeRole DruidNodeRoleType) string {
}

func (d *Druid) DruidNodeContainerPort(nodeRole DruidNodeRoleType) int32 {
if nodeRole == DruidNodeRoleCoordinators {
return kubedb.DruidPortCoordinators
} else if nodeRole == DruidNodeRoleOverlords {
return kubedb.DruidPortOverlords
} else if nodeRole == DruidNodeRoleMiddleManagers {
return kubedb.DruidPortMiddleManagers
} else if nodeRole == DruidNodeRoleHistoricals {
return kubedb.DruidPortHistoricals
} else if nodeRole == DruidNodeRoleBrokers {
return kubedb.DruidPortBrokers
if !d.Spec.EnableSSL {
switch nodeRole {
case DruidNodeRoleCoordinators:
return kubedb.DruidPlainTextPortCoordinators
case DruidNodeRoleOverlords:
return kubedb.DruidPlainTextPortOverlords
case DruidNodeRoleMiddleManagers:
return kubedb.DruidPlainTextPortMiddleManagers
case DruidNodeRoleHistoricals:
return kubedb.DruidPlainTextPortHistoricals
case DruidNodeRoleBrokers:
return kubedb.DruidPlainTextPortBrokers
case DruidNodeRoleRouters:
return kubedb.DruidPlainTextPortRouters
default:
klog.Errorf("unknown druid node role %s\n", nodeRole)
}
} else {
switch nodeRole {
case DruidNodeRoleCoordinators:
return kubedb.DruidTLSPortCoordinators
case DruidNodeRoleOverlords:
return kubedb.DruidTLSPortOverlords
case DruidNodeRoleMiddleManagers:
return kubedb.DruidTLSPortMiddleManagers
case DruidNodeRoleHistoricals:
return kubedb.DruidTLSPortHistoricals
case DruidNodeRoleBrokers:
return kubedb.DruidTLSPortBrokers
case DruidNodeRoleRouters:
return kubedb.DruidTLSPortRouters
default:
klog.Errorf("unknown node role %s\n", nodeRole)
}
}
// Routers
return kubedb.DruidPortRouters
return -1
}

func (d *Druid) SetHealthCheckerDefaults() {
Expand Down Expand Up @@ -320,7 +350,8 @@ func (d *Druid) GetMetadataStorageType(metadataStorage string) DruidMetadataStor
metadataStorage == kubedb.DruidMetadataStorageTypePostgres || metadataStorage == strings.ToLower(string(kubedb.DruidMetadataStorageTypePostgres)) {
return DruidMetadataStoragePostgreSQL
} else {
panic(fmt.Sprintf("Unknown metadata storage type: %s", metadataStorage))
klog.Errorf("Unknown metadata storage type: %s", metadataStorage)
return ""
}
}

Expand Down Expand Up @@ -365,32 +396,55 @@ func (d *Druid) OffshootSelectors(extraSelectors ...map[string]string) map[strin
return meta_util.OverwriteKeys(selector, extraSelectors...)
}

func (d Druid) OffshootLabels() map[string]string {
func (d *Druid) OffshootLabels() map[string]string {
return d.offshootLabels(d.OffshootSelectors(), nil)
}

func (e Druid) offshootLabels(selector, override map[string]string) map[string]string {
func (d *Druid) offshootLabels(selector, override map[string]string) map[string]string {
selector[meta_util.ComponentLabelKey] = kubedb.ComponentDatabase
return meta_util.FilterKeys(kubedb.GroupName, selector, meta_util.OverwriteKeys(nil, e.Labels, override))
return meta_util.FilterKeys(kubedb.GroupName, selector, meta_util.OverwriteKeys(nil, d.Labels, override))
}

// CertificateName returns the default certificate name and/or certificate secret name for a certificate alias
func (d *Druid) CertificateName(alias DruidCertificateAlias) string {
return meta_util.NameWithSuffix(d.Name, fmt.Sprintf("%s-cert", string(alias)))
}

// GetCertSecretName returns the secret name for a certificate alias if any,
// otherwise returns default certificate secret name for the given alias.
func (d *Druid) GetCertSecretName(alias DruidCertificateAlias) string {
if d.Spec.TLS != nil {
name, ok := kmapi.GetCertificateSecretName(d.Spec.TLS.Certificates, string(alias))
if ok {
return name
}
}
return d.CertificateName(alias)
}

func (d *Druid) SetDefaults() {
if d.Spec.DeletionPolicy == "" {
d.Spec.DeletionPolicy = TerminationPolicyDelete
}

if d.Spec.DisableSecurity == nil {
d.Spec.DisableSecurity = pointer.BoolP(false)
}

if !*d.Spec.DisableSecurity {
if !d.Spec.DisableSecurity {
if d.Spec.AuthSecret == nil {
d.Spec.AuthSecret = &v1.LocalObjectReference{
Name: d.DefaultUserCredSecretName(kubedb.DruidUserAdmin),
}
}
}

if d.Spec.EnableSSL {
if d.Spec.KeystoreCredSecret == nil {
d.Spec.KeystoreCredSecret = &SecretReference{
LocalObjectReference: core.LocalObjectReference{
Name: d.DruidSecretName(kubedb.DruidKeystoreSecretKey),
},
}
}
}

var druidVersion catalog.DruidVersion
err := DefaultClient.Get(context.TODO(), types.NamespacedName{
Name: d.Spec.Version,
Expand Down Expand Up @@ -523,6 +577,18 @@ func (d *Druid) SetDefaults() {
}
d.Spec.Monitor.SetDefaults()
}

if d.Spec.EnableSSL {
d.SetTLSDefaults()
}
}

func (d *Druid) SetTLSDefaults() {
if d.Spec.TLS == nil || d.Spec.TLS.IssuerRef == nil {
return
}
d.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(d.Spec.TLS.Certificates, string(DruidServerCert), d.CertificateName(DruidServerCert))
d.Spec.TLS.Certificates = kmapi.SetMissingSecretNameForCertificate(d.Spec.TLS.Certificates, string(DruidClientCert), d.CertificateName(DruidClientCert))
}

func (d *Druid) SetDefaultsToMetadataStorage() {
Expand Down Expand Up @@ -725,3 +791,16 @@ func (d *Druid) GetZooKeeperName() string {
func (d *Druid) GetInitConfigMapName() string {
return d.OffShootName() + "-init-script"
}

// CertSecretVolumeName returns the CertSecretVolumeName
// Values will be like: client-certs, server-certs etc.
func (d *Druid) CertSecretVolumeName(alias DruidCertificateAlias) string {
return string(alias) + "-certs"
}

// CertSecretVolumeMountPath returns the CertSecretVolumeMountPath
// if configDir is "/var/druid/ssl",
// mountPath will be, "/var/druid/ssl/<alias>".
func (d *Druid) CertSecretVolumeMountPath(configDir string, cert string) string {
return filepath.Join(configDir, cert)
}
23 changes: 19 additions & 4 deletions apis/kubedb/v1alpha2/druid_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ type DruidSpec struct {
// disable security. It disables authentication security of user.
// If unset, default is false
// +optional
DisableSecurity *bool `json:"disableSecurity,omitempty"`
DisableSecurity bool `json:"disableSecurity,omitempty"`

// Database authentication secret
// +optional
Expand All @@ -79,9 +79,16 @@ type DruidSpec struct {
// +optional
ConfigSecret *core.LocalObjectReference `json:"configSecret,omitempty"`

//// TLS contains tls configurations
//// +optional
//TLS *kmapi.TLSConfig `json:"tls,omitempty"`
// To enable ssl for http layer
EnableSSL bool `json:"enableSSL,omitempty"`

// Keystore encryption secret
// +optional
KeystoreCredSecret *SecretReference `json:"keystoreCredSecret,omitempty"`

// TLS contains tls configurations
// +optional
TLS *kmapi.TLSConfig `json:"tls,omitempty"`

// MetadataStorage contains information for Druid to connect to external dependency metadata storage
// +optional
Expand Down Expand Up @@ -274,3 +281,11 @@ const (
DruidDeepStorageAzure DruidDeepStorageType = "azure"
DruidDeepStorageHDFS DruidDeepStorageType = "hdfs"
)

// +kubebuilder:validation:Enum=server;client
type DruidCertificateAlias string

const (
DruidServerCert DruidCertificateAlias = "server"
DruidClientCert DruidCertificateAlias = "client"
)
Loading

0 comments on commit 405c179

Please sign in to comment.