Merge pull request #1803 from bmichalkiewicz/gelf_output_plugin

Gelf output plugin: improved docs and add missing flags
kube-logging · Sep 26, 2024 · 922a9ec · 922a9ec
2 parents 7ed7801 + cd12738
commit 922a9ec
Show file tree

Hide file tree

Showing 14 changed files with 81 additions and 91 deletions.
diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_clusteroutputs.yaml
@@ -2685,6 +2685,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:
@@ -10084,6 +10086,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:

diff --git a/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml b/charts/logging-operator/crds/logging.banzaicloud.io_outputs.yaml
@@ -2681,6 +2681,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:
@@ -9354,6 +9356,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:

diff --git a/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml b/config/crd/bases/logging.banzaicloud.io_clusteroutputs.yaml
@@ -2685,6 +2685,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:
@@ -10084,6 +10086,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:

diff --git a/config/crd/bases/logging.banzaicloud.io_outputs.yaml b/config/crd/bases/logging.banzaicloud.io_outputs.yaml
@@ -2681,6 +2681,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:
@@ -9354,6 +9356,8 @@ spec:
                     type: object
                   host:
                     type: string
+                  max_bytes:
+                    type: integer
                   port:
                     type: integer
                   protocol:

diff --git a/docs/configuration/crds/v1beta1/output_types.md b/docs/configuration/crds/v1beta1/output_types.md
@@ -32,7 +32,7 @@ OutputSpec defines the desired state of Output
 ### gcs (*output.GCSOutput, optional) {#outputspec-gcs}
 
 
-### gelf (*output.GELFOutputConfig, optional) {#outputspec-gelf}
+### gelf (*output.GelfOutputConfig, optional) {#outputspec-gelf}
 
 
 ### http (*output.HTTPOutputConfig, optional) {#outputspec-http}

diff --git a/docs/configuration/plugins/_index.md b/docs/configuration/plugins/_index.md
@@ -38,7 +38,7 @@ For more information please click on the plugin name
 | **[Format rfc5424](outputs/format_rfc5424/)** | outputs | Specify how to format output record. | GA | [more info](https://github.com/cloudfoundry/fluent-plugin-syslog_rfc5424#format-section) |
 | **[Forward](outputs/forward/)** | outputs | Forwards events to other fluentd nodes. | GA | [more info](https://docs.fluentd.org/output/forward) |
 | **[Google Cloud Storage](outputs/gcs/)** | outputs | Store logs in Google Cloud Storage | GA | [0.4.0](https://github.com/kube-logging/fluent-plugin-gcs) |
-| **[Gelf](outputs/gelf/)** | outputs | Output plugin writes events to GELF | Testing | [1.0.8](https://github.com/hotschedules/fluent-plugin-gelf-hs) |
+| **[Gelf](outputs/gelf/)** | outputs | Output plugin writes logs to Graylog | Testing | [1.3.4](https://github.com/bmichalkiewicz/fluent-plugin-gelf-best) |
 | **[Http](outputs/http/)** | outputs | Sends logs to HTTP/HTTPS endpoints. | GA | [more info](https://docs.fluentd.org/output/http) |
 | **[Kafka](outputs/kafka/)** | outputs | Send your logs to Kafka | GA | [0.17.5](https://github.com/fluent/fluent-plugin-kafka/releases/tag/v0.17.5) |
 | **[Amazon Kinesis Firehose](outputs/kinesis_firehose/)** | outputs | Fluent plugin for Amazon Kinesis | Testing | [3.4.2](https://github.com/awslabs/aws-fluent-plugin-kinesis/releases/tag/v3.4.2) |

diff --git a/docs/configuration/plugins/outputs/gelf.md b/docs/configuration/plugins/outputs/gelf.md
@@ -1,12 +1,22 @@
 ---
-title: GELF
+title: Gelf
 weight: 200
 generated_file: true
 ---
 
-# [GELF Output](https://github.com/hotschedules/fluent-plugin-gelf-hs)
+# Gelf output plugin for Fluentd
 ## Overview
- Fluentd output plugin for GELF.
+
+For details, see [https://github.com/bmichalkiewicz/fluent-plugin-gelf-best](https://github.com/bmichalkiewicz/fluent-plugin-gelf-best).
+
+## Example
+```yaml
+spec:
+  gelf:
+    host: gelf-host
+    port: 12201
+```
+
 
 ## Configuration
 ## Output Config
@@ -21,6 +31,12 @@ Available since ghcr.io/kube-logging/fluentd:v1.16-4.8-full [Buffer](../buffer/)
 Destination host 
 
 
+### max_bytes (int, optional) {#output config-max_bytes}
+
+MaxBytes specifies the maximum size, in bytes, of each individual log message. For details, see [https://github.com/Graylog2/graylog2-server/issues/873](https://github.com/Graylog2/graylog2-server/issues/873) Available since ghcr.io/kube-logging/fluentd:v1.16-4.10-full
+
+Default: 3200
+
 ### port (int, required) {#output config-port}
 
 Destination host port 
@@ -34,42 +50,14 @@ Default: "udp"
 
 ### tls (*bool, optional) {#output config-tls}
 
-Enable TlS
+Enable TLS
 
 Default: false
 
 ### tls_options (map[string]string, optional) {#output config-tls_options}
 
-TLS options .
-
-Default: {}). For details, see [https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12](https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12
+TLS Options. For details, see [https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12](https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12).
 
+Default: {}
 
 
-
-## Example `GELF` output configurations
-
-{{< highlight yaml >}}
-apiVersion: logging.banzaicloud.io/v1beta1
-kind: Output
-metadata:
-  name: gelf-output-sample
-spec:
-  gelf:
-    host: gelf-host
-    port: 12201
-{{</ highlight >}}
-
-Fluentd config result:
-
-{{< highlight xml >}}
-<match **>
-	@type gelf
-	@id test_gelf
-	host gelf-host
-	port 12201
-</match>
-{{</ highlight >}}
-
-
----
diff --git a/pkg/sdk/logging/api/v1alpha1/output_types.go b/pkg/sdk/logging/api/v1alpha1/output_types.go
@@ -54,7 +54,7 @@ type OutputSpec struct {
 	AwsElasticsearchOutputConfig *output.AwsElasticsearchOutputConfig `json:"awsElasticsearch,omitempty"`
 	RedisOutputConfig            *output.RedisOutputConfig            `json:"redis,omitempty"`
 	SyslogOutputConfig           *output.SyslogOutputConfig           `json:"syslog,omitempty"`
-	GELFOutputConfig             *output.GELFOutputConfig             `json:"gelf,omitempty"`
+	GelfOutputConfig             *output.GelfOutputConfig             `json:"gelf,omitempty"`
 	SQSOutputConfig              *output.SQSOutputConfig              `json:"sqs,omitempty"`
 	RelabelOutputConfig          *output.RelabelOutputConfig          `json:"relabel,omitempty"`
 }

diff --git a/pkg/sdk/logging/api/v1alpha1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/sdk/logging/api/v1beta1/output_types.go b/pkg/sdk/logging/api/v1beta1/output_types.go
@@ -56,7 +56,7 @@ type OutputSpec struct {
 	AwsElasticsearchOutputConfig      *output.AwsElasticsearchOutputConfig      `json:"awsElasticsearch,omitempty"`
 	RedisOutputConfig                 *output.RedisOutputConfig                 `json:"redis,omitempty"`
 	SyslogOutputConfig                *output.SyslogOutputConfig                `json:"syslog,omitempty"`
-	GELFOutputConfig                  *output.GELFOutputConfig                  `json:"gelf,omitempty"`
+	GelfOutputConfig                  *output.GelfOutputConfig                  `json:"gelf,omitempty"`
 	SQSOutputConfig                   *output.SQSOutputConfig                   `json:"sqs,omitempty"`
 	MattermostOutputConfig            *output.MattermostOutputConfig            `json:"mattermost,omitempty"`
 	RelabelOutputConfig               *output.RelabelOutputConfig               `json:"relabel,omitempty"`

diff --git a/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go b/pkg/sdk/logging/api/v1beta1/zz_generated.deepcopy.go
diff --git a/pkg/sdk/logging/model/output/gelf.go b/pkg/sdk/logging/model/output/gelf.go
@@ -20,69 +20,55 @@ import (
 	"github.com/kube-logging/logging-operator/pkg/sdk/logging/model/types"
 )
 
-// +name:"GELF"
+// +name:"Gelf"
 // +weight:"200"
-type _hugoGELF interface{} //nolint:deadcode,unused
+type _hugoGelf interface{} //nolint:deadcode,unused
 
-// +kubebuilder:object:generate=true
-// +docName:"[GELF Output](https://github.com/hotschedules/fluent-plugin-gelf-hs)"
-// Fluentd output plugin for GELF.
-type _docGELF interface{} //nolint:deadcode,unused
+// +docName:"Gelf output plugin for Fluentd"
+/*
+For details, see [https://github.com/bmichalkiewicz/fluent-plugin-gelf-best](https://github.com/bmichalkiewicz/fluent-plugin-gelf-best).
+
+## Example
+```yaml
+spec:
+  gelf:
+    host: gelf-host
+    port: 12201
+```
+*/
+type _docGelf interface{} //nolint:deadcode,unused
 
 // +name:"Gelf"
-// +url:"https://github.com/hotschedules/fluent-plugin-gelf-hs"
-// +version:"1.0.8"
-// +description:"Output plugin writes events to GELF"
+// +url:"https://github.com/bmichalkiewicz/fluent-plugin-gelf-best"
+// +version:"1.3.4"
+// +description:"Output plugin writes logs to Graylog"
 // +status:"Testing"
 type _metaGelf interface{} //nolint:deadcode,unused
 
 // +kubebuilder:object:generate=true
 // +docName:"Output Config"
-type GELFOutputConfig struct {
+type GelfOutputConfig struct {
 	// Destination host
 	Host string `json:"host"`
 	// Destination host port
 	Port int `json:"port"`
 	// Transport Protocol (default: "udp")
 	Protocol string `json:"protocol,omitempty"`
-	// Enable TlS (default: false)
+	// Enable TLS (default: false)
 	TLS *bool `json:"tls,omitempty"`
-	// TLS options (default: {}). For details, see [https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12](https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12).
+	// TLS Options.
+	// For details, see [https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12](https://github.com/graylog-labs/gelf-rb/blob/72916932b789f7a6768c3cdd6ab69a3c942dbcef/lib/gelf/transport/tcp_tls.rb#L7-L12). (default: {})
 	TLSOptions map[string]string `json:"tls_options,omitempty"`
+	// MaxBytes specifies the maximum size, in bytes, of each individual log message.
+	// For details, see [https://github.com/Graylog2/graylog2-server/issues/873](https://github.com/Graylog2/graylog2-server/issues/873)
+	// Available since ghcr.io/kube-logging/fluentd:v1.16-4.10-full (default: 3200)
+	MaxBytes int `json:"max_bytes,omitempty"`
 	// Available since ghcr.io/kube-logging/fluentd:v1.16-4.8-full
 	// +docLink:"Buffer,../buffer/"
 	Buffer *Buffer `json:"buffer,omitempty"`
 }
 
-//
-/*
-## Example `GELF` output configurations
-
-{{< highlight yaml >}}
-apiVersion: logging.banzaicloud.io/v1beta1
-kind: Output
-metadata:
-  name: gelf-output-sample
-spec:
-  gelf:
-    host: gelf-host
-    port: 12201
-{{</ highlight >}}
-
-Fluentd config result:
-
-{{< highlight xml >}}
-<match **>
-	@type gelf
-	@id test_gelf
-	host gelf-host
-	port 12201
-</match>
-{{</ highlight >}}
-*/
-type _expGELF interface{} //nolint:deadcode,unused
-
-func (s *GELFOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
+func (s *GelfOutputConfig) ToDirective(secretLoader secret.SecretLoader, id string) (types.Directive, error) {
 	pluginType := "gelf"
 	gelf := &types.OutputPlugin{
 		PluginMeta: types.PluginMeta{

diff --git a/pkg/sdk/logging/model/output/gelf_test.go b/pkg/sdk/logging/model/output/gelf_test.go
@@ -23,7 +23,7 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-func TestGELFOutputConfig(t *testing.T) {
+func TestGelfOutputConfig(t *testing.T) {
 	CONFIG := []byte(`
 host: gelf-host
 port: 12201
@@ -43,7 +43,7 @@ port: 12201
     </buffer>
   </match>
 `
-	s := &output.GELFOutputConfig{}
+	s := &output.GelfOutputConfig{}
 	require.NoError(t, yaml.Unmarshal(CONFIG, s))
 	test := render.NewOutputPluginTest(t, s)
 	test.DiffResult(expected)

diff --git a/pkg/sdk/logging/model/output/zz_generated.deepcopy.go b/pkg/sdk/logging/model/output/zz_generated.deepcopy.go