doc/userguide: generate eve documentation

Add EVE documentation for QUIC and Pgsql to their respective sections of
the userguide.

Also add a complete EVE reference as an appendix.

Other protocols can be done, but its a manual process to document in the
schema, then add the glue to pull them into the documentation.

The documentation is generated during "make dist", or if it doesn't
exist, "conf.py" will attempt to generate the eve documentation for
building on Readthedocs.

Makefile.am

@@ -10,6 +10,7 @@ EXTRA_DIST = ChangeLog COPYING LICENSE suricata.yaml.in \
 	     scripts/generate-images.sh \
 	     scripts/docs-almalinux9-minimal-build.sh \
 	     scripts/docs-ubuntu-debian-minimal-build.sh \
+       scripts/evedoc.py \
 	     examples/plugins
 SUBDIRS = $(HTP_DIR) rust src plugins qa rules doc contrib etc python ebpf \
           $(SURICATA_UPDATE_DIR)

doc/userguide/.gitignore

@@ -1 +1,2 @@
 _build
+_generated

doc/userguide/Makefile.am

@@ -1,7 +1,9 @@
 EXTRA_DIST = \
+	_generated \
 	_static \
 	3rd-party-integration \
 	acknowledgements.rst \
+	appendix \
 	capture-hardware \
 	command-line-options.rst \
 	conf.py \
@@ -47,14 +49,14 @@ endif
 
 SPHINX_BUILD = sphinx-build -q
 
-html:
+html: _generated
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
 	version=$(PACKAGE_VERSION) \
 		$(SPHINX_BUILD) -W -b html -d _build/doctrees \
 		$(top_srcdir)/doc/userguide _build/html
 
-_build/latex/Suricata.pdf:
+_build/latex/Suricata.pdf: _generated
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
 	version=$(PACKAGE_VERSION) \
@@ -74,7 +76,7 @@ userguide.pdf: _build/latex/Suricata.pdf
 
 pdf: userguide.pdf
 
-_build/man: manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
+_build/man: _generated manpages/suricata.rst manpages/suricatasc.rst manpages/suricatactl.rst manpages/suricatactl-filestore.rst
 	RELEASE_DATE=$(RELEASE_DATE) \
 	sysconfdir=$(sysconfdir) \
 	localstatedir=$(localstatedir) \
@@ -95,3 +97,6 @@ clean-local:
 	rm -f $(top_builddir)/doc/userguide/userguide.pdf
 
 endif # SPHINX_BUILD
+
+_generated:
+	./generate-evedoc.sh

doc/userguide/appendix/eve-index.rst

@@ -0,0 +1,7 @@
+EVE Index
+=========
+
+.. toctree::
+   :maxdepth: 1
+
+.. include:: ../_generated/eve-index.rst

doc/userguide/appendix/index.rst

@@ -0,0 +1,7 @@
+Appendix
+========
+
+.. toctree::
+   :maxdepth: 1
+
+   eve-index

doc/userguide/conf.py

@@ -151,6 +151,11 @@ def setup(app):
     else:
         app.add_stylesheet('css/suricata.css')
 
+    # Build generated pages if they don't exist. For example, when on
+    # RTD and we're build from git instead of a distribution package.
+    if not os.path.exists("./_generated"):
+        os.system("./generate-evedoc.sh")
+
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
@@ -338,3 +343,11 @@ def setup(app):
     "sysconfdir": os.getenv("sysconfdir", "/etc"),
     "localstatedir": os.getenv("localstatedir", "/var"),
 }
+
+# Custom code generate some documentation.
+# evedoc = "./evedoc.py"
+# eve_schema = "../../etc/schema.json"
+# os.makedirs("_generated", exist_ok=True)
+# subprocess.call([evedoc, "--output", "_generated/eve-index.rst", eve_schema])
+# for proto in ["quic", "pgsql"]:
+#     subprocess.call([evedoc, "--output", "_generated/{}.rst".format(proto), "--object", proto, eve_schema])

doc/userguide/generate-evedoc.sh

@@ -0,0 +1,13 @@
+#! /bin/sh
+#
+# Generate RST EVE documentation.
+#
+# This has been broken out of the Makefile so it can be called by
+# make, and Sphinx via conf.py.
+
+set -e
+
+mkdir -p _generated
+../../scripts/evedoc.py --output _generated/eve-index.rst ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/quic.rst --object quic ../../etc/schema.json
+../../scripts/evedoc.py --output _generated/pgsql.rst --object pgsql ../../etc/schema.json

doc/userguide/index.rst

@@ -34,4 +34,5 @@ This is the documentation for Suricata |version|.
    acknowledgements
    licenses/index.rst
    devguide/index.rst
-   verifying-source-files
+   verifying-source-files
+   appendix/index.rst

doc/userguide/output/eve/eve-json-format.rst

@@ -2682,6 +2682,10 @@ References:
 .. _PostgreSQL message format - BackendKeyData: https://www.postgresql.org/docs
    /current/protocol-message-formats.html#PROTOCOL-MESSAGE-FORMATS-BACKENDKEYDATA
 
+Field Reference
+~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/pgsql.rst
 
 Event type: IKE
 ---------------
@@ -2937,6 +2941,11 @@ Example of QUIC logging with CYU, JA3 and JA4 hashes (note that the JA4 hash is
     "ja4": "q13d0310h3_55b375c5d22e_cd85d2d88918"
   }
 
+Output Reference
+~~~~~~~~~~~~~~~~
+
+.. include:: ../../_generated/quic.rst
+
 Event type: DHCP
 -----------------
 
@@ -3048,4 +3057,3 @@ Example of ARP logging: request and response
     "dest_mac": "00:1d:09:f0:92:ab",
     "dest_ip": "10.10.10.1"
   }
-

etc/Makefile.am

@@ -1,6 +1,7 @@
 etcdatadir =	$(datadir)/suricata
 
-EXTRA_DIST =	suricata.logrotate.in \
+EXTRA_DIST =	schema.json \
+		suricata.logrotate.in \
 		suricata.service.in
 
 dist_etcdata_DATA =	classification.config \