Issue #91: Add improved event validator API (#94)

* Issue #91: Add improved event validator API

* Change observable type_id caption so they match up better during validation of observables. Add (limited) observable validation.

* Change validating array of elements from /api/v2/validate to use of event bundle with /api/v2/validate_bundle. Remove try/rescue pattern from schema_controller.ex as it returns exception message, leaking internal implementation details (a security problem). Improve Swagger docs for new APIs.

* Dependency updates

Dockerfile

@@ -1,4 +1,6 @@
-FROM elixir:1.16.2-alpine as builder
+ARG elixir_image=elixir:1.17.0-alpine
+
+FROM ${elixir_image} as builder
 
 # prepare build dir
 WORKDIR /app
@@ -38,7 +40,7 @@ RUN mix release
 
 # start a new build stage so that the final image will only contain
 # the compiled release and other runtime necessities
-FROM elixir:1.16.2-alpine
+FROM ${elixir_image}
 
 # Set the locale
 # RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen

config/config.exs

@@ -15,7 +15,7 @@ config :logger, :console,
   handle_otp_reports: true,
   handle_sasl_reports: true,
   format: "$date $time [$level] $metadata $message\n",
-  metadata: [:request_id]
+  metadata: [:request_id, :mfa, :line]
 
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason

lib/schema.ex

@@ -556,12 +556,44 @@ defmodule Schema do
   defp reduce_attributes(data) do
     reduce_data(data)
     |> Map.update(:attributes, [], fn attributes ->
-      Enum.into(attributes, %{}, fn {name, attribute} ->
-        {name, reduce_data(attribute)}
+      Enum.into(attributes, %{}, fn {attribute_name, attribute_details} ->
+        {attribute_name, reduce_attribute(attribute_details)}
       end)
     end)
   end
 
+  defp reduce_attribute(attribute_details) do
+    attribute_details
+    |> filter_internal()
+    |> reduce_enum()
+  end
+
+  defp filter_internal(m) do
+    Map.filter(m, fn {key, _} ->
+      s = Atom.to_string(key)
+      not String.starts_with?(s, "_")
+    end)
+  end
+
+  defp reduce_enum(attribute_details) do
+    if Map.has_key?(attribute_details, :enum) do
+      Map.update!(attribute_details, :enum, fn enum ->
+        Enum.map(
+          enum,
+          fn {enum_value_key, enum_value_details} ->
+            {
+              enum_value_key,
+              filter_internal(enum_value_details)
+            }
+          end
+        )
+        |> Enum.into(%{})
+      end)
+    else
+      attribute_details
+    end
+  end
+
   @spec reduce_class(map) :: map
   def reduce_class(data) do
     delete_attributes(data) |> delete_associations()

lib/schema/cache.ex

@@ -492,14 +492,16 @@ defmodule Schema.Cache do
 
               observable_type_id_map
             else
+              observable_kind = "#{kind}-Specific Attribute"
+
               Map.put(
                 observable_type_id_map,
                 observable_type_id,
-                %{
-                  caption: "#{caption} #{kind}: #{attribute_key} (#{kind}-Specific Attribute)",
-                  description:
-                    "#{kind}-specific attribute \"#{attribute_key}\" for the #{caption} #{kind}."
-                }
+                make_observable_enum_entry(
+                  "#{caption} #{kind}: #{attribute_key}",
+                  "#{kind}-specific attribute \"#{attribute_key}\" for the #{caption} #{kind}.",
+                  observable_kind
+                )
               )
             end
           else
@@ -534,16 +536,16 @@ defmodule Schema.Cache do
 
             observable_type_id_map
           else
+            observable_kind = "#{kind}-Specific Attribute"
+
             Map.put(
               observable_type_id_map,
               observable_type_id,
-              %{
-                caption:
-                  "#{caption} #{kind}: #{attribute_path} (#{kind}-Specific Attribute Path)",
-                description:
-                  "#{kind}-specific attribute on path \"#{attribute_path}\"" <>
-                    " for the #{caption} #{kind}."
-              }
+              make_observable_enum_entry(
+                "#{caption} #{kind}: #{attribute_path}",
+                "#{kind}-specific attribute \"#{attribute_path}\" for the #{caption} #{kind}.",
+                observable_kind
+              )
             )
           end
         end
@@ -636,7 +638,7 @@ defmodule Schema.Cache do
         Map.put(
           observable_type_id_map,
           observable_type_id,
-          %{caption: "#{caption} (Object)", description: description}
+          make_observable_enum_entry(caption, description, "Object")
         )
       end
     else
@@ -673,10 +675,7 @@ defmodule Schema.Cache do
               Map.put(
                 observable_type_id_map,
                 observable_type_id,
-                %{
-                  caption: "#{item[:caption]} (#{kind})",
-                  description: item[:description]
-                }
+                make_observable_enum_entry(item[:caption], item[:description], kind)
               )
             end
           else
@@ -689,6 +688,16 @@ defmodule Schema.Cache do
     end
   end
 
+  # make an observable type_id enum entry
+  @spec make_observable_enum_entry(String.t(), String.t(), String.t()) :: map()
+  defp make_observable_enum_entry(caption, description, observable_kind) do
+    %{
+      caption: caption,
+      description: "Observable by #{observable_kind}.<br>#{description}",
+      _observable_kind: observable_kind
+    }
+  end
+
   @spec find_item_caption_and_description(map(), atom(), map() | nil) :: {String.t(), String.t()}
   defp find_item_caption_and_description(items, item_key, item)
        when is_map(items) and is_atom(item_key) do

lib/schema/inspector.ex → lib/schema/validator.ex

@@ -7,9 +7,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-defmodule Schema.Inspector do
+defmodule Schema.Validator do
   @moduledoc """
-  OCSF Event data inspector.
+  OCSF Event validator.
   """
 
   require Logger