diff --git a/src/ctia/entity/relationship.clj b/src/ctia/entity/relationship.clj index 9c692bf2a..647220f1f 100644 --- a/src/ctia/entity/relationship.clj +++ b/src/ctia/entity/relationship.clj @@ -18,6 +18,11 @@ [schema-tools.core :as st] [schema.core :as s])) +(def node-type + (assoc em/token + :fields {:type {:type "text" + :analyzer "type_analyzer"}})) + (def relationship-mapping {"relationship" {:dynamic false @@ -28,8 +33,8 @@ em/sourcable-entity-mapping em/stored-entity-mapping {:relationship_type em/token - :source_ref em/token - :target_ref em/token})}}) + :source_ref node-type + :target_ref node-type})}}) (def-es-store RelationshipStore :relationship diff --git a/src/ctia/stores/es/init.clj b/src/ctia/stores/es/init.clj index fa4cfaecf..5605b6b7d 100644 --- a/src/ctia/stores/es/init.clj +++ b/src/ctia/stores/es/init.clj @@ -47,8 +47,9 @@ settings {:refresh_interval refresh_interval :number_of_shards shards :number_of_replicas replicas} - mappings (cond-> (get-in entity-fields [entity :es-mapping] mappings) - (< 5 version) (some-> first val)) + mappings (some-> (get-in entity-fields [entity :es-mapping] mappings) + first + val) searchable-fields (get-in entity-fields [entity :searchable-fields])] {:index indexname :props (assoc props :write-index write-index) @@ -89,21 +90,17 @@ (s/defn update-mappings! [{:keys [conn index] {:keys [mappings]} :config} :- ESConnState] - (let [[entity-type type-mappings] (when (= (:version conn) 5) - (first mappings)) - update-body (or type-mappings mappings)] (try (log/info "updating mapping: " index) (index/update-mappings! conn index - entity-type - update-body) + mappings) (catch clojure.lang.ExceptionInfo e (log/error "cannot update mapping. You probably tried to update the mapping of an existing field. It's only possible to add new field to existing mappings. If you need to modify the type of a field in an existing index, you must perform a migration" (assoc (ex-data e) :conn conn :mappings mappings)) - (system-exit-error))))) + (system-exit-error)))) (s/defn refresh-mappings! [{:keys [conn index] diff --git a/src/ctia/stores/es/mapping.clj b/src/ctia/stores/es/mapping.clj index bce375004..b31877587 100644 --- a/src/ctia/stores/es/mapping.clj +++ b/src/ctia/stores/es/mapping.clj @@ -1,4 +1,5 @@ (ns ctia.stores.es.mapping + (:require [clojure.string :as string]) (:refer-clojure :exclude [identity])) ;; This provides a reasonable default mapping for all of our entities. @@ -260,6 +261,34 @@ {:properties {:type token :text text}}) +(def type-simple-pattern + (->> [:actor + "asset([-](mapping|properties))?" + :attack-pattern + :campaign + :casebook + :coa + :data-table + :event + :feed + :feedback + :identity-assertion + :identity + :incident + :indicator + :investigation + :judgement + :malware + :note + :relationship + :sighting + :target-record + :tool + :vulnerability + :weakness] + (map (comp #(string/replace % "-" "\\-") name)) + (string/join "|"))) + (def store-settings {:number_of_replicas 1 :number_of_shards 1 @@ -284,6 +313,10 @@ :english_stemmer {:type "stemmer" :language "english"}} ;; when applying filters, order matters + :tokenizer + {:type_tokenizer + {:type "simple_pattern", + :pattern type-simple-pattern}} :analyzer {:default ;; same as text_analyzer {:type "custom" @@ -298,6 +331,10 @@ :filter ["lowercase" "ctia_stemmer" "english_stemmer"]} + :type_analyzer { + :tokenizer "type_tokenizer" + :filter [ "fingerprint"] + } :search_analyzer {:type "custom" :tokenizer "standard" diff --git a/test/ctia/entity/relationship_test.clj b/test/ctia/entity/relationship_test.clj index 7e3ec0c72..063a8fc8b 100644 --- a/test/ctia/entity/relationship_test.clj +++ b/test/ctia/entity/relationship_test.clj @@ -6,7 +6,7 @@ [ctia.test-helpers [access-control :refer [access-control-test]] [auth :refer [all-capabilities]] - [core :as helpers :refer [POST]] + [core :as helpers :refer [POST GET]] [crud :refer [entity-crud-test]] [aggregate :refer [test-metric-routes]] [fake-whoami-service :as whoami-helpers] @@ -43,6 +43,75 @@ "http://ex.tld/ctia/relationship/relationship-456"]) (dissoc :id))) +(deftest test-relationship-source-target-type-search + (test-for-each-store-with-app + (fn [app] + (establish-user! app) + (testing "POST /ctia/relationship" + (let [new-relationship-1 + (-> new-relationship-minimal + (assoc + :source_ref (str "http://example.com/ctia/judgement/judgement-" + "f9832ac2-ee90-4e18-9ce6-0c4e4ff61a7a") + :target_ref (str "http://example.com/ctia/indicator/indicator-" + "8c94ca8d-fb2b-4556-8517-8e6923d8d3c7")) + (dissoc :id)) + new-relationship-2 (update new-relationship-1 + :source_ref + str/replace + "judgement" + "sighting") + new-relationship-3 (-> new-relationship-1 + (update :source_ref + str/replace + "judgement" + "asset") + (update :target_ref + str/replace + "indicator" + "asset-properties")) + {status-1 :status rel1 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-1 + :headers {"Authorization" "45c1f5e3f05d0"}) + {status-2 :status rel2 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-2 + :headers {"Authorization" "45c1f5e3f05d0"}) + {status-3 :status rel3 :parsed-body} + (POST app + "ctia/relationship" + :body new-relationship-3 + :headers {"Authorization" "45c1f5e3f05d0"}) + test-plan [{:expected [rel1 rel2] + :query "target_ref.type=indicator"} + {:expected [rel1] + :query "source_ref.type=judgement"} + {:expected [rel2] + :query "source_ref.type=sighting"} + {:expected [rel3] + :query "source_ref.type=asset"} + {:expected [rel3] + :query "target_ref.type=\"asset-properties\""} + {:expected [] + :query "target_ref.type=asset"} + {:expected [] + :query "source_ref.type=malware"}]] + (assert (= 201 status-1)) + (assert (= 201 status-2)) + (assert (= 201 status-3)) + (doseq [{:keys [expected query]} test-plan] + (testing query + (is (= (set expected) + (-> (GET app + "ctia/relationship/search" + :query-params {:query query} + :headers {"Authorization" "45c1f5e3f05d0"}) + :parsed-body + set)))))))))) + (deftest test-relationship-routes-bad-reference (test-for-each-store-with-app (fn [app]