Merge pull request #483 from InQuest/issue/472-enable-match-type-bulk…

…-edit-import

Issue/472 enable match type bulk edit import

app/routes/batch.py

@@ -37,6 +37,8 @@ def batch_update(batch, artifact, session, include_tags=True):
     fields_to_update = dict()
     if 'description' in batch and batch['description']:
         fields_to_update['description'] = batch['description']
+    if 'match_type' in batch and batch['match_type']:
+        fields_to_update['match_type'] = batch['match_type']
     if 'expiration_timestamp' in batch and batch['expiration_timestamp'] and hasattr(artifact, 'expiration_timestamp'):
         fields_to_update['expiration_timestamp'] = batch['expiration_timestamp']
     if 'category' in batch and batch['category'] and hasattr(artifact, 'category'):

app/routes/c2dns.py

@@ -234,6 +234,7 @@ def batch_update_c2dns():
     """Batch update c2dns artifacts
     From Data: batch {
                  state (str),
+                 match_type (str),
                  owner_user (str),
                  tags (array),
                  ids (array)

app/routes/import_.py

@@ -13,7 +13,7 @@
 #####################################################################
 
 def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared_reference=None,
-                   shared_description=None, shared_state=None, shared_owner=None, metadata_field_mapping={},
+                   shared_description=None, shared_state=None, shared_match_type=None, shared_owner=None, metadata_field_mapping={},
                    resurrect_retired_artifacts=True):
     default_state = "Imported"
     return_artifacts = []
@@ -134,6 +134,7 @@ def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared
                                               entity_id=dns.id, user_id=current_user.id))
                     dns.created_user_id, dns.modified_user_id = current_user.id, current_user.id
                     dns.state = default_state if not shared_state else shared_state
+                    dns.match_type = None if not shared_match_type else shared_match_type
                     if Whitelist.hits_whitelist(dns.domain_name, dns.state):
                         error_artifacts.append((dns.domain_name, f"Whitelist validation failed {dns.domain_name}"))
                         continue
@@ -144,6 +145,8 @@ def save_artifacts(extract_ip, extract_dns, extract_signature, artifacts, shared
                         dns.description = shared_description
                     if shared_state:
                         dns.state = shared_state
+                    if shared_match_type:
+                        dns.match_type = shared_match_type
                     if shared_owner:
                         dns.owner_user_id = shared_owner
 
@@ -239,6 +242,7 @@ def import_artifacts():
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     import_text = request.json.get('import_text', None)
     shared_state = request.json.get('shared_state', None)
+    shared_match_type = request.json.get('shared_match_type', None)
     shared_reference = request.json.get("shared_reference", None)
     shared_description = request.json.get("shared_description", None)
     shared_owner = request.json.get("shared_owner", None)
@@ -259,7 +263,7 @@ def import_artifacts():
     if autocommit:
         artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                    artifacts=artifacts, shared_reference=shared_reference,
-                                   shared_description=shared_description, shared_state=shared_state,
+                                   shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                    shared_owner=shared_owner, metadata_field_mapping=metadata_field_mapping,
                                    resurrect_retired_artifacts=resurrect_retired_artifacts)
 
@@ -280,6 +284,7 @@ def import_artifacts_by_filek():
     import_text = request.files['file'].stream.read()
     import_text = import_text.strip()
     shared_state = request.values.get('shared_state', None)
+    shared_match_type = request.values.get('shared_match_type', None)
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     shared_reference = request.values.get("shared_reference", None) or None
     shared_description = request.values.get("shared_description", None) or None
@@ -301,7 +306,7 @@ def import_artifacts_by_filek():
     if autocommit:
         artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                    artifacts=artifacts, shared_reference=shared_reference,
-                                   shared_description=shared_description, shared_state=shared_state,
+                                   shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                    shared_owner=shared_owner, metadata_field_mapping=metadata_field_mapping,
                                    resurrect_retired_artifacts=resurrect_retired_artifacts)
 
@@ -322,6 +327,7 @@ def commit_artifacts():
     shared_description = request.json.get("shared_description", None)
     resurrect_retired_artifacts = request.json.get("resurrect_retired_artifacts", True)
     shared_state = request.json.get('shared_state', None)
+    shared_match_type = request.json.get('shared_match_type', None)
     extract_ip = request.json.get('extract_ip', True)
     extract_dns = request.json.get('extract_dns', True)
     shared_owner = request.json.get("shared_owner", None)
@@ -336,7 +342,7 @@ def commit_artifacts():
 
     artifacts = save_artifacts(extract_ip=extract_ip, extract_dns=extract_dns, extract_signature=extract_signature,
                                artifacts=artifacts, shared_reference=shared_reference,
-                               shared_description=shared_description, shared_state=shared_state,
+                               shared_description=shared_description, shared_state=shared_state, shared_match_type=shared_match_type,
                                metadata_field_mapping=metadata_field_mapping, shared_owner=shared_owner,
                                resurrect_retired_artifacts=resurrect_retired_artifacts)
     return jsonify({"artifacts": artifacts}), 201

app/static/js/c2dns/c2dns-controller.js

@@ -420,6 +420,7 @@ angular.module('ThreatKB')
                 var c2dnsToUpdate = {
                     owner_user: $scope.batch.owner,
                     state: $scope.batch.state,
+                    match_type: $scope.batch.match_type,
                     description: $scope.batch.description,
                     expiration_timestamp: $scope.batch.expiration_timestamp,
                     tags: $scope.batch.tags,
@@ -505,6 +506,7 @@ angular.module('ThreatKB')
                 $scope.batch = {
                     owner: null,
                     state: null,
+                    match_type: null,
                     description: null,
                     expiration_timestamp: null,
                     tags: null
@@ -829,6 +831,8 @@ angular.module('ThreatKB')
 
             $scope.cfg_states = Cfg_states.query();
 
+            $scope.match_types = ['exact', 'wildcard'];
+
             $scope.ok = function () {
                 $uibModalInstance.close($scope.batch);
             };

app/static/js/import/import-controller.js

@@ -6,6 +6,8 @@ angular.module('ThreatKB').controller('ImportController',
 
             $scope.cfg_states = Cfg_states.query();
             $scope.shared_state = {};
+            $scope.shared_match_type = {};
+            $scope.match_types = ['exact', 'wildcard'];
             $scope.shared_owner = null;
             $scope.users = Users.query();
             $scope.default_mapping = Cfg_settings.get({key: "DEFAULT_METADATA_MAPPING"});
@@ -39,6 +41,7 @@ angular.module('ThreatKB').controller('ImportController',
                                     shared_reference: $scope.shared_reference,
                                     shared_description: $scope.shared_description,
                                     shared_state: $scope.shared_state,
+                                    shared_match_type: $scope.shared_match_type,
                                     shared_owner: $scope.shared_owner,
                                     extract_ip: $scope.extract_ip,
                                     extract_dns: $scope.extract_dns,
@@ -89,6 +92,9 @@ angular.module('ThreatKB').controller('ImportController',
                     $scope.shared_state.state = {};
                 }
 
+                if ($scope.shared_match_type === undefined) {
+                    $scope.shared_match_type = {};
+                }
                 var artifacts_to_commit = [];
                 for (var i = 0; i < $scope.artifacts.length; i++) {
                     if ($scope.checked_indexes[i]) {
@@ -99,7 +105,7 @@ angular.module('ThreatKB').controller('ImportController',
                 blockUI.start($scope.block_message);
 
                 var field_mapping = JSON.parse($scope.default_mapping.value);
-                Import.commit_artifacts(artifacts_to_commit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
+                Import.commit_artifacts(artifacts_to_commit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_match_type.match_type, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
                     blockUI.stop();
                     var ttl = 3000;
                     var message = "";
@@ -137,12 +143,15 @@ angular.module('ThreatKB').controller('ImportController',
                     $scope.shared_state.state = {};
                 }
 
+                if ($scope.shared_match_type === undefined) {
+                    $scope.shared_match_type = {};
+                }
                 if ($scope.autocommit) {
                     blockUI.start($scope.block_message);
                 }
 
                 var field_mapping = JSON.parse($scope.default_mapping.value);
-                Import.import_artifacts($scope.import_text, $scope.autocommit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
+                Import.import_artifacts($scope.import_text, $scope.autocommit, $scope.resurrect_retired_artifacts, $scope.shared_reference, $scope.shared_description, $scope.shared_state.state.state, $scope.shared_match_type.match_type, $scope.shared_owner, $scope.extract_ip, $scope.extract_dns, $scope.extract_signature, field_mapping).then(function (data) {
                     if ($scope.autocommit) {
                         blockUI.stop();
                         var message = "";
@@ -206,6 +215,7 @@ angular.module('ThreatKB').controller('ImportController',
                 $scope.users = Users.query();
                 $scope.cfg_states = Cfg_states.query();
                 $scope.shared_state = {};
+                $scope.shared_match_type = {};
                 $scope.extract_ip = true;
                 $scope.extract_dns = true;
                 $scope.extract_signature = true;

app/static/js/import/import-service.js

@@ -9,7 +9,7 @@ angular.module('ThreatKB').factory('Import',
                 commit_artifacts: commit_artifacts
             });
 
-            function import_artifacts(import_text, autocommit, resurrect_retired_artifacts, shared_reference, shared_description, shared_state, shared_owner, extract_ip, extract_dns, extract_signature, metadata_field_mapping) {
+            function import_artifacts(import_text, autocommit, resurrect_retired_artifacts, shared_reference, shared_description, shared_state, shared_match_type, shared_owner, extract_ip, extract_dns, extract_signature, metadata_field_mapping) {
                 // send a post request to the server
                 return $http.post('/ThreatKB/import', {
                     import_text: import_text,
@@ -18,6 +18,7 @@ angular.module('ThreatKB').factory('Import',
                     shared_reference: shared_reference,
                     shared_description: shared_description,
                     shared_state: shared_state,
+                    shared_match_type: shared_match_type,
                     shared_owner: shared_owner,
                     extract_ip: extract_ip,
                     extract_dns: extract_dns,
@@ -37,14 +38,15 @@ angular.module('ThreatKB').factory('Import',
 
             }
 
-            function commit_artifacts(artifacts, resurrect_retired_artifacts, shared_reference, shared_description, shared_state, shared_owner, extract_ip, extract_dns, extract_signature, metadata_field_mapping) {
+            function commit_artifacts(artifacts, resurrect_retired_artifacts, shared_reference, shared_description, shared_state, shared_match_type, shared_owner, extract_ip, extract_dns, extract_signature, metadata_field_mapping) {
                 // send a post request to the server
                 return $http.post('/ThreatKB/import/commit', {
                     artifacts: artifacts,
                     resurrect_retired_artifacts: resurrect_retired_artifacts,
                     shared_reference: shared_reference,
                     shared_description: shared_description,
                     shared_state: shared_state,
+                    shared_match_type: shared_match_type,
                     shared_owner: shared_owner,
                     extract_ip: extract_ip,
                     extract_dns: extract_dns,

app/static/views/c2dns/c2dns.html

@@ -37,6 +37,17 @@ <h4 class="modal-title" style="float: left; margin-right: 10px;">
                         </ui-select-choices>
                     </ui-select>
                 </div>
+                <div class="form-group">
+                    <label>Match Type</label>
+                    <ui-select ng-model="batch.match_type">
+                        <ui-select-match placeholder="Select a match type or press delete to clear..">
+                            <span ng-bind="$select.selected.match_type || batch.match_type"></span>
+                        </ui-select-match>
+                        <ui-select-choices repeat="match_type in (match_types | filter: $select.search)">
+                            <span>{{ match_type }}</span>
+                        </ui-select-choices>
+                    </ui-select>
+                </div>
                 <div class="form-group">
                     <label>Description</label>
                     <textarea class="form-control" placeholder="Enter description..." rows="6"

app/static/views/import/import.html

@@ -43,6 +43,16 @@
                     </ui-select-choices>
                 </ui-select>
             </div>
+            <div class="form-group">
+                <ui-select ng-model="shared_match_type.match_type">
+                    <ui-select-match placeholder="Select a shared match type..">
+                        <span ng-bind="$select.selected.match_type || shared_match_type.match_type"></span>
+                    </ui-select-match>
+                    <ui-select-choices repeat="match_type in (match_types | filter: $select.search)">
+                        <span>{{ match_type }}</span>
+                    </ui-select-choices>
+                </ui-select>
+            </div>
             <div class="form-group">
                 <textarea class="form-control" rows="10" placeholder="Metadata field mapping..."
                           ng-model="default_mapping.value" name="metadata_field_mapping"></textarea>