Skip to content

Commit

Permalink
[cisco_duo] Fix auth CEL cursor handling (#11456)
Browse files Browse the repository at this point in the history 
In the auth CEL program, the error `type conversion error from 'string'
to 'int'` seems to have been happening because of
`cursor.last_published` being set to a value of the form
`1532951895000,af0ba235-0b33-23c8-bc23-a31aa0231de8`, which can't be
parsed as an int.

Now `cursor.last_published` is replaced with `cursor.last_timestamp_ms`,
which is taken from the last result, and so will be available if the
last page of sequence has results but no value in
`response.metadata.next_offset`.

Also, the date is no longer shared across requests, the request
building is simplified, and redundant overrides of state are removed.

Related documentation: https://duo.com/docs/adminapi#authentication-logs
  • Loading branch information
@chrisberkhout
chrisberkhout authored Oct 17, 2024
1 parent 389e6c2 commit 19bd853
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 108 deletions.
6 changes: 3 additions & 3 deletions packages/cisco_duo/_dev/deploy/docker/files/config.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ rules:
],
"metadata": {
"next_offset": ["1666714065305","5bf1a860-fe39-49e3-be29-217659663a74"],
"total_objects": 5
"total_objects": 4
}},
"stat":"OK"}
- path: /admin/v2/logs/authentication
Expand All @@ -50,7 +50,7 @@ rules:
],
"metadata": {
"next_offset": ["1666714065306","5bf1a860-fe39-49e3-be29-217659663a74"],
"total_objects": 5
"total_objects": 3
}},
"stat":"OK"}
- path: /admin/v2/logs/authentication
Expand All @@ -65,7 +65,7 @@ rules:
{"access_device":{"browser":"Chrome","browser_version":"67.0.3396.99","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":true,"is_firewall_enabled":true,"is_password_set":true,"java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Mac OS X","os_version":"10.14.1","security_agents":[]},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Microsoft Azure Active Directory"},"auth_device":{"ip":"192.168.225.254","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"My iPhone X (734-555-2342)"},"email":"[email protected]","event_type":"authentication","factor":"duo_push","isotimestamp":"2020-02-13T18:56:20.351346+00:00","ood_software":null,"reason":"user_approved","result":"success","timestamp":1581620180,"trusted_endpoint_status":"not trusted","txid":"340a23e3-23f3-23c1-87dc-1491a23dfdbe","user":{"groups":["Duo Users","CorpHQ Users"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"[email protected]"}}
],
"metadata": {
"total_objects": 5
"total_objects": 2
}},
"stat":"OK"}
- path: /admin/v1/logs/offline_enrollment
Expand Down
5 changes: 5 additions & 0 deletions packages/cisco_duo/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "2.0.4"
changes:
- description: Fix auth CEL cursor handling.
type: bugfix
link: https://github.com/elastic/integrations/pull/11456
- version: "2.0.3"
changes:
- description: Set request rate limits.
Expand Down
171 changes: 67 additions & 104 deletions packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,121 +27,84 @@ program: |
state
:
state.with({
"mintime": state.?cursor.last_published.orValue(int(now - duration(state.initial_interval)) * 1000),
"maxtime": int(now - duration("2m")) * 1000,
"date": now.format(time_layout.RFC1123Z),
"mintime": state.?cursor.last_timestamp_ms.orValue(string(int(now - duration(state.initial_interval)) * 1000)),
"maxtime": string(int(now - duration("2m")) * 1000),
})
).as(state, state.with(
request(
"GET",
state.?want_more.orValue(false) ?
state.next_url
:
state.url.trim_right("/") + "/admin/v2/logs/authentication?" + {
"limit": [string(int(state.limit))],
"maxtime": [string(int(state.maxtime))],
"mintime": [string(int(state.mintime))],
"sort": ["ts:asc"],
}.format_query()
).with(
{
{
// prepare request data
"date": now.format(time_layout.RFC1123Z),
"method": "GET",
"url_base": state.url.trim_right("/"),
"url_path": "/admin/v2/logs/authentication",
"query_string": {
"limit": [string(int(state.limit))],
"maxtime": [state.maxtime],
"mintime": [state.mintime],
?"next_offset": state.?next_offset_joined.optMap(v, [v]),
"sort": ["ts:asc"],
}.format_query(),
}.as(r, r.with({
// add an authorization header value
"authorization": "Basic " + (
state.integration_key + ":" + (
[
r.date,
r.method,
r.url_base.trim_prefix("https://"),
r.url_path,
r.query_string,
].join("\n")
.hmac("sha1", bytes(state.secret_key))
.hex()
)
).base64(),
})).as(r,
// now do the request using the prepared data
request(
r.method,
[r.url_base, r.url_path, "?", r.query_string].join("")
).with({
"Header": {
"Content-Type": ["application/x-www-form-urlencoded"],
"Date": [state.date],
"Authorization": ["Basic " + (
state.integration_key + ":" + (
[
state.date,
"GET",
state.url.trim_prefix("https://"),
"/admin/v2/logs/authentication",
{
"limit": [string(int(state.limit))],
"maxtime": [string(int(state.maxtime))],
"mintime": [string(int(state.mintime))],
?"next_offset": has(state.next_offset) ?
optional.of([string(state.next_offset)])
:
optional.none(),
"sort": ["ts:asc"],
}.format_query()
].join("\n")
.hmac("sha1", bytes(state.secret_key))
.hex()
)
).base64()],
"Date": [r.date],
"Authorization": [r.authorization],
},
}
).do_request().as(resp, (resp.StatusCode == 200) ?
bytes(resp.Body).decode_json().as(body, has(body.?response.authlogs) && size(body.response.authlogs) > 0 ?
(
body.?response.metadata.next_offset.orValue(null) == null ?
optional.none()
: type(body.response.metadata.next_offset) == type([]) && size(body.response.metadata.next_offset) == 2 ?
optional.of(string(body.response.metadata.next_offset[0])+","+string(body.response.metadata.next_offset[1]))
:
// Fall back to the actual value. This will result in a
// failure on the next iteration, but will expose the
// value to logging to aid identification of a change
// in the format of this field should that happen.
body.?response.metadata.next_offset
).as(next_offset,
{
"events": body.response.authlogs.map(item,
{
"message": item.encode_json(),
}
),
"url": state.url,
"integration_key": state.integration_key,
"secret_key": state.secret_key,
"limit": state.limit,
"mintime": state.mintime,
"maxtime": state.maxtime,
"date": now.format(time_layout.RFC1123Z),
"want_more": next_offset.hasValue(),
?"next_offset": next_offset,
"next_url": next_offset.hasValue() ?
(
state.url.trim_right("/") + "/admin/v2/logs/authentication?" + {
"limit": [string(int(state.limit))],
"maxtime": [string(int(state.maxtime))],
"mintime": [string(int(state.mintime))],
"next_offset": [next_offset.value()],
"sort": ["ts:asc"],
}.format_query()
)
:
state.url,
}).do_request().as(resp, (resp.StatusCode == 200) ?
bytes(resp.Body).decode_json().as(body, has(body.?response.authlogs) && size(body.response.authlogs) > 0 ?
body.response.as(r, {
"events": r.authlogs.map(item, { "message": item.encode_json() }),
"want_more": r.?metadata.next_offset.hasValue(),
?"next_offset_joined": r.?metadata.next_offset.optMap(v, [v].flatten().join(",")),
"cursor": {
?"last_published": next_offset,
"last_timestamp_ms": string(int(r.authlogs[size(r.authlogs) - 1].timestamp) * 1000),
}
})
:
{
"events":[],
"want_more": false,
}

)
:
{
"events":[],
"want_more": false,
}

)
:
bytes(resp.Body).decode_json().as(body,
{
"events": {
"error": {
"code": has(body.code) ? string(body.code) : string(resp.StatusCode),
"id": string(resp.Status),
"message": "GET:"+(
size(resp.Body) != 0 ?
string(resp.Body)
:
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
),
bytes(resp.Body).decode_json().as(body,
{
"events": {
"error": {
"code": has(body.code) ? string(body.code) : string(resp.StatusCode),
"id": string(resp.Status),
"message": "GET:"+(
size(resp.Body) != 0 ?
string(resp.Body)
:
string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
),
},
},
},
"want_more": false,
}
"want_more": false,
}
)
)
)
))
Expand Down
2 changes: 1 addition & 1 deletion packages/cisco_duo/manifest.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: "3.0.2"
name: cisco_duo
title: Cisco Duo
version: "2.0.3"
version: "2.0.4"
description: Collect logs from Cisco Duo with Elastic Agent.
type: integration
categories:
Expand Down

0 comments on commit 19bd853

Please sign in to comment.