Skip to content

Commit 5b9b3b8

Browse files
efd6efd6
authored
qualys_vmdr: implement X-RateLimit header handling (#14899)
1 parent e0c5a36 commit 5b9b3b8

File tree

6 files changed

+231
-15
lines changed

6 files changed

+231
-15
lines changed

‎packages/qualys_vmdr/_dev/deploy/docker/files/config.yml

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,13 @@ rules:
1212
ids: 1,2,3
1313
responses:
1414
- status_code: 200
15+
headers:
16+
x-ratelimit-limit: ["300"]
17+
x-ratelimit-window-sec: ["3600"]
18+
x-concurrency-limit-limit: ["2"]
19+
x-concurrency-limit-running: ["0"]
20+
x-ratelimit-towait-sec: ["0"]
21+
x-ratelimit-remaining: ["299"]
1522
# Response only has host.id = [1,2] because of truncation limit
1623
body: |-
1724
<?xml version="1.0" encoding="UTF-8"?>
@@ -209,6 +216,13 @@ rules:
209216
show_cloud_tags: 1
210217
responses:
211218
- status_code: 200
219+
headers:
220+
x-ratelimit-limit: ["300"]
221+
x-ratelimit-window-sec: ["3600"]
222+
x-concurrency-limit-limit: ["2"]
223+
x-concurrency-limit-running: ["0"]
224+
x-ratelimit-towait-sec: ["0"]
225+
x-ratelimit-remaining: ["299"]
212226
body: |-
213227
<?xml version="1.0" encoding="UTF-8"?>
214228
<!DOCTYPE HOST_LIST_VM_DETECTION_OUTPUT SYSTEM "https://qualysapi.qualys.com/api/3.0/fo/asset/host/vm/detection/dtd/output.dtd">
@@ -319,6 +333,13 @@ rules:
319333
show_cloud_tags: 1
320334
responses:
321335
- status_code: 200
336+
headers:
337+
x-ratelimit-limit: ["300"]
338+
x-ratelimit-window-sec: ["3600"]
339+
x-concurrency-limit-limit: ["2"]
340+
x-concurrency-limit-running: ["0"]
341+
x-ratelimit-towait-sec: ["0"]
342+
x-ratelimit-remaining: ["299"]
322343
body: "" # handling empty XML response
323344
# Request knowledge_base with QID from Asset Host QID.
324345
# QID: 101,102,103 (3 unique QIDs for host ID: 1,2)
@@ -329,6 +350,13 @@ rules:
329350
action: list
330351
responses:
331352
- status_code: 200
353+
headers:
354+
x-ratelimit-limit: ["300"]
355+
x-ratelimit-window-sec: ["3600"]
356+
x-concurrency-limit-limit: ["2"]
357+
x-concurrency-limit-running: ["0"]
358+
x-ratelimit-towait-sec: ["0"]
359+
x-ratelimit-remaining: ["299"]
332360
body: |-
333361
<?xml version="1.0" encoding="UTF-8" ?>
334362
<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM "https://qualysapi.qualys.com/api/3.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd">
@@ -509,6 +537,13 @@ rules:
509537
action: list
510538
responses:
511539
- status_code: 200
540+
headers:
541+
x-ratelimit-limit: ["300"]
542+
x-ratelimit-window-sec: ["3600"]
543+
x-concurrency-limit-limit: ["2"]
544+
x-concurrency-limit-running: ["0"]
545+
x-ratelimit-towait-sec: ["0"]
546+
x-ratelimit-remaining: ["299"]
512547
body: |-
513548
<?xml version="1.0" encoding="UTF-8" ?>
514549
<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM "https://qualysapi.qg2.apps.qualys.com/api/3.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd">
@@ -644,6 +679,13 @@ rules:
644679
last_modified_after: '{last_modified_after:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}}Z'
645680
responses:
646681
- status_code: 200
682+
headers:
683+
x-ratelimit-limit: ["300"]
684+
x-ratelimit-window-sec: ["3600"]
685+
x-concurrency-limit-limit: ["2"]
686+
x-concurrency-limit-running: ["0"]
687+
x-ratelimit-towait-sec: ["0"]
688+
x-ratelimit-remaining: ["299"]
647689
body: |-
648690
<?xml version="1.0" encoding="UTF-8" ?>
649691
<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM "https://qualysapi.qualys.com/api/3.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd">
@@ -714,6 +756,13 @@ rules:
714756
last_modified_after: '{last_modified_after:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}}Z'
715757
responses:
716758
- status_code: 200
759+
headers:
760+
x-ratelimit-limit: ["300"]
761+
x-ratelimit-window-sec: ["3600"]
762+
x-concurrency-limit-limit: ["2"]
763+
x-concurrency-limit-running: ["0"]
764+
x-ratelimit-towait-sec: ["0"]
765+
x-ratelimit-remaining: ["299"]
717766
body: |-
718767
<?xml version="1.0" encoding="UTF-8" ?>
719768
<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM "https://qualysapi.qg2.apps.qualys.com/api/3.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd">
@@ -910,6 +959,13 @@ rules:
910959
last_modified_after: '{last_modified_after:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}}Z'
911960
responses:
912961
- status_code: 200
962+
headers:
963+
x-ratelimit-limit: ["300"]
964+
x-ratelimit-window-sec: ["3600"]
965+
x-concurrency-limit-limit: ["2"]
966+
x-concurrency-limit-running: ["0"]
967+
x-ratelimit-towait-sec: ["0"]
968+
x-ratelimit-remaining: ["299"]
913969
body: |-
914970
<?xml version="1.0" encoding="UTF-8" ?>
915971
<!DOCTYPE KNOWLEDGE_BASE_VULN_LIST_OUTPUT SYSTEM "https://qualysapi.qg1.apps.qualys.in/api/3.0/fo/knowledge_base/vuln/knowledge_base_vuln_list_output.dtd">
@@ -928,6 +984,13 @@ rules:
928984
since_datetime: '{since_datetime:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}}Z'
929985
responses:
930986
- status_code: 200
987+
headers:
988+
x-ratelimit-limit: ["300"]
989+
x-ratelimit-window-sec: ["3600"]
990+
x-concurrency-limit-limit: ["2"]
991+
x-concurrency-limit-running: ["0"]
992+
x-ratelimit-towait-sec: ["0"]
993+
x-ratelimit-remaining: ["299"]
931994
body: |-
932995
----BEGIN_RESPONSE_BODY_CSV
933996
"Date","Action","Module","Details","User Name","User Role","User IP"
@@ -941,6 +1004,13 @@ rules:
9411004
since_datetime: '{since_datetime:\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}}Z'
9421005
responses:
9431006
- status_code: 200
1007+
headers:
1008+
x-ratelimit-limit: ["300"]
1009+
x-ratelimit-window-sec: ["3600"]
1010+
x-concurrency-limit-limit: ["2"]
1011+
x-concurrency-limit-running: ["0"]
1012+
x-ratelimit-towait-sec: ["0"]
1013+
x-ratelimit-remaining: ["299"]
9441014
body: |-
9451015
----BEGIN_RESPONSE_BODY_CSV
9461016
"Date","Action","Module","Details","User Name","User Role","User IP"

‎packages/qualys_vmdr/changelog.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
# newer versions go on top
2+
- version: "6.9.0"
3+
changes:
4+
- description: Implement X-RateLimit header handling.
5+
type: enhancement
6+
link: https://github.com/elastic/integrations/pull/14899
27
- version: "6.8.1"
38
changes:
49
- description: Update the logic for populating the `vulnerability.score.base`, `vulnerability.score.version`, and `vulnerability.severity` fields.

‎packages/qualys_vmdr/data_stream/asset_host_detection/agent/stream/input.yml.hbs

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,8 @@ program: |
5555
"X-Requested-With": ["curl"],
5656
"Authorization": ["Basic "+(state.user+":"+state.password).base64()],
5757
}
58-
}).do_request().as(resp,
59-
resp.StatusCode == 200
60-
?
58+
}).do_request().as(resp, (
59+
resp.StatusCode == 200 ?
6160
(
6261
has(resp.Body) && size(resp.Body) != 0
6362
?
@@ -152,7 +151,43 @@ program: |
152151
},
153152
"want_more": false,
154153
})
155-
)
154+
).with(
155+
resp.Header.transformMapEntry(k, v,
156+
// Canonicalise header keys to match rate_limit conventions.
157+
// -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
158+
{
159+
k.has_suffix("-Limit") ?
160+
(k.trim_suffix("-Limit").to_lower() + "-Limit")
161+
: k.has_suffix("-Remaining") ?
162+
(k.trim_suffix("-Remaining").to_lower() + "-Remaining")
163+
:
164+
k.to_lower(): v,
165+
}
166+
).as(headers,
167+
// Calculate rate limits.
168+
rate_limit(
169+
headers.with(
170+
{
171+
"x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
172+
}
173+
),
174+
"x-ratelimit",
175+
false,
176+
true,
177+
duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
178+
0
179+
)
180+
).as(rate_headers, rate_headers.with({
181+
// Work around inf detection in input.
182+
// If the headers are missing or rate_limit failed, rate and
183+
// next may be missing. So use optional types.
184+
?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
185+
?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
186+
})).as(limit, {
187+
"header": resp.Header,
188+
"rate_limit": limit,
189+
})
190+
))
156191
)
157192
).as(state, state.with(
158193
!has(state.worklist) ? state :
@@ -167,7 +202,7 @@ program: |
167202
"X-Requested-With": ["curl"],
168203
"Authorization": ["Basic "+(state.user+":"+state.password).base64()],
169204
}
170-
}).do_request().as(resp, resp.StatusCode == 200 ?
205+
}).do_request().as(resp, (resp.StatusCode == 200 ?
171206
resp.Body.as(xml, try(xml.decode_xml('qualys_api_3_0_kb'), "decode_xml_error_kb").as(kb_body,
172207
!has(kb_body.decode_xml_error_kb)
173208
?
@@ -271,7 +306,43 @@ program: |
271306
},
272307
"want_more": false,
273308
}
274-
)
309+
).with(
310+
resp.Header.transformMapEntry(k, v,
311+
// Canonicalise header keys to match rate_limit conventions.
312+
// -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
313+
{
314+
k.has_suffix("-Limit") ?
315+
(k.trim_suffix("-Limit").to_lower() + "-Limit")
316+
: k.has_suffix("-Remaining") ?
317+
(k.trim_suffix("-Remaining").to_lower() + "-Remaining")
318+
:
319+
k.to_lower(): v,
320+
}
321+
).as(headers,
322+
// Calculate rate limits.
323+
rate_limit(
324+
headers.with(
325+
{
326+
"x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
327+
}
328+
),
329+
"x-ratelimit",
330+
false,
331+
true,
332+
duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
333+
0
334+
)
335+
).as(rate_headers, rate_headers.with({
336+
// Work around inf detection in input.
337+
// If the headers are missing or rate_limit failed, rate and
338+
// next may be missing. So use optional types.
339+
?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
340+
?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
341+
})).as(limit, {
342+
"header": resp.Header,
343+
"rate_limit": limit,
344+
})
345+
))
275346
:
276347
{
277348
"events": [],

‎packages/qualys_vmdr/data_stream/knowledge_base/agent/stream/input.yml.hbs

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,8 @@ program: |
3636
"X-Requested-With": ["curl"],
3737
"Authorization": ["Basic "+(state.user+":"+state.password).base64()],
3838
}
39-
}).do_request().as(resp,
40-
resp.StatusCode == 200
41-
?
39+
}).do_request().as(resp, (
40+
resp.StatusCode == 200 ?
4241
resp.Body.as(xml, bytes(xml).decode_xml('qualys_api_3_0').as(body, {
4342
"events": (
4443
has(body.doc.KNOWLEDGE_BASE_VULN_LIST_OUTPUT.RESPONSE.VULN_LIST)
@@ -83,7 +82,43 @@ program: |
8382
},
8483
"want_more": false,
8584
}
86-
)
85+
).with(
86+
resp.Header.transformMapEntry(k, v,
87+
// Canonicalise header keys to match rate_limit conventions.
88+
// -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
89+
{
90+
k.has_suffix("-Limit") ?
91+
(k.trim_suffix("-Limit").to_lower() + "-Limit")
92+
: k.has_suffix("-Remaining") ?
93+
(k.trim_suffix("-Remaining").to_lower() + "-Remaining")
94+
:
95+
k.to_lower(): v,
96+
}
97+
).as(headers,
98+
// Calculate rate limits.
99+
rate_limit(
100+
headers.with(
101+
{
102+
"x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
103+
}
104+
),
105+
"x-ratelimit",
106+
false,
107+
true,
108+
duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
109+
0
110+
)
111+
).as(rate_headers, rate_headers.with({
112+
// Work around inf detection in input.
113+
// If the headers are missing or rate_limit failed, rate and
114+
// next may be missing. So use optional types.
115+
?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
116+
?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
117+
})).as(limit, {
118+
"header": resp.Header,
119+
"rate_limit": limit,
120+
})
121+
))
87122
)
88123
tags:
89124
{{#if preserve_original_event}}

‎packages/qualys_vmdr/data_stream/user_activity/agent/stream/cel.yml.hbs

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,8 @@ program: |
4242
"Authorization": ["Basic " + (state.user + ":" + state.password).base64()],
4343
"X-Requested-With": ["curl"],
4444
}
45-
}).do_request().as(resp,
46-
resp.StatusCode == 200
47-
?
45+
}).do_request().as(resp, (
46+
resp.StatusCode == 200 ?
4847
string(resp.Body).as(text, {
4948
"csv": (
5049
text.contains_substr("----BEGIN_RESPONSE_BODY_CSV") ?
@@ -92,7 +91,43 @@ program: |
9291
},
9392
"want_more": false,
9493
}
95-
)
94+
).with(
95+
resp.Header.transformMapEntry(k, v,
96+
// Canonicalise header keys to match rate_limit conventions.
97+
// -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
98+
{
99+
k.has_suffix("-Limit") ?
100+
(k.trim_suffix("-Limit").to_lower() + "-Limit")
101+
: k.has_suffix("-Remaining") ?
102+
(k.trim_suffix("-Remaining").to_lower() + "-Remaining")
103+
:
104+
k.to_lower(): v,
105+
}
106+
).as(headers,
107+
// Calculate rate limits.
108+
rate_limit(
109+
headers.with(
110+
{
111+
"x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
112+
}
113+
),
114+
"x-ratelimit",
115+
false,
116+
true,
117+
duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
118+
0
119+
)
120+
).as(rate_headers, rate_headers.with({
121+
// Work around inf detection in input.
122+
// If the headers are missing or rate_limit failed, rate and
123+
// next may be missing. So use optional types.
124+
?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
125+
?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
126+
})).as(limit, {
127+
"header": resp.Header,
128+
"rate_limit": limit,
129+
})
130+
))
96131
)
97132
tags:
98133
{{#if preserve_duplicate_custom_fields}}

‎packages/qualys_vmdr/manifest.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
format_version: "3.4.0"
22
name: qualys_vmdr
33
title: Qualys VMDR
4-
version: "6.8.1"
4+
version: "6.9.0"
55
description: Collect data from Qualys VMDR platform with Elastic Agent.
66
type: integration
77
categories:

0 commit comments

Comments
 (0)