Skip to content

Commit 841ad6b

Browse files
bitterpanda63bitterpanda63
authored
Merge pull request #166 from AikidoSec/AIK-4442
AIK-4442 Report blocked requests per IPList/Botlist
2 parents 544befc + fc3cea3 commit 841ad6b

File tree

17 files changed

+968
-168
lines changed

17 files changed

+968
-168
lines changed

‎agent_api/src/main/java/dev/aikido/agent_api/background/cloud/api/ReportingApi.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,18 @@ public ReportingApi(int timeoutInSec) {
3030

3131
public record APIListsResponse(
3232
List<ListsResponseEntry> blockedIPAddresses,
33+
List<ListsResponseEntry> monitoredIPAddresses,
3334
List<ListsResponseEntry> allowedIPAddresses,
34-
String blockedUserAgents
35+
String blockedUserAgents,
36+
String monitoredUserAgents,
37+
List<UserAgentDetail> userAgentDetails
3538
) {}
36-
public record ListsResponseEntry(String source, String description, List<String> ips) {}
39+
40+
public record ListsResponseEntry(String key, String source, String description, List<String> ips) {
41+
}
42+
43+
public record UserAgentDetail(String key, String pattern) {
44+
}
3745
/**
3846
* Fetch blocked lists using a separate API call, these can include :
3947
* -> blocked IP Addresses (e.g. geo restrictions)

‎agent_api/src/main/java/dev/aikido/agent_api/collectors/WebRequestCollector.java

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import dev.aikido.agent_api.background.Endpoint;
44
import dev.aikido.agent_api.context.Context;
55
import dev.aikido.agent_api.context.ContextObject;
6+
import dev.aikido.agent_api.context.RouteMetadata;
67
import dev.aikido.agent_api.storage.ServiceConfiguration;
78
import dev.aikido.agent_api.storage.statistics.StatisticsStore;
89

@@ -38,31 +39,46 @@ public static Res report(ContextObject newContext) {
3839
// Increment total hits :
3940
StatisticsStore.incrementHits();
4041

41-
// Per-route IP allowlists :
42-
List<Endpoint> matchedEndpoints = matchEndpoints(newContext.getRouteMetadata(), config.getEndpoints());
43-
if (!ipAllowedToAccessRoute(newContext.getRemoteAddress(), matchedEndpoints)) {
42+
Res endpointAllowlistRes = checkEndpointAllowlist(newContext.getRouteMetadata(), newContext.getRemoteAddress(), config);
43+
if (endpointAllowlistRes != null)
44+
return endpointAllowlistRes;
45+
46+
Res blockedIpsRes = checkBlockedIps(newContext.getRemoteAddress(), config);
47+
if (blockedIpsRes != null)
48+
return blockedIpsRes;
49+
50+
return checkBlockedUserAgents(newContext.getHeader("user-agent"), config);
51+
}
52+
53+
private static Res checkEndpointAllowlist(RouteMetadata routeMetadata, String remoteAddress, ServiceConfiguration config) {
54+
List<Endpoint> matchedEndpoints = matchEndpoints(routeMetadata, config.getEndpoints());
55+
if (!ipAllowedToAccessRoute(remoteAddress, matchedEndpoints)) {
4456
String msg = "Your IP address is not allowed to access this resource.";
45-
msg += " (Your IP: " + newContext.getRemoteAddress() + ")";
57+
msg += " (Your IP: " + remoteAddress + ")";
4658
return new Res(msg, 403);
4759
}
60+
return null; // not blocked
61+
}
4862

49-
// Blocked IP lists (e.g. Geo restrictions)
50-
ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(newContext.getRemoteAddress());
63+
private static Res checkBlockedIps(String remoteAddress, ServiceConfiguration config) {
64+
ServiceConfiguration.BlockedResult ipBlocked = config.isIpBlocked(remoteAddress);
5165
if (ipBlocked.blocked()) {
5266
String msg = "Your IP address is blocked. Reason: " + ipBlocked.description();
53-
msg += " (Your IP: " + newContext.getRemoteAddress() + ")";
67+
msg += " (Your IP: " + remoteAddress + ")";
5468
return new Res(msg, 403);
5569
}
70+
return null; // not blocked
71+
}
5672

57-
// User-Agent blocking (e.g. blocking bots)
58-
String userAgent = newContext.getHeader("user-agent");
59-
if (userAgent != null && !userAgent.isEmpty()) {
60-
if (config.isBlockedUserAgent(userAgent)) {
61-
String msg = "You are not allowed to access this resource because you have been identified as a bot.";
62-
return new Res(msg, 403);
63-
}
73+
private static Res checkBlockedUserAgents(String userAgent, ServiceConfiguration config) {
74+
if (userAgent == null || userAgent.isEmpty()) {
75+
return null; // not blocked
76+
}
77+
if (config.isBlockedUserAgent(userAgent)) {
78+
String msg = "You are not allowed to access this resource because you have been identified as a bot.";
79+
return new Res(msg, 403);
6480
}
65-
return null;
81+
return null; // not blocked
6682
}
6783

6884
public record Res(String msg, Integer status) {

‎agent_api/src/main/java/dev/aikido/agent_api/helpers/patterns/SafePatternCompiler.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package dev.aikido.agent_api.helpers.patterns;
2+
3+
import java.util.regex.Pattern;
4+
import java.util.regex.PatternSyntaxException;
5+
6+
public final class SafePatternCompiler {
7+
private SafePatternCompiler() {
8+
}
9+
10+
public static Pattern compilePatternSafely(String regex, int flags) {
11+
if (regex == null || regex.isEmpty()) {
12+
return null;
13+
}
14+
try {
15+
return Pattern.compile(regex, flags);
16+
} catch (PatternSyntaxException ignored) {
17+
return null;
18+
}
19+
}
20+
}

‎agent_api/src/main/java/dev/aikido/agent_api/storage/ServiceConfiguration.java

Lines changed: 18 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,12 @@
44
import dev.aikido.agent_api.background.cloud.api.APIResponse;
55
import dev.aikido.agent_api.background.cloud.api.ReportingApi;
66
import dev.aikido.agent_api.helpers.net.IPList;
7+
import dev.aikido.agent_api.storage.service_configuration.ParsedFirewallLists;
8+
import dev.aikido.agent_api.storage.statistics.StatisticsStore;
79

810
import java.util.ArrayList;
911
import java.util.HashSet;
1012
import java.util.List;
11-
import java.util.regex.Pattern;
1213

1314
import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList;
1415
import static dev.aikido.agent_api.vulnerabilities.ssrf.IsPrivateIP.isPrivateIp;
@@ -18,16 +19,13 @@
1819
* It is essential for e.g. rate limiting
1920
*/
2021
public class ServiceConfiguration {
21-
private final List<IPListEntry> blockedIps = new ArrayList<>();
22-
private final List<IPListEntry> allowedIps = new ArrayList<>();
22+
private final ParsedFirewallLists firewallLists = new ParsedFirewallLists();
2323
private boolean blockingEnabled;
2424
private boolean receivedAnyStats;
2525
private boolean middlewareInstalled;
2626
private IPList bypassedIPs = new IPList();
2727
private HashSet<String> blockedUserIDs = new HashSet<>();
2828
private List<Endpoint> endpoints = new ArrayList<>();
29-
// User-Agent Blocking (e.g. bot blocking) :
30-
private Pattern blockedUserAgentRegex;
3129

3230
public ServiceConfiguration() {
3331
this.receivedAnyStats = true; // true by default, waiting for the startup event
@@ -89,66 +87,37 @@ public boolean isIpBypassed(String ip) {
8987
public BlockedResult isIpBlocked(String ip) {
9088
// Check for allowed ip addresses (i.e. only one country is allowed to visit the site)
9189
// Always allow access from private IP addresses (those include local IP addresses)
92-
if (!allowedIps.isEmpty() && !isPrivateIp(ip)) {
93-
boolean ipAllowed = false;
94-
for (IPListEntry entry : allowedIps) {
95-
if (entry.ipList.matches(ip)) {
96-
ipAllowed = true; // We allow IP addresses as long as they match with one of the lists.
97-
break;
98-
}
99-
}
100-
if (!ipAllowed) {
101-
return new BlockedResult(true, "not in allowlist");
102-
}
90+
if (!isPrivateIp(ip) && !firewallLists.matchesAllowedIps(ip)) {
91+
return new BlockedResult(true, "not in allowlist");
10392
}
10493

10594
// Check for blocked ip addresses
106-
for (IPListEntry entry : blockedIps) {
107-
if (entry.ipList.matches(ip)) {
108-
return new BlockedResult(true, entry.description);
95+
List<ParsedFirewallLists.Match> blockedIpMatches = firewallLists.matchBlockedIps(ip);
96+
for (ParsedFirewallLists.Match match : blockedIpMatches) {
97+
StatisticsStore.incrementIpHits(match.key());
98+
}
99+
for (ParsedFirewallLists.Match match : firewallLists.matchBlockedIps(ip)) {
100+
if (match.block()) {
101+
return new BlockedResult(true, match.description());
109102
}
110103
}
104+
111105
return new BlockedResult(false, null);
112106
}
113107

114108
public void updateBlockedLists(ReportingApi.APIListsResponse res) {
115-
// Update blocked IP addresses (e.g. for geo restrictions) :
116-
blockedIps.clear();
117-
if (res.blockedIPAddresses() != null) {
118-
for (ReportingApi.ListsResponseEntry entry : res.blockedIPAddresses()) {
119-
IPList ipList = createIPList(entry.ips());
120-
blockedIps.add(new IPListEntry(ipList, entry.description()));
121-
}
122-
}
123-
124-
// Update allowed IP addresses (e.g. for geo restrictions) :
125-
allowedIps.clear();
126-
if (res.allowedIPAddresses() != null) {
127-
for (ReportingApi.ListsResponseEntry entry : res.allowedIPAddresses()) {
128-
IPList ipList = createIPList(entry.ips());
129-
this.allowedIps.add(new IPListEntry(ipList, entry.description()));
130-
}
131-
}
132-
133-
// Update Blocked User-Agents regex
134-
blockedUserAgentRegex = null;
135-
if (res.blockedUserAgents() != null && !res.blockedUserAgents().isEmpty()) {
136-
this.blockedUserAgentRegex = Pattern.compile(res.blockedUserAgents(), Pattern.CASE_INSENSITIVE);
137-
}
109+
this.firewallLists.update(res);
138110
}
139111

140112
/**
141113
* Check if a given User-Agent is blocked or not :
142114
*/
143115
public boolean isBlockedUserAgent(String userAgent) {
144-
if (blockedUserAgentRegex != null) {
145-
return blockedUserAgentRegex.matcher(userAgent).find();
116+
ParsedFirewallLists.UABlockedResult result = this.firewallLists.matchBlockedUserAgents(userAgent);
117+
for (String matchedKey : result.matchedKeys()) {
118+
StatisticsStore.incrementUAHits(matchedKey);
146119
}
147-
return false;
148-
}
149-
150-
// IP restrictions (e.g. Geo-IP Restrictions) :
151-
public record IPListEntry(IPList ipList, String description) {
120+
return result.block();
152121
}
153122

154123
public record BlockedResult(boolean blocked, String description) {

‎agent_api/src/main/java/dev/aikido/agent_api/storage/service_configuration/ParsedFirewallLists.java

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package dev.aikido.agent_api.storage.service_configuration;
2+
3+
import dev.aikido.agent_api.background.cloud.api.ReportingApi;
4+
import dev.aikido.agent_api.helpers.net.IPList;
5+
6+
import java.util.ArrayList;
7+
import java.util.List;
8+
import java.util.regex.Pattern;
9+
10+
import static dev.aikido.agent_api.helpers.IPListBuilder.createIPList;
11+
import static dev.aikido.agent_api.helpers.patterns.SafePatternCompiler.compilePatternSafely;
12+
13+
public class ParsedFirewallLists {
14+
private final List<IPEntry> blockedIps = new ArrayList<>();
15+
private final List<IPEntry> allowedIps = new ArrayList<>();
16+
private final List<UADetailsEntry> uaDetails = new ArrayList<>();
17+
private Pattern blockedUserAgents = null;
18+
private Pattern monitoredUserAgents = null;
19+
20+
public ParsedFirewallLists() {
21+
22+
}
23+
24+
public List<Match> matchBlockedIps(String ip) {
25+
List<Match> matches = new ArrayList<>();
26+
for (IPEntry entry : this.blockedIps) {
27+
if (entry.ips().matches(ip)) {
28+
matches.add(new Match(entry.key(), !entry.monitor(), entry.description()));
29+
}
30+
}
31+
return matches;
32+
}
33+
34+
// returns true if one or more matches has been found with allowlist.
35+
public boolean matchesAllowedIps(String ip) {
36+
if (this.allowedIps.isEmpty()) {
37+
return true; // Empty allowed is means all ips match
38+
}
39+
for (IPEntry entry : this.allowedIps) {
40+
if (entry.ips().matches(ip)) {
41+
return true;
42+
}
43+
}
44+
return false;
45+
}
46+
47+
public UABlockedResult matchBlockedUserAgents(String userAgent) {
48+
boolean isBlocked = false;
49+
if (blockedUserAgents != null)
50+
isBlocked = blockedUserAgents.matcher(userAgent).find();
51+
52+
boolean isMonitored = false;
53+
if (monitoredUserAgents != null)
54+
isMonitored = monitoredUserAgents.matcher(userAgent).find();
55+
56+
if (!isMonitored && !isBlocked)
57+
// only run the more detailed matches if it's an actual attack/monitored.
58+
return new UABlockedResult(false, List.of());
59+
60+
List<String> matchedUAKeys = new ArrayList<>();
61+
for (UADetailsEntry entry : this.uaDetails) {
62+
if (entry.pattern().matcher(userAgent).find()) {
63+
matchedUAKeys.add(entry.key());
64+
}
65+
}
66+
return new UABlockedResult(isBlocked, matchedUAKeys);
67+
}
68+
69+
public void update(ReportingApi.APIListsResponse response) {
70+
blockedIps.clear();
71+
updateBlockedIps(response.blockedIPAddresses());
72+
updateMonitoredIps(response.monitoredIPAddresses());
73+
74+
updateAllowedIps(response.allowedIPAddresses());
75+
76+
updateBlockedAndMonitoredUAs(response.blockedUserAgents(), response.monitoredUserAgents());
77+
updateUADetails(response.userAgentDetails());
78+
}
79+
80+
public void updateBlockedIps(List<ReportingApi.ListsResponseEntry> blockedIpLists) {
81+
if (blockedIpLists == null)
82+
return;
83+
for (ReportingApi.ListsResponseEntry entry : blockedIpLists) {
84+
IPList ipList = createIPList(entry.ips());
85+
blockedIps.add(new IPEntry(/* monitor */ false, entry.key(), entry.source(), entry.description(), ipList));
86+
}
87+
}
88+
89+
public void updateMonitoredIps(List<ReportingApi.ListsResponseEntry> monitoredIpsList) {
90+
if (monitoredIpsList == null)
91+
return;
92+
for (ReportingApi.ListsResponseEntry entry : monitoredIpsList) {
93+
IPList ipList = createIPList(entry.ips());
94+
blockedIps.add(new IPEntry(/* monitor */ true, entry.key(), entry.source(), entry.description(), ipList));
95+
}
96+
}
97+
98+
public void updateAllowedIps(List<ReportingApi.ListsResponseEntry> allowedIpLists) {
99+
allowedIps.clear();
100+
if (allowedIpLists == null)
101+
return;
102+
for (ReportingApi.ListsResponseEntry entry : allowedIpLists) {
103+
IPList ipList = createIPList(entry.ips());
104+
boolean shouldMonitor = false; // we don't monitor allowed ips
105+
allowedIps.add(new IPEntry(shouldMonitor, entry.key(), entry.source(), entry.description(), ipList));
106+
}
107+
}
108+
109+
public void updateUADetails(List<ReportingApi.UserAgentDetail> userAgentDetails) {
110+
this.uaDetails.clear();
111+
if (userAgentDetails == null)
112+
return;
113+
for (ReportingApi.UserAgentDetail entry : userAgentDetails) {
114+
Pattern pattern = compilePatternSafely(entry.pattern(), Pattern.CASE_INSENSITIVE);
115+
if (pattern != null) {
116+
this.uaDetails.add(new UADetailsEntry(entry.key(), pattern));
117+
}
118+
}
119+
}
120+
121+
public void updateBlockedAndMonitoredUAs(String blockedUAs, String monitoredUAs) {
122+
this.blockedUserAgents = compilePatternSafely(blockedUAs, Pattern.CASE_INSENSITIVE);
123+
this.monitoredUserAgents = compilePatternSafely(monitoredUAs, Pattern.CASE_INSENSITIVE);
124+
}
125+
126+
127+
public record Match(String key, boolean block, String description) {
128+
}
129+
130+
public record UABlockedResult(boolean block, List<String> matchedKeys) {
131+
}
132+
133+
private record IPEntry(boolean monitor, String key, String source, String description, IPList ips) {
134+
}
135+
136+
private record UADetailsEntry(String key, Pattern pattern) {
137+
}
138+
}

0 commit comments

Comments
 (0)