Skip to content

Commit 4157b00

Browse files
Ark2307Ark2307
committed
Fixing duplicate hosts
1 parent 0010c44 commit 4157b00

File tree

9 files changed

+123
-12
lines changed

9 files changed

+123
-12
lines changed

‎apps/api-runtime/src/main/java/com/akto/runtime/APICatalogSync.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1335,8 +1335,9 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog
13351335
for(String key: deltaInfoMap.keySet()) {
13361336
SingleTypeInfo dbInfo = dbInfoMap.get(key);
13371337
SingleTypeInfo deltaInfo = deltaInfoMap.get(key);
1338+
boolean isQueryParam = false;
13381339
if(deltaInfo.getParam().contains("_queryParam")) {
1339-
deltaInfo.setIsQueryParam(true);
1340+
isQueryParam = true;
13401341
String originalParam = deltaInfo.getParam().split("_queryParam")[0];
13411342
deltaInfo.setParam(originalParam);
13421343
}
@@ -1363,6 +1364,7 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog
13631364
update = Updates.combine(update, Updates.max(SingleTypeInfo.LAST_SEEN, deltaInfo.getLastSeen()));
13641365
update = Updates.combine(update, Updates.max(SingleTypeInfo.MAX_VALUE, deltaInfo.getMaxValue()));
13651366
update = Updates.combine(update, Updates.min(SingleTypeInfo.MIN_VALUE, deltaInfo.getMinValue()));
1367+
update = Updates.combine(update, Updates.set("isQueryParam", isQueryParam));
13661368
if (source != null) {
13671369
Bson updateSourceMap = Updates.set(SingleTypeInfo.SOURCES + "." + source.name(), new Document("timestamp", timestamp) );
13681370
update = Updates.combine(update, updateSourceMap);
@@ -1439,7 +1441,7 @@ public DbUpdateReturn getDBUpdatesForParams(APICatalog currentDelta, APICatalog
14391441
);
14401442
}
14411443

1442-
Bson updateKey = Filters.and(SingleTypeInfoDao.createFilters(deltaInfo), Filters.eq("isQueryParam", deltaInfo.getIsQueryParam()));
1444+
Bson updateKey = SingleTypeInfoDao.createFilters(deltaInfo);
14431445
update = Updates.combine(update,
14441446
Updates.setOnInsert(SingleTypeInfo._COLLECTION_IDS, Arrays.asList(deltaInfo.getApiCollectionId())));
14451447

‎apps/dashboard/src/main/java/com/akto/action/ApiCollectionsAction.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -319,14 +319,14 @@ public String deleteMultipleCollections() {
319319
Bson filter = Filters.in(SingleTypeInfo._COLLECTION_IDS, apiCollectionIds);
320320
Bson update = Updates.pullAll(SingleTypeInfo._COLLECTION_IDS, apiCollectionIds);
321321

322-
SingleTypeInfoDao.instance.deleteAll(Filters.in("apiCollectionId", apiCollectionIds));
322+
SingleTypeInfoDao.instance.getMCollection().deleteMany(Filters.in("apiCollectionId", apiCollectionIds));
323323
SingleTypeInfoDao.instance.updateMany(filter, update);
324-
APISpecDao.instance.deleteAll(Filters.in("apiCollectionId", apiCollectionIds));
325-
SensitiveParamInfoDao.instance.deleteAll(Filters.in("apiCollectionId", apiCollectionIds));
326-
SampleDataDao.instance.deleteAll(Filters.in("_id.apiCollectionId", apiCollectionIds));
327-
SensitiveSampleDataDao.instance.deleteAll(Filters.in("_id.apiCollectionId", apiCollectionIds));
328-
TrafficInfoDao.instance.deleteAll(Filters.in("_id.apiCollectionId", apiCollectionIds));
329-
DeleteResult apiInfoDeleteResult = ApiInfoDao.instance.deleteAll(Filters.in("_id.apiCollectionId", apiCollectionIds));
324+
APISpecDao.instance.getMCollection().deleteMany(Filters.in("apiCollectionId", apiCollectionIds));
325+
SensitiveParamInfoDao.instance.getMCollection().deleteMany(Filters.in("apiCollectionId", apiCollectionIds));
326+
SampleDataDao.instance.getMCollection().deleteMany(Filters.in("_id.apiCollectionId", apiCollectionIds));
327+
SensitiveSampleDataDao.instance.getMCollection().deleteMany(Filters.in("_id.apiCollectionId", apiCollectionIds));
328+
TrafficInfoDao.instance.getMCollection().deleteMany(Filters.in("_id.apiCollectionId", apiCollectionIds));
329+
DeleteResult apiInfoDeleteResult = ApiInfoDao.instance.getMCollection().deleteMany(Filters.in("_id.apiCollectionId", apiCollectionIds));
330330
SensitiveParamInfoDao.instance.updateMany(filter, update);
331331

332332
/*

‎apps/dashboard/src/main/java/com/akto/action/CleanAction.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,19 @@
88

99
import org.bson.conversions.Bson;
1010

11+
import com.akto.dao.ApiCollectionsDao;
1112
import com.akto.dao.ApiInfoDao;
1213
import com.akto.dao.SingleTypeInfoDao;
1314
import com.akto.dao.context.Context;
15+
import com.akto.dto.ApiCollection;
1416
import com.akto.dto.ApiInfo;
1517
import com.akto.dto.ApiInfo.ApiInfoKey;
1618
import com.akto.dto.type.SingleTypeInfo;
1719
import com.akto.dto.type.URLMethods.Method;
1820
import com.akto.log.LoggerMaker;
1921
import com.akto.log.LoggerMaker.LogDb;
22+
import com.akto.util.Constants;
23+
import com.akto.util.enums.GlobalEnums.CONTEXT_SOURCE;
2024
import com.mongodb.client.model.Filters;
2125
import com.mongodb.client.model.Projections;
2226
import com.mongodb.client.model.Updates;
@@ -206,6 +210,18 @@ public String unsetTemp() {
206210
return Action.SUCCESS.toUpperCase();
207211
}
208212

213+
public String deleteDuplicateHosts(){
214+
int accountId = Context.accountId.get();
215+
CONTEXT_SOURCE contextSource = Context.contextSource.get();
216+
217+
service.submit(() -> {
218+
Context.accountId.set(accountId);
219+
Context.contextSource.set(contextSource);
220+
221+
});
222+
return SUCCESS.toUpperCase();
223+
}
224+
209225
public List<Integer> getApiCollectionIds() {
210226
return apiCollectionIds;
211227
}

‎apps/dashboard/src/main/java/com/akto/action/LoginAction.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,25 @@
77
import com.akto.dao.SingleTypeInfoDao;
88
import com.akto.dao.UsersDao;
99
import com.akto.dao.context.Context;
10+
import com.akto.dao.monitoring.ModuleInfoDao;
1011
import com.akto.dao.testing.DefaultTestSuitesDao;
1112
import com.akto.dto.BackwardCompatibility;
1213
import com.akto.dto.Config;
1314
import com.akto.dto.SignupInfo;
1415
import com.akto.dto.SignupUserInfo;
1516
import com.akto.dto.User;
17+
import com.akto.dto.monitoring.ModuleInfo;
1618
import com.akto.dto.type.SingleTypeInfo;
1719
import com.akto.listener.InitializerListener;
1820
import com.akto.listener.RuntimeListener;
1921
import com.akto.log.LoggerMaker;
2022
import com.akto.log.LoggerMaker.LogDb;
2123
import com.akto.notifications.email.SendgridEmail;
2224
import com.akto.password_reset.PasswordResetUtils;
25+
import com.akto.util.Constants;
2326
import com.akto.util.DashboardMode;
2427
import com.akto.utils.JWT;
2528
import com.akto.utils.Token;
26-
import com.akto.utils.jobs.CleanInventory;
2729
import com.mongodb.BasicDBObject;
2830
import com.mongodb.client.model.Filters;
2931
import com.mongodb.client.model.FindOneAndUpdateOptions;
@@ -281,6 +283,17 @@ public static String loginUser(User user, HttpServletResponse servletResponse, b
281283
service.submit(() ->{
282284
triggerVulnColUpdation(user);
283285
});
286+
// temporary clean up
287+
// TODO: remove this after 2 days
288+
service.submit(() ->{
289+
for (String accountIdStr : user.getAccounts().keySet()) {
290+
int accountId = Integer.parseInt(accountIdStr);
291+
Context.accountId.set(accountId);
292+
if(ModuleInfoDao.instance.isValidForTimestamp(ModuleInfo.ModuleType.MINI_RUNTIME, Context.now() - 4 * Constants.ONE_DAY_TIMESTAMP, Context.now(), "1.50.0")){
293+
SingleTypeInfoDao.deleteDuplicateHostsForSameApi();
294+
}
295+
}
296+
});
284297
}
285298
}
286299
return Action.SUCCESS.toUpperCase();

‎apps/dashboard/src/main/resources/struts.xml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9382,6 +9382,28 @@
93829382
</result>
93839383
</action>
93849384

9385+
<action name="api/deleteDuplicateEntries" class="com.akto.action.CleanAction" method="deleteDuplicateHosts">
9386+
<interceptor-ref name="json"/>
9387+
<interceptor-ref name="defaultStack" />
9388+
<interceptor-ref name="roleAccessInterceptor">
9389+
<param name="featureLabel">API_COLLECTIONS</param>
9390+
<param name="accessType">READ_WRITE</param>
9391+
</interceptor-ref>
9392+
9393+
<result name="FORBIDDEN" type="json">
9394+
<param name="statusCode">403</param>
9395+
<param name="ignoreHierarchy">false</param>
9396+
<param name="includeProperties">^actionErrors.*</param>
9397+
</result>
9398+
<result name="SUCCESS" type="json">
9399+
</result>
9400+
<result name="ERROR" type="json">
9401+
<param name="statusCode">422</param>
9402+
<param name="ignoreHierarchy">false</param>
9403+
<param name="includeProperties">^actionErrors.*</param>
9404+
</result>
9405+
</action>
9406+
93859407
<action name="api/deleteNonHostSTIs" class="com.akto.action.CleanAction" method="deleteNonHostSTIs">
93869408
<interceptor-ref name="json"/>
93879409
<interceptor-ref name="defaultStack" />

‎apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/api.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -675,6 +675,13 @@ const settingRequests = {
675675
method: 'post',
676676
data: {}
677677
})
678+
},
679+
async deleteDuplicateEntries() {
680+
return await request({
681+
url: '/api/deleteDuplicateEntries',
682+
method: 'post',
683+
data: {}
684+
})
678685
}
679686
}
680687

‎apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/settings/undo_demerged_apis/UndoDemergedApis.jsx

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Button, LegacyCard, ResourceItem, ResourceList, Text } from '@shopify/polaris'
1+
import { Button, HorizontalStack, LegacyCard, ResourceItem, ResourceList, Text } from '@shopify/polaris'
22
import React, { useEffect, useState } from 'react'
33
import settingRequests from '../api'
44
import func from '@/util/func'
@@ -24,6 +24,10 @@ const UndoDemergedApis = () => {
2424
})
2525
}
2626

27+
const deleteDuplicateEntries = async () => {
28+
await settingRequests.deleteDuplicateEntries();
29+
}
30+
2731
useEffect(() => {
2832
fetchMergedApis()
2933
}, [])
@@ -85,7 +89,11 @@ const UndoDemergedApis = () => {
8589
)
8690

8791
const secondaryActionsComp = (
88-
<Button disabled={mergedApis?.length === 0} onClick={() => undoDemergedApis(mergedApis)}>Undo All De-merged APIs</Button>
92+
<HorizontalStack gap={"2"}>
93+
<Button disabled={mergedApis?.length === 0} onClick={() => undoDemergedApis(mergedApis)}>Undo All De-merged APIs</Button>
94+
{window.USER_NAME?.toLowerCase()?.includes("@akto.io") ? <Button onClick={() => deleteDuplicateEntries()}>Delete duplicates</Button> : null}
95+
</HorizontalStack>
96+
8997
)
9098

9199
return (

‎libs/dao/src/main/java/com/akto/dao/SingleTypeInfoDao.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
import com.akto.dao.context.Context;
88
import com.akto.dto.AktoDataType;
9+
import com.akto.dto.ApiCollection;
910
import com.akto.dto.ApiInfo;
1011
import com.akto.dto.CollectionConditions.MethodCondition;
1112
import com.akto.dto.rbac.UsersCollectionsList;
@@ -21,6 +22,7 @@
2122
import com.mongodb.client.model.*;
2223

2324
import org.bson.conversions.Bson;
25+
import org.bson.types.ObjectId;
2426

2527
public class SingleTypeInfoDao extends AccountsContextDaoWithRbac<SingleTypeInfo> {
2628

@@ -854,6 +856,36 @@ public List<BasicDBObject> fetchRecentEndpoints(int startTimestamp, int endTimes
854856
return endpoints;
855857
}
856858

859+
public static void deleteDuplicateHostsForSameApi(){
860+
List<ApiCollection> apiCollections = ApiCollectionsDao.instance.findAll(
861+
Filters.and(
862+
Filters.ne(ApiCollection._DEACTIVATED, true),
863+
Filters.exists(ApiCollection.HOST_NAME)
864+
),
865+
Projections.include(Constants.ID)
866+
);
867+
for(ApiCollection apiCollection: apiCollections) {
868+
SingleTypeInfoDao.deleteDuplicateHostsForSameApi(apiCollection.getId());
869+
}
870+
}
871+
872+
873+
public static void deleteDuplicateHostsForSameApi(int apiCollectionId){
874+
List<SingleTypeInfo> allList = SingleTypeInfoDao.instance.findAll(filterForHostHeader(apiCollectionId, true), Projections.include(SingleTypeInfo._API_COLLECTION_ID, SingleTypeInfo._URL, SingleTypeInfo._METHOD));
875+
Set<String> uniqueKeys = new HashSet<>();
876+
List<ObjectId> deleteIds = new ArrayList<>();
877+
for (SingleTypeInfo sti : allList) {
878+
String key = sti.getApiCollectionId() + "_" + sti.getUrl() + "_" + sti.getMethod();
879+
if(uniqueKeys.contains(key)){
880+
deleteIds.add(sti.getId());
881+
} else {
882+
uniqueKeys.add(key);
883+
}
884+
}
885+
SingleTypeInfoDao.instance.getMCollection().deleteMany(Filters.in("_id", deleteIds));
886+
}
887+
888+
857889
public static BasicDBObject getApiInfoGroupedId() {
858890
BasicDBObject groupedId =
859891
new BasicDBObject("apiCollectionId", "$apiCollectionId")

‎libs/dao/src/main/java/com/akto/dao/monitoring/ModuleInfoDao.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package com.akto.dao.monitoring;
22

3+
34
import com.akto.dao.AccountsContextDao;
45
import com.akto.dto.monitoring.ModuleInfo;
6+
import com.mongodb.client.model.Filters;
57

68
public class ModuleInfoDao extends AccountsContextDao<ModuleInfo> {
79
@Override
@@ -16,4 +18,13 @@ private ModuleInfoDao(){}
1618
public Class<ModuleInfo> getClassT() {
1719
return ModuleInfo.class;
1820
}
21+
22+
public boolean isValidForTimestamp(ModuleInfo.ModuleType moduleType, int startTs, int endTs, String version){
23+
return instance.count(Filters.and(
24+
Filters.eq(ModuleInfo.MODULE_TYPE, moduleType),
25+
Filters.gte(ModuleInfo.LAST_HEARTBEAT_RECEIVED, startTs),
26+
Filters.lte(ModuleInfo.LAST_HEARTBEAT_RECEIVED, endTs),
27+
Filters.regex("currentVersion", version)
28+
)) > 0;
29+
}
1930
}

0 commit comments

Comments
 (0)