Skip to content

Commit 3c2a1b8

Browse files
towfiqitowfiqi
committed
feat: adds the ability to add url as a domain.
You can now track specific marketplace/social domain URLs. For example a reddit.com post, an amazon.com product, github repo etc. closes: #53, #90, #119
1 parent e2ecdef commit 3c2a1b8

File tree

11 files changed

+85
-51
lines changed

11 files changed

+85
-51
lines changed

‎components/domains/AddDomain.tsx

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import React, { useState } from 'react';
22
import Modal from '../common/Modal';
33
import { useAddDomain } from '../../services/domains';
4-
import { isValidDomain } from '../../utils/client/validators';
4+
import { isValidUrl } from '../../utils/client/validators';
55

66
type AddDomainProps = {
77
domains: DomainType[],
@@ -16,24 +16,30 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
1616
const addDomain = () => {
1717
setNewDomainError('');
1818
const existingDomains = domains.map((d) => d.domain);
19-
const insertedDomains = newDomain.split('\n');
19+
const insertedURLs = newDomain.split('\n');
2020
const domainsTobeAdded:string[] = [];
2121
const invalidDomains:string[] = [];
22-
insertedDomains.forEach((dom) => {
23-
const domain = dom.trim();
24-
if (isValidDomain(domain)) {
25-
if (!existingDomains.includes(domain)) {
26-
domainsTobeAdded.push(domain);
22+
insertedURLs.forEach((url) => {
23+
const theURL = url.trim();
24+
if (isValidUrl(theURL)) {
25+
const domURL = new URL(theURL);
26+
const isDomain = domURL.pathname === '/';
27+
if (isDomain && !existingDomains.includes(domURL.host)) {
28+
domainsTobeAdded.push(domURL.host);
29+
}
30+
if (!isDomain && !existingDomains.includes(domURL.href)) {
31+
const cleanedURL = domURL.href.replace('https://', '').replace('http://', '').replace(/^\/+|\/+$/g, '');
32+
domainsTobeAdded.push(cleanedURL);
2733
}
2834
} else {
29-
invalidDomains.push(domain);
35+
invalidDomains.push(theURL);
3036
}
3137
});
3238
if (invalidDomains.length > 0) {
33-
setNewDomainError(`Please Insert Valid Domain names. Invalid Domains: ${invalidDomains.join(', ')}`);
39+
setNewDomainError(`Please Insert Valid Domain URL. Invalid URLs: ${invalidDomains.join(', ')}`);
3440
} else if (domainsTobeAdded.length > 0) {
35-
// TODO: Domain Action
36-
addMutate(domainsTobeAdded);
41+
console.log('domainsTobeAdded :', domainsTobeAdded);
42+
addMutate(domainsTobeAdded);
3743
}
3844
};
3945

@@ -45,11 +51,11 @@ const AddDomain = ({ closeModal, domains = [] }: AddDomainProps) => {
4551
return (
4652
<Modal closeModal={() => { closeModal(false); }} title={'Add New Domain'}>
4753
<div data-testid="adddomain_modal">
48-
<h4 className='text-sm mt-4'>Domain Names</h4>
54+
<h4 className='text-sm mt-4'>Domain URL</h4>
4955
<textarea
5056
className={`w-full h-40 border rounded border-gray-200 p-4 outline-none
5157
focus:border-indigo-300 ${newDomainError ? ' border-red-400 focus:border-red-400' : ''}`}
52-
placeholder="Type or Paste Domains here. Insert Each Domain in a New line."
58+
placeholder="Type or Paste URLs here. Insert Each URL in a New line."
5359
value={newDomain}
5460
autoFocus={true}
5561
onChange={handleDomainInput}>

‎components/domains/DomainItem.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ const DomainItem = ({ domain, selected, isConsoleIntegrated = false, thumb, upda
3838
/>
3939
</div>
4040
<div className="domain_details flex-1">
41-
<h3 className='font-semibold text-base mb-2'>{domain.domain}</h3>
41+
<h3 className='font-semibold text-base mb-2 max-w-[200px] text-ellipsis overflow-hidden' title={domain.domain}>{domain.domain}</h3>
4242
{keywordsUpdated && (
4343
<span className=' text-gray-600 text-xs'>
4444
Updated <TimeAgo title={dayjs(keywordsUpdated).format('DD-MMM-YYYY, hh:mm:ss A')} date={keywordsUpdated} />

‎components/keywords/SCKeywordsTable.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleInt
3131
const [filterParams, setFilterParams] = useState<KeywordFilters>({ countries: [], tags: [], search: '' });
3232
const [sortBy, setSortBy] = useState<string>('imp_desc');
3333
const [SCListHeight, setSCListHeight] = useState(500);
34-
const { keywordsData } = useFetchKeywords(router);
34+
const { keywordsData } = useFetchKeywords(router, domain?.domain || '');
3535
const addedkeywords: string[] = keywordsData?.keywords?.map((key: KeywordType) => `${key.keyword}:${key.country}:${key.device}`) || [];
3636
const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); });
3737
const [isMobile] = useIsMobile();

‎pages/api/domains.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ const addDomain = async (req: NextApiRequest, res: NextApiResponse<DomainsAddRes
6969
domains.forEach((domain: string) => {
7070
domainsToAdd.push({
7171
domain: domain.trim(),
72-
slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-'),
72+
slug: domain.trim().replaceAll('-', '_').replaceAll('.', '-').replaceAll('/', '-'),
7373
lastUpdated: new Date().toJSON(),
7474
added: new Date().toJSON(),
7575
});

‎pages/api/insight.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,14 @@ const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiRe
3737

3838
// First try and read the Local SC Domain Data file.
3939
const localSCData = await readLocalSCData(domainname);
40-
const oldFetchedDate = localSCData.lastFetched;
41-
const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
4240

43-
if (localSCData && localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
44-
const response = getInsightFromSCData(localSCData);
45-
return res.status(200).json({ data: response });
41+
if (localSCData) {
42+
const oldFetchedDate = localSCData.lastFetched;
43+
const fetchTimeDiff = new Date().getTime() - (oldFetchedDate ? new Date(oldFetchedDate as string).getTime() : 0);
44+
if (localSCData.stats && localSCData.stats.length && fetchTimeDiff <= 86400000) {
45+
const response = getInsightFromSCData(localSCData);
46+
return res.status(200).json({ data: response });
47+
}
4648
}
4749

4850
// If the Local SC Domain Data file does not exist, fetch from Googel Search Console.

‎pages/api/keywords.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse<KeywordsGet
4545
if (!req.query.domain && typeof req.query.domain !== 'string') {
4646
return res.status(400).json({ error: 'Domain is Required!' });
4747
}
48-
const domain = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-');
48+
const domain = (req.query.domain as string);
4949
const integratedSC = process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL;
5050
const domainSCData = integratedSC ? await readLocalSCData(domain) : false;
5151

‎pages/domain/[slug]/index.tsx

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,6 @@ const SingleDomain: NextPage = () => {
2727
const [keywordSPollInterval, setKeywordSPollInterval] = useState<undefined|number>(undefined);
2828
const { data: appSettings } = useFetchSettings();
2929
const { data: domainsData } = useFetchDomains(router);
30-
const { keywordsData, keywordsLoading } = useFetchKeywords(router, setKeywordSPollInterval, keywordSPollInterval);
31-
32-
const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
33-
const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
3430

3531
const activDomain: DomainType|null = useMemo(() => {
3632
let active:DomainType|null = null;
@@ -40,14 +36,18 @@ const SingleDomain: NextPage = () => {
4036
return active;
4137
}, [router.query.slug, domainsData]);
4238

39+
const { keywordsData, keywordsLoading } = useFetchKeywords(router, activDomain?.domain || '', setKeywordSPollInterval, keywordSPollInterval);
40+
const theDomains: DomainType[] = (domainsData && domainsData.domains) || [];
41+
const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords;
42+
4343
useEffect(() => {
4444
// console.log('appSettings.settings: ', appSettings && appSettings.settings);
4545
if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) {
4646
setNoScrapprtError(true);
4747
}
4848
}, [appSettings]);
4949

50-
// console.log('Domains Data:', router, activDomain, theKeywords);
50+
// console.log('Websites Data:', router, activDomain, theKeywords);
5151

5252
return (
5353
<div className="Domain ">

‎services/keywords.tsx

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,21 @@ import toast from 'react-hot-toast';
22
import { NextRouter } from 'next/router';
33
import { useMutation, useQuery, useQueryClient } from 'react-query';
44

5-
export const fetchKeywords = async (router: NextRouter) => {
6-
if (!router.query.slug) { return []; }
7-
const res = await fetch(`${window.location.origin}/api/keywords?domain=${router.query.slug}`, { method: 'GET' });
5+
export const fetchKeywords = async (router: NextRouter, domain: string) => {
6+
if (!domain) { return []; }
7+
const res = await fetch(`${window.location.origin}/api/keywords?domain=${domain}`, { method: 'GET' });
88
return res.json();
99
};
1010

11-
export function useFetchKeywords(router: NextRouter, setKeywordSPollInterval?:Function, keywordSPollInterval:undefined|number = undefined) {
11+
export function useFetchKeywords(
12+
router: NextRouter,
13+
domain: string,
14+
setKeywordSPollInterval?:Function,
15+
keywordSPollInterval:undefined|number = undefined,
16+
) {
1217
const { data: keywordsData, isLoading: keywordsLoading, isError } = useQuery(
13-
['keywords', router.query.slug],
14-
() => fetchKeywords(router),
18+
['keywords', domain],
19+
() => fetchKeywords(router, domain),
1520
{
1621
refetchInterval: keywordSPollInterval,
1722
onSuccess: (data) => {

‎utils/client/validators.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,14 @@ export const isValidDomain = (domain:string): boolean => {
3333

3434
return isValid;
3535
};
36+
37+
export const isValidUrl = (str: string) => {
38+
let url;
39+
40+
try {
41+
url = new URL(str);
42+
} catch (e) {
43+
return false;
44+
}
45+
return url.protocol === 'http:' || url.protocol === 'https:';
46+
};

‎utils/scraper.ts

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -187,15 +187,17 @@ export const extractScrapedResult = (content: string, device: string): SearchRes
187187

188188
/**
189189
* Find in the domain's position from the extracted search result.
190-
* @param {string} domain - Domain Name to look for.
190+
* @param {string} domainURL - URL Name to look for.
191191
* @param {SearchResult[]} result - The search result array extracted from the Google Search result.
192192
* @returns {SERPObject}
193193
*/
194-
export const getSerp = (domain:string, result:SearchResult[]) : SERPObject => {
195-
if (result.length === 0 || !domain) { return { postion: 0, url: '' }; }
194+
export const getSerp = (domainURL:string, result:SearchResult[]) : SERPObject => {
195+
if (result.length === 0 || !domainURL) { return { postion: 0, url: '' }; }
196+
const URLToFind = new URL(domainURL.includes('https://') ? domainURL : `https://${domainURL}`);
197+
const theURL = URLToFind.hostname + URLToFind.pathname;
196198
const foundItem = result.find((item) => {
197-
const itemDomain = item.url.replace('www.', '').match(/^(?:https?:)?(?:\/\/)?([^/?]+)/i);
198-
return itemDomain && itemDomain.includes(domain.replace('www.', ''));
199+
const itemURL = new URL(item.url.includes('https://') ? item.url : `https://${item.url}`);
200+
return theURL === itemURL.hostname + itemURL.pathname || `${theURL}/` === itemURL.hostname + itemURL.pathname;
199201
});
200202
return { postion: foundItem ? foundItem.position : 0, url: foundItem && foundItem.url ? foundItem.url : '' };
201203
};

0 commit comments

Comments
 (0)