|
| 1 | +import requests |
| 2 | +import json |
| 3 | +from os import getenv |
| 4 | +from time import sleep |
| 5 | +import re |
| 6 | +from configparser import ConfigParser |
| 7 | + |
| 8 | + |
| 9 | +VERIFIED = json.load(open('verified.json', "r+", encoding='utf-8')) |
| 10 | +INJECT = json.load(open('inject.json', "r+", encoding='utf-8')) |
| 11 | + |
| 12 | +RETRY_COUNT = 25 |
| 13 | + |
| 14 | +GH_TOKEN = getenv("GH_TOKEN") |
| 15 | +HEADERS = {"Authorization": f"token {GH_TOKEN}", "Accept": "application/vnd.github.v3+json", "User-Agent": "AntiCope/anticope.ml"} |
| 16 | + |
| 17 | +# regex |
| 18 | +FEATURE_RE = re.compile("(?:add\(new )([^(]+)(?:\([^)]*)\)\)") |
| 19 | +INVITE_RE = re.compile("((?:https?:\/\/)?(?:www.)?(?:discord.(?:gg|io|me|li|com)|discordapp.com\/invite|dsc.gg)\/[a-zA-z0-9-\/]+)") |
| 20 | +MCVER_RE = re.compile("(?:['\"]com\.mojang:minecraft:)([0-9a-z.]+)(?:[\"'])") |
| 21 | + |
| 22 | +def sleep_if_rate_limited(type="search"): |
| 23 | + for _ in range(RETRY_COUNT): |
| 24 | + try: |
| 25 | + r = requests.get("https://api.github.com/rate_limit", headers=HEADERS) |
| 26 | + if r.status_code != 304 and r.json()['resources'][type]['remaining'] > 0: |
| 27 | + return |
| 28 | + print("rate limited. sleeping...") |
| 29 | + except Exception: |
| 30 | + print("[rate limit] error. ignoring...") |
| 31 | + sleep(25) |
| 32 | + |
| 33 | +repos = set(VERIFIED) |
| 34 | + |
| 35 | +# Fetch all repo names that contain meteor entrypoint in fabric.mod.json |
| 36 | +incomplete = True |
| 37 | +page = 0 |
| 38 | +print("Fetching based on fabric.mod.json") |
| 39 | +while incomplete: |
| 40 | + print(f"Fetching page {page}") |
| 41 | + for _ in range(RETRY_COUNT): |
| 42 | + try: |
| 43 | + sleep_if_rate_limited() |
| 44 | + r = requests.get( |
| 45 | + f"https://api.github.com/search/code?q=entrypoints+meteor+extension:json+filename:fabric.mod.json+fork:true+in:file&per_page=100&page={page}", headers=HEADERS).json() |
| 46 | + if 'message' in r.keys() and "rate limit" in r['message']: |
| 47 | + print("[search fetch] rate limited. sleeping...") |
| 48 | + sleep(60) |
| 49 | + continue |
| 50 | + for file in r['items']: |
| 51 | + repo = file['repository'] |
| 52 | + if not repo['private']: |
| 53 | + repos.add(repo['full_name']) |
| 54 | + incomplete = len(r["items"]) != 0 |
| 55 | + break |
| 56 | + except Exception: |
| 57 | + print("[search fetch] error. ignoring...") |
| 58 | + page += 1 |
| 59 | + if page > 10: # fallback |
| 60 | + break |
| 61 | + |
| 62 | +# Fetch all repo names that extend MeteorAddon |
| 63 | +incomplete = True |
| 64 | +page = 0 |
| 65 | +print("Fetching based on extends MeteorAddon") |
| 66 | +while incomplete: |
| 67 | + print(f"Fetching page {page}") |
| 68 | + for _ in range(RETRY_COUNT): |
| 69 | + try: |
| 70 | + sleep_if_rate_limited() |
| 71 | + r = requests.get( |
| 72 | + f"https://api.github.com/search/code?q=extends+MeteorAddon+language:java+in:file&per_page=100&page={page}", headers=HEADERS).json() |
| 73 | + if 'message' in r.keys() and "rate limit" in r['message']: |
| 74 | + print("[search fetch] rate limited. sleeping...") |
| 75 | + sleep(60) |
| 76 | + continue |
| 77 | + for file in r['items']: |
| 78 | + repo = file['repository'] |
| 79 | + if not repo['private']: |
| 80 | + repos.add(repo['full_name']) |
| 81 | + incomplete = len(r["items"]) != 0 |
| 82 | + break |
| 83 | + except Exception: |
| 84 | + print("[search fetch] error. ignoring...") |
| 85 | + page += 1 |
| 86 | + if page > 10: # fallback |
| 87 | + break |
| 88 | + |
| 89 | +# Request all forks of templates because some people cant click generate |
| 90 | +r = requests.get("https://api.github.com/repos/MeteorDevelopment/meteor-addon-template/forks?per_page=100", headers=HEADERS).json() |
| 91 | +for fork in r: |
| 92 | + try: |
| 93 | + repos.add(fork['full_name']) |
| 94 | + except Exception: |
| 95 | + print("[fork fetch] error. ignoring...") |
| 96 | + |
| 97 | +# filter templates |
| 98 | +repos = list(filter(lambda x: "-addon-template" not in x.lower(), repos)) |
| 99 | + |
| 100 | +def parse_repo(name): |
| 101 | + sleep_if_rate_limited(type="core") |
| 102 | + print(f"parsing: {name}") |
| 103 | + |
| 104 | + repo = requests.get(f"https://api.github.com/repos/{name}", headers=HEADERS).json() |
| 105 | + fabric = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/resources/fabric.mod.json").json() |
| 106 | + |
| 107 | + # find authors from mod metadata or from github username |
| 108 | + authors = [] |
| 109 | + for author in fabric['authors']: |
| 110 | + if type(author) == str: |
| 111 | + authors.append(author) |
| 112 | + if len(authors) == 0: |
| 113 | + authors.append(repo['owner']['login']) |
| 114 | + |
| 115 | + links = {"github": repo['html_url']} |
| 116 | + |
| 117 | + if "meteor" not in fabric["entrypoints"].keys(): |
| 118 | + print("Missing meteor entrypoint") |
| 119 | + raise Exception("Missing meteor entrypoint") |
| 120 | + |
| 121 | + summary = "" |
| 122 | + try: |
| 123 | + summary = repo['description'] or fabric['description'] |
| 124 | + except Exception: |
| 125 | + print("[summary] error. ignoring...") |
| 126 | + |
| 127 | + # direct download from releases |
| 128 | + downloads = 0 |
| 129 | + try: |
| 130 | + releases = requests.get(f"https://api.github.com/repos/{name}/releases", headers=HEADERS).json() |
| 131 | + url = None |
| 132 | + for release in releases: |
| 133 | + for asset in release['assets']: |
| 134 | + asset_name: str = asset['name'].lower() |
| 135 | + if asset_name.endswith("-dev.jar") or asset_name.endswith("-sources.jar"): |
| 136 | + continue |
| 137 | + if asset_name.endswith(".jar"): |
| 138 | + url = asset['browser_download_url'] |
| 139 | + downloads = asset['download_count'] |
| 140 | + break |
| 141 | + if url != None: |
| 142 | + break |
| 143 | + if url == None: |
| 144 | + print("missing release") |
| 145 | + else: |
| 146 | + links["download"] = url |
| 147 | + except Exception: |
| 148 | + print("[dl] error. ignoring...") |
| 149 | + |
| 150 | + # icon from mod metadata |
| 151 | + icon = None |
| 152 | + try: |
| 153 | + icon = f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/resources/{fabric['icon']}" |
| 154 | + if requests.head(icon).status_code == 404: |
| 155 | + print("missing icon") |
| 156 | + icon = None |
| 157 | + except Exception: |
| 158 | + print("[icon] error. ignoring...") |
| 159 | + |
| 160 | + # find discord server by looking at readme mod and repository metadata |
| 161 | + try: |
| 162 | + readme = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/README.md").text |
| 163 | + invites = INVITE_RE.findall(readme) + INVITE_RE.findall(str(fabric)) + INVITE_RE.findall(str(repo)) |
| 164 | + for invite in invites: |
| 165 | + if requests.head(invite).status_code != 404: |
| 166 | + links["discord"] = invite |
| 167 | + break |
| 168 | + except Exception: |
| 169 | + print("[discord invite] error. ignoring...") |
| 170 | + |
| 171 | + try: |
| 172 | + site = repo['homepage'] |
| 173 | + if not INVITE_RE.match(site) and site: # skip discord invites |
| 174 | + links["homepage"] = site |
| 175 | + except Exception: |
| 176 | + print("[homepage] error. ignoring...") |
| 177 | + |
| 178 | + # find features by parsing the entrypoint |
| 179 | + features = [] |
| 180 | + feature_count = 0 |
| 181 | + try: |
| 182 | + entrypoint = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/src/main/java/{fabric['entrypoints']['meteor'][0].replace('.', '/')}.java").text |
| 183 | + features.extend([str(x) for x in FEATURE_RE.findall(entrypoint)]) |
| 184 | + feature_count = len(features) |
| 185 | + if len(features) > 50: |
| 186 | + count = len(features) - 50 |
| 187 | + features = features[:50] |
| 188 | + features.append(f"...and {count} more") |
| 189 | + except Exception: |
| 190 | + print("[features] error. ignoring...") |
| 191 | + |
| 192 | + # parse build.gradle |
| 193 | + mc_version = None |
| 194 | + try: |
| 195 | + build_gradle = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/build.gradle").text |
| 196 | + try: |
| 197 | + props = requests.get(f"https://raw.githubusercontent.com/{name}/{repo['default_branch']}/gradle.properties").text |
| 198 | + props = "[conf]\n"+props # convert to ini format |
| 199 | + gradle_props = ConfigParser() |
| 200 | + gradle_props.read_string(props) |
| 201 | + for key, val in dict(gradle_props['conf']).items(): |
| 202 | + build_gradle = build_gradle.replace("${project."+key+"}", val) |
| 203 | + build_gradle = build_gradle.replace(f"$project.{key}", val) |
| 204 | + build_gradle = build_gradle.replace(f"project.{key}", val) |
| 205 | + except Exception as ex: |
| 206 | + print(f"[build.gradle] failed to read gradle.properties.") |
| 207 | + mc_version = MCVER_RE.findall(build_gradle)[0] |
| 208 | + except Exception: |
| 209 | + print("[build.gradle] error. ignoring...") |
| 210 | + |
| 211 | + result = { |
| 212 | + "authors": authors, |
| 213 | + "features": features, |
| 214 | + "feature_count": feature_count, |
| 215 | + "icon": icon, |
| 216 | + "id": repo['full_name'], |
| 217 | + "links": links, |
| 218 | + "name": fabric['name'], |
| 219 | + "stars": repo['stargazers_count'], |
| 220 | + "last_update": repo['pushed_at'], |
| 221 | + "downloads": downloads, |
| 222 | + "mc_version": mc_version, |
| 223 | + "status": { |
| 224 | + "archived": repo['archived'] |
| 225 | + }, |
| 226 | + "verified": (repo['full_name'] in VERIFIED), |
| 227 | + "summary": summary |
| 228 | + } |
| 229 | + |
| 230 | + result.update(INJECT.get(repo['full_name'], {})) |
| 231 | + return result |
| 232 | + |
| 233 | +verified_json = [] |
| 234 | +unverified_json = [] |
| 235 | +for repo in repos: |
| 236 | + if repo in VERIFIED: |
| 237 | + try: |
| 238 | + verified_json.append(parse_repo(repo)) |
| 239 | + except Exception as ex: |
| 240 | + print(f"error {ex}. ignoring..., repo: {repo}") |
| 241 | + else: |
| 242 | + try: |
| 243 | + unverified_json.append(parse_repo(repo)) |
| 244 | + except Exception as ex: |
| 245 | + print(f"error {ex}. ignoring..., repo: {repo}") |
| 246 | + |
| 247 | +json.dump(verified_json, open("addons-ver.json", "w+", encoding='utf-8'), indent=None) |
| 248 | +json.dump(unverified_json, open("addons-unver.json", "w+", encoding='utf-8'), indent=None) |
0 commit comments