Toph-Server/fetchLeaderboard.py at main · IamTheSadik/Toph-Server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
import httpx

import json

from logger import getLogger
from customTypes import Function
from getProblems import getAllProblemUrls
from helper import makeBulkRequests, findLeaderboard, dumpData

logger = getLogger(__name__)


async def fetchLeaderboard(req: Function, ses: httpx.AsyncClient):
    allProblems = await getAllProblemUrls(req, ses)
    allProblems = sorted(set(allProblems))
    unsolvedProblems = []

    problemResponses = await makeBulkRequests(allProblems, req, ses)

    leaderboardData = []

    for resp in problemResponses:
        if isinstance(resp, Exception):
            logger.error(f"Error while fetching leaderboard: {resp.url}")
            continue
        try:
            x = findLeaderboard(resp)
            leaderboardData.append(x)
        except ValueError:
            url = str(resp.url).split("/")[-1]  # only the problem name
            unsolvedProblems.append(url)

    logger.info("Fetching leaderboard done")
    logger.info("Dumping unsolved problems")
    dumpData(unsolvedProblems, "Data/unsolved.json")
    unsolvedProblems.clear()  # clear from memory

    final = {"fastest": {}, "lightest": {}, "shortest": {}}
    for _, url, fastestSolver, lightestSolver, shortestSolver in leaderboardData:
        # Update Fastest
        if fastestSolver in final["fastest"]:
            final["fastest"][fastestSolver]["count"] += 1
            final["fastest"][fastestSolver]["urls"].append(url)

        else:
            final["fastest"][fastestSolver] = {"count": 1, "urls": [url]}

        # Update Lightest
        if lightestSolver in final["lightest"]:
            final["lightest"][lightestSolver]["count"] += 1
            final["lightest"][lightestSolver]["urls"].append(url)

        else:
            final["lightest"][lightestSolver] = {"count": 1, "urls": [url]}

        # Update Shortest
        if shortestSolver in final["shortest"]:
            final["shortest"][shortestSolver]["count"] += 1
            final["shortest"][shortestSolver]["urls"].append(url)

        else:
            final["shortest"][shortestSolver] = {"count": 1, "urls": [url]}

    logger.info("Sorting leaderboard")

    leaderboardData = final.copy()
    final.clear()
    extras = ["Safin01", "anonyo.akand"]
    extrasData = {i:leaderboardData["shortest"].get(i) for i in extras}
    # imposter = leaderboardData["shortest"].get("anonyo.akand")
    # Sort the leaderboard
    n = 19  # Top 19
    for k, v in leaderboardData.items():
        sortedList = sorted(v.items(), key=lambda x: -x[1]["count"])
        leaderboardData[k] = dict(sortedList[:n])

    # leaderboardData["shortest"]["anonyo.akand"] = imposter
    for user, value in extrasData.items():
        leaderboardData["shortest"][user] = value


    fastestUsers = {}
    lightestUsers = {}
    shortestUsers = {}

    for i, j in leaderboardData.items():
        if i == "fastest":
            for u, d in j.items():
                fastestUsers[u] = d["urls"]
                count = d["count"]
                leaderboardData[i][u] = count

        if i == "lightest":
            for u, d in j.items():
                lightestUsers[u] = d["urls"]
                count = d["count"]
                leaderboardData[i][u] = count

        if i == "shortest":
            for u, d in j.items():
                shortestUsers[u] = d["urls"]
                count = d["count"]
                leaderboardData[i][u] = count

    allUsers = {
        "fastest": fastestUsers,
        "lightest": lightestUsers,
        "shortest": shortestUsers,
    }
    dumpData(leaderboardData, "Data/leaderboard.json")
    dumpData(allUsers, "Data/users/all.json")
    dumpData(fastestUsers, "Data/users/fastest.json")
    dumpData(lightestUsers, "Data/users/lightest.json")
    dumpData(shortestUsers, "Data/users/shortest.json")

    logger.info("Dumped leaderboard data")