diff --git a/codedigger/codechef/cron.py b/codedigger/codechef/cron.py index 1f79687..6301676 100644 --- a/codedigger/codechef/cron.py +++ b/codedigger/codechef/cron.py @@ -4,7 +4,7 @@ def update_AllContests(): # Creates new contests and problems in Database - all_contests = ContestData('past') + all_contests = ContestData('all', 'past') for contest in all_contests: create_or_update_codechefContest(contest) contest_problems_info = ProblemData(contest['ContestCode']) diff --git a/codedigger/codechef/model_utils.py b/codedigger/codechef/model_utils.py index bfba88f..d886b02 100644 --- a/codedigger/codechef/model_utils.py +++ b/codedigger/codechef/model_utils.py @@ -6,16 +6,19 @@ def create_or_update_codechefProblem(problemdata): for problem in problemdata: Prob, created = Problem.objects.get_or_create( - name=problem['Name'], prob_id=problem['ProblemCode'], - url=problem['ProblemURL'], - contest_id=problem['ContestId'], - platform=problem['Platform']) + platform=problem['Platform'], + defaults={ + 'name': problem['Name'], + 'url': problem['ProblemURL'], + 'contest_id': problem['ContestId'], + }, + ) + cont = CodechefContest.objects.get(contestId=problem['ContestId']) - prob = Problem.objects.get(prob_id=problem['ProblemCode'], - contest_id=problem['ContestId']) + ccprob, created = CodechefContestProblems.objects.get_or_create( - contest=cont, problem=prob) + contest=cont, problem=Prob) def create_or_update_codechefContest(contest): diff --git a/codedigger/codechef/scraper.py b/codedigger/codechef/scraper.py index baf087f..23fe9aa 100644 --- a/codedigger/codechef/scraper.py +++ b/codedigger/codechef/scraper.py @@ -21,7 +21,8 @@ def divisionScraper(contest_id): def contestScraper(offset, contest_type): - query_contest_url = f"https://www.codechef.com/api/list/contests/{contest_type}?sort_by=START&sorting_order=desc&offset={offset}&mode=premium" + query_contest_url = f"https://www.codechef.com/api/list/contests/" + contest_type + "?sort_by=START&sorting_order=desc&offset=" + str( + offset) + "&mode=premium" # Query URL might change in future. contest_data = requests.get(query_contest_url) @@ -46,104 +47,19 @@ def problemScraper(contest_code): return problem_data -def UserSubmissionDetail(problemcode, contest, user): - URL = f"https://www.codechef.com/{contest}/status/{problemcode},{user}" +def UserSubmissionScraper(URL): + r = requests.get(URL) soup = BeautifulSoup(r.content, 'html5lib') - problemTable = soup.findAll('table', class_="dataTable") - problemRow = problemTable[0].findAll('tr') - problemRow.pop(0) - submissionlist = [] - if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity': - return submissionlist - - for problem in problemRow: - baseurl = "https://www.codechef.com" - problemDetails = problem.findAll('td') - subid = problemDetails[0].text - subtime = problemDetails[1].text - verdict = problemDetails[3].find('span').get('title') - if len(verdict) == 0: - verdict = (problemDetails[3].find('span').text) - verdict = verdict[:verdict.index('[')] - if int(verdict) == 100: - verdict = "accepted [100/100]" - else: - verdict = "partially accepted [" + verdict + "/100]" - lang = problemDetails[6].text - link = baseurl + problemDetails[7].find('a').get('href') - - subformat = { - 'subid': subid, - 'subtime': subtime, - 'verdict': verdict, - 'lang': lang, - 'link': link, - } - - submissionlist.append(subformat) - - return submissionlist - - -def recentSubmissions(userid): - URL = f"https://www.codechef.com/recent/user?user_handle={userid}" + return soup + + +def recentSubmissionScraper(user_handle): + + URL = f"https://www.codechef.com/recent/user?user_handle={user_handle}" r = requests.get(URL) - r = BeautifulSoup(r.content, 'html5lib') - recentSubs = r.findAll('tbody') - - recentlist = [] - for sub in recentSubs: - subd = sub.findAll('tr') - subd.pop(-1) - try: - query = subd[0].text[:18] - except: - query = "Profile found successfully" - if query == 'No Recent Activity': - break - - for prob in subd: - baseurl = "https://www.codechef.com" - det = prob.findAll('td') - - probid = det[1].find('a').text - probid = probid[:probid.index('<')] - - link = det[1].find('a').get('href') - link = link.replace("\\", "") - link = baseurl + link - - subtime = prob.find('span', class_="tooltiptext") - try: - subtime = subtime.text - subtime = subtime[:subtime.index('<')] - except: - break - - verdict = det[2].find('span').get('title') - if len(verdict) == 0: - verdict = (det[2].find('span').text) - verdict = verdict[:verdict.index('[')] - if int(verdict) == 100: - verdict = "accepted [100/100]" - else: - verdict = "partially accepted [" + verdict + "/100]" - - lang = det[3].text - lang = lang[:lang.index('<')] - - subformat = { - 'probid': probid, - 'subtime': subtime, - 'verdict': verdict, - 'lang': lang, - 'link': link, - } - - recentlist.append(subformat) - - return recentlist + soup = BeautifulSoup(r.content, 'html5lib') + return soup def profilePageScraper(user_handle): diff --git a/codedigger/codechef/scraper_utils.py b/codedigger/codechef/scraper_utils.py index d063b1d..49cc913 100644 --- a/codedigger/codechef/scraper_utils.py +++ b/codedigger/codechef/scraper_utils.py @@ -1,4 +1,4 @@ -from codechef.scraper import contestScraper, problemScraper, profilePageScraper +from codechef.scraper import contestScraper, problemScraper, profilePageScraper, recentSubmissionScraper, UserSubmissionScraper def OffsetLoader(contest_type): @@ -30,9 +30,9 @@ def getContestDivision(contest_id): return subcontests -def ContestData(type): +def ContestData(cont_type, cont_time): - contests_data = OffsetLoader(type) + contests_data = OffsetLoader(cont_time) all_contests = [] dateDict = { "Jan": "January", @@ -48,7 +48,35 @@ def ContestData(type): "Nov": "November", "Dec": "December" } + + longContestCode = [ + 'JAN', 'FEB', 'MARCH', 'APRIL', 'MAY', 'JUNE', 'JULY', 'AUG', 'SEPT', + 'OCT', 'NOV', 'DEC' + ] + for contest in contests_data: + + if cont_type == 'starters': + if contest['contest_code'][:5] != 'START': + continue + + if cont_type == 'lunchtime': + if contest['contest_code'][:5] != 'LTIME': + continue + + if cont_type == 'cookoff': + if contest['contest_code'][:4] != 'COOK': + continue + + if cont_type == 'long': + found = 0 + for month in longContestCode: + if contest['contest_code'][:len(month)] == month: + found = 1 + break + if found == 0: + continue + childContests = getContestDivision(contest['contest_code']) for contest_id in childContests: @@ -174,3 +202,139 @@ def userScraper(user_handle): # user.country_rank = country_rank # user.global_rank = global_rank # user.save() + + +def RecentSubmission(user_handle): + soup = recentSubmissionScraper(user_handle) + recentSubs = soup.findAll('tbody') + + recentlist = [] + for sub in recentSubs: + subd = sub.findAll('tr') + subd.pop(-1) + try: + query = subd[0].text[:18] + except: + query = "Profile found successfully" + if query == 'No Recent Activity': + break + + for prob in subd: + baseurl = "https://www.codechef.com" + det = prob.findAll('td') + + probid = det[1].find('a').text + probid = probid[:probid.index('<')] + + link = det[1].find('a').get('href') + link = link.replace("\\", "") + link = baseurl + link + + subtime = prob.find('span', class_="tooltiptext") + try: + subtime = subtime.text + subtime = subtime[:subtime.index('<')] + except: + break + + subtime = subtime.replace("\\", "") + + verdict = det[2].find('span').get('title') + if len(verdict) == 0: + verdict = (det[2].find('span').text) + verdict = verdict[:verdict.index('[')] + if int(verdict) == 100: + verdict = "accepted [100/100]" + else: + verdict = "partially accepted [" + verdict + "/100]" + + lang = det[3].text + lang = lang[:lang.index('<')] + + subformat = { + 'probid': probid, + 'subtime': subtime, + 'verdict': verdict, + 'lang': lang, + 'link': link, + } + + recentlist.append(subformat) + + return recentlist + + +def allProblemsSolved(user_handle): + + soup = profilePageScraper(user_handle) + print(user_handle) + problems_solved = [] + + all_contests = soup.find('article') + contests_list = all_contests.find_all('p') + cont = (contests_list[0].find('strong').contents)[0][:-1] + + # if cont == "Practice": + # probs = contests_list[0].find_all('a') + # for prob in probs: + # upsolved_problems.append(prob.contents[0]) + + probs = all_contests.find_all('a') + for prob in probs: + link = prob['href'] + name = prob.contents[0] + + problems_solved.append((name, link)) + + return problems_solved + + +def UserSubmissionDetails(problemcode, user_handle): + + url = "" + solvedByUser = allProblemsSolved(user_handle) + + for solved in solvedByUser: + if solved[0] == problemcode: + url = solved[1] + break + + if len(url) == 0: + return [] + + baseurl = f'https://www.codechef.com/' + soup = UserSubmissionScraper(baseurl + url) + problemTable = soup.findAll('table', class_="dataTable") + problemRow = problemTable[0].findAll('tr') + problemRow.pop(0) + submissionlist = [] + if len(problemRow) == 0 or problemRow[0].text == 'No Recent Activity': + return submissionlist + + for problem in problemRow: + baseurl = "https://www.codechef.com" + problemDetails = problem.findAll('td') + subid = problemDetails[0].text + subtime = problemDetails[1].text + verdict = problemDetails[3].find('span').get('title') + if len(verdict) == 0: + verdict = (problemDetails[3].find('span').text) + verdict = verdict[:verdict.index('[')] + if int(verdict) == 100: + verdict = "accepted [100/100]" + else: + verdict = "partially accepted [" + verdict + "/100]" + lang = problemDetails[6].text + link = baseurl + problemDetails[7].find('a').get('href') + + subformat = { + 'subid': subid, + 'subtime': subtime, + 'verdict': verdict, + 'lang': lang, + 'link': link, + } + + submissionlist.append(subformat) + + return submissionlist diff --git a/codedigger/codechef/urls.py b/codedigger/codechef/urls.py index d95c84f..565f2fc 100644 --- a/codedigger/codechef/urls.py +++ b/codedigger/codechef/urls.py @@ -5,5 +5,13 @@ path('upsolve', views.CodechefUpsolveAPIView.as_view(), name="codechef-upsolve"), + path('recentsub/', + views.CodechefRecentSubmissionAPIView.as_view()), + path('problemsub//', + views.CodechefUserSubmissionAPIView.as_view()), + path('contestproblems/', + views.CodechefContestProblemsAPIView.as_view()), + path('contests//', + views.CodechefContestsAPIView.as_view()), path('testing', views.testing) ] diff --git a/codedigger/codechef/views.py b/codedigger/codechef/views.py index 01e2669..c0960dd 100644 --- a/codedigger/codechef/views.py +++ b/codedigger/codechef/views.py @@ -5,9 +5,11 @@ from user.exception import ValidationException from .models import CodechefContest from .serializers import CodechefUpsolveSerializer -from .scraper_utils import contestgivenScrapper, problems_solved + +from .scraper_utils import contestgivenScrapper, problems_solved, RecentSubmission, UserSubmissionDetails, ContestData from user.models import Profile from problem.utils import getqs, get_total_page, get_upsolve_response_dict + from codechef.cron import * # Create your views here. @@ -56,6 +58,70 @@ def get(self, request): return Response(res) +class CodechefRecentSubmissionAPIView(generics.GenericAPIView): + + def get(self, request, username): + + handle = request.GET.get('handle', username) + if handle == None: + raise ValidationException( + 'Any of handle or Bearer Token is required.') + + result = RecentSubmission(handle) + + return Response({'status': 'OK', 'result': result}) + + +class CodechefUserSubmissionAPIView(generics.GenericAPIView): + + def get(self, request, username, problem): + + handle = request.GET.get('handle', username) + problemcode = request.GET.get('problemcode', problem) + if handle == None: + raise ValidationException( + 'Any of handle or Bearer Token is required.') + + if problemcode == None: + raise ValidationException('Any of valid problem code is required.') + + result = UserSubmissionDetails(problemcode, handle) + + return Response({'status': 'OK', 'result': result}) + + +class CodechefContestProblemsAPIView(generics.GenericAPIView): + + def get(self, request, contest): + + contest_id = request.GET.get('contest_id', contest) + + if contest_id == None: + raise ValidationException('A valid Contest ID is required.') + + result = ProblemData(contest_id) + + return Response({'status': 'OK', 'result': result}) + + +class CodechefContestsAPIView(generics.GenericAPIView): + + def get(self, request, time, typec): + + ctime = request.GET.get('ctime', time) + ctype = request.GET.get('ctype', typec) + if ((ctime != 'past') & (ctime != 'present') & (ctime != 'future')): + raise ValidationException( + 'A valid parameter for contest time is required') + + if ((ctype != 'lunchtime') & (ctime != 'cookoff') & + (ctime != 'starters') & (ctime != 'long')): + ctype = 'all' + + result = ContestData(ctype, ctime) + return Response({'status': 'OK', 'result': result}) + + def testing(request): update_AllContests() return HttpResponse("Successfully Scrapped!")