You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
367 lines
12 KiB
367 lines
12 KiB
#! /usr/bin/python3
|
|
|
|
from sys import argv
|
|
from itertools import chain, islice
|
|
from re import search, sub
|
|
from functools import total_ordering
|
|
|
|
from sylla import textbookInfo
|
|
from collections import MutableMapping
|
|
|
|
import datetime as dt
|
|
import lxml.html as lxh
|
|
import requests
|
|
import sys
|
|
import copy
|
|
|
|
fall = "2169"
|
|
spring_summer = "2175"
|
|
winter = "2171"
|
|
|
|
def parseSem(sem):
|
|
if sem == "TBA":
|
|
return "TBA"
|
|
return sem[0] + sem[2:4] + sem[6]
|
|
|
|
# threading stuff
|
|
import queue as q
|
|
import threading as thd
|
|
|
|
baseurl = "https://applicants.mcmaster.ca/psp/prepprd/EMPLOYEE/PSFT_LS/c/COMMUNITY_ACCESS.CLASS_SEARCH.GBL"
|
|
|
|
searchurl = "https://csprd.mcmaster.ca/psc/prcsprd/EMPLOYEE/PSFT_LS/c/COMMUNITY_ACCESS.CLASS_SEARCH.GBL"
|
|
|
|
custom_headers = {
|
|
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0",
|
|
"Content-Type" : "application/x-www-form-urlencoded; charset=UTF-8",
|
|
}
|
|
|
|
courseCodes1 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=CLASS_SRCH_WRK2_SSR_PB_SUBJ_SRCH%240&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=5tq9x%2Fjt42mf62Sh5z%2BrjxT0gT15kiIyQ2cecCSmRB4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={1}"
|
|
|
|
courseCodes2 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=SSR_CLSRCH_WRK2_SSR_ALPHANUM_{1}&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=vIUgl6ZXw045S07EPbQw4RDzv7NmKCDdJFdT4CTRQNM%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={2}"
|
|
|
|
payload2 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=%23ICSave&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=aWx3w6lJ6d2wZui6hwRVSEnzsPgCA3afYJEFBLLkxe4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={1}"
|
|
|
|
payload = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=CLASS_SRCH_WRK2_SSR_PB_CLASS_SRCH&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=aWx3w6lJ6d2wZui6hwRVSEnzsPgCA3afYJEFBLLkxe4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&SSR_CLSRCH_WRK_SUBJECT$75$$0={1}&CLASS_SRCH_WRK2_STRM$45$={2}"
|
|
|
|
|
|
year = dt.date.today().year
|
|
month = dt.date.today().month
|
|
|
|
days = {
|
|
"Mo" : 0,
|
|
"Tu" : 1,
|
|
"We" : 2,
|
|
"Th" : 3,
|
|
"Fr" : 4,
|
|
"Sa" : 5,
|
|
"Su" : 6
|
|
}
|
|
|
|
day_descs = {
|
|
"Mo" : "Monday Mon Mo",
|
|
"Tu" : "Tuesday Tues Tu Tue",
|
|
"We" : "Wednesday Wed We",
|
|
"Th" : "Thursday Th Thurs",
|
|
"Fr" : "Friday Fr Fri",
|
|
"Sa" : "Saturday Sat Sa",
|
|
"Su" : "Sunday Su Sun",
|
|
"T" : "TBA"
|
|
}
|
|
|
|
def timeparse(time):
|
|
"""
|
|
Parse the time into numbers
|
|
"""
|
|
if len(time) == 7:
|
|
hour = int(time[0:2])
|
|
minutes = int(time[3:5])
|
|
half = time[5:7]
|
|
else:
|
|
hour = int(time[0])
|
|
minutes = int(time[2:4])
|
|
half = time[4:6]
|
|
if half == "PM":
|
|
if hour < 12:
|
|
hour = hour + 12
|
|
|
|
return (str(hour), str(minutes), half)
|
|
|
|
class Class(object):
|
|
def __init__(self, dept, title, sections):
|
|
self.title = title.encode("UTF-8")
|
|
self.sections = sections
|
|
self.dept = dept
|
|
|
|
def __repr__(self):
|
|
return repr((self.title, self.sections))
|
|
|
|
def __iter__(self):
|
|
return iter((self.title, sec) for sec in self.sections)
|
|
|
|
def hasCode(self):
|
|
splitted = self.title.strip().split(" ")
|
|
return ((len(splitted) >= 2) and
|
|
(splitted[0].upper() == splitted[0]) and
|
|
(splitted[1].upper() == splitted[1]))
|
|
|
|
@property
|
|
def code(self):
|
|
if self.hasCode():
|
|
return self.title.strip().split(" ")[1].strip()
|
|
return False
|
|
|
|
@property
|
|
def books(self):
|
|
if self.dept and self.code:
|
|
return textbookInfo(self.dept, self.code, withPrices=True)
|
|
return False
|
|
|
|
@total_ordering
|
|
class Section(dict):
|
|
def __init__(self, time, loc, prof, sem):
|
|
self.time = time.encode("UTF-8")
|
|
self.loc = loc.encode("UTF-8")
|
|
self.prof = prof.encode("UTF-8")
|
|
self._sem = sem.encode("UTF-8")
|
|
self._date = False
|
|
self._day = False
|
|
|
|
@property
|
|
def sem(self):
|
|
if parseSem(self._sem) == fall:
|
|
return "Fall"
|
|
elif parseSem(self._sem) == winter:
|
|
return "Winter"
|
|
elif parseSem(self._sem) == "TBA":
|
|
return "TBA"
|
|
else:
|
|
return "Spring/Summer"
|
|
|
|
@property
|
|
def date(self):
|
|
if self.time != "TBA":
|
|
day, start, _, end = self.time.split()
|
|
|
|
if self._day:
|
|
assert len(self._day) == 2
|
|
day = self._day
|
|
else:
|
|
day = [day[n:n+2] for n in range(0, len(day)-1, 2)]
|
|
|
|
self._date = (day, timeparse(start), timeparse(end))
|
|
|
|
return self._date
|
|
|
|
return self.time
|
|
|
|
@property
|
|
def day(self):
|
|
return self.date[0]
|
|
|
|
@property
|
|
def start(self):
|
|
return self.date[1][0] + self.date[1][1]
|
|
|
|
def __repr__(self):
|
|
return ("""
|
|
Time = %s, Location = %s, Instructor = %s, Semester Running = %s
|
|
""" % (self.date, self.loc, self.prof, self.sem))
|
|
def __gt__(self, x):
|
|
if isinstance(self.day, list):
|
|
raise NotImplementedError
|
|
|
|
if (self.date == "TBA" or
|
|
x.date == "TBA"):
|
|
return False
|
|
|
|
return ((days[self.day] > days[x.day]) or
|
|
((self.day == x.day) and
|
|
(self.start > x.start)))
|
|
|
|
def __eq__(self, x):
|
|
return (x.date == self.date and
|
|
x.prof == self.prof and
|
|
x.loc == self.loc and
|
|
x.sem == self.sem)
|
|
|
|
|
|
def getStateNum(html):
|
|
"""
|
|
Get the state num from Mosaic
|
|
This is unique to each requester
|
|
"""
|
|
parsed = lxh.fromstring(html)
|
|
return parsed.xpath(".//input[@name=\"ICStateNum\"]")[0].value
|
|
|
|
def parseSection(section):
|
|
cols = section.xpath(".//td")
|
|
assert len(cols) == 4
|
|
time, loc, prof, sem = [col.text_content().encode("UTF-8").strip() for col in cols]
|
|
|
|
classinfo = Section(time, loc, prof, sem)
|
|
return classinfo
|
|
|
|
def getSectionInfo(table):
|
|
trs = table.xpath(".//tr")
|
|
for tr in trs:
|
|
if tr.xpath("@id") and search(r"SSR_CLSRCH", tr.xpath("@id")[0]):
|
|
yield parseSection(tr)
|
|
|
|
def parseColumns(subject, html):
|
|
parsed = lxh.fromstring(html)
|
|
|
|
classInfo = (list(getSectionInfo(table)) for table in
|
|
islice((table for table in parsed.xpath(".//table")
|
|
if table.xpath("@id") and
|
|
search(r"ICField[0-9]+\$scroll", table.xpath("@id")[0])), 1, sys.maxsize))
|
|
|
|
classNames = ((subject, span.text_content().strip()) for span in parsed.xpath(".//span")
|
|
if span.xpath("@id") and
|
|
search(r"DERIVED_CLSRCH_DESCR", span.xpath("@id")[0]))
|
|
|
|
return zip(classNames, classInfo)
|
|
|
|
def getCodes(html):
|
|
parsed = lxh.fromstring(html)
|
|
|
|
return (code.text_content().encode("UTF-8") for code in
|
|
parsed.xpath("//span")
|
|
if code.xpath("@id") and
|
|
search(r"SSR_CLSRCH_SUBJ_SUBJECT\$[0-9]+", code.xpath("@id")[0]))
|
|
|
|
class MosReq(object):
|
|
def __init__(self, semester):
|
|
self.semester = semester
|
|
s = requests.Session()
|
|
resp = s.get(baseurl, allow_redirects=True, headers=custom_headers).content
|
|
|
|
# Let the server set some cookies before doing the searching
|
|
cookies = {}
|
|
for key, val in s.cookies.items():
|
|
cookies[key] = val
|
|
self.cookies = cookies
|
|
self.statenum = False
|
|
self.codes_ = []
|
|
|
|
def getlist(self, subject):
|
|
sys.stderr.write("Getting %s\n" % subject.decode("UTF-8"))
|
|
first_req = requests.get(searchurl, cookies=self.cookies).content
|
|
# for some reason Mosaic wants us to request it twice, ??????????????????
|
|
self.statenum = getStateNum(first_req)
|
|
first_req = requests.post(searchurl,
|
|
data=payload.format(self.statenum, subject, self.semester),
|
|
cookies=self.cookies,
|
|
allow_redirects=False,
|
|
headers=custom_headers).content
|
|
# we make a first request to get the ICStateNum in case it thinks there are too many results
|
|
try:
|
|
self.statenum = getStateNum(first_req)
|
|
except IndexError:
|
|
pass
|
|
if b"Your search will return over" in first_req:
|
|
|
|
return requests.post(searchurl,
|
|
data=payload2.format(self.statenum, self.semester),
|
|
cookies=self.cookies,
|
|
allow_redirects=False,
|
|
headers=custom_headers).content
|
|
else:
|
|
return first_req
|
|
|
|
def classes(self, subject):
|
|
return list(parseColumns(subject, self.getlist(subject)))
|
|
|
|
def getCodes(self, letter):
|
|
sys.stderr.write("Getting letter " + letter + "\n")
|
|
first_req = requests.get(searchurl, cookies=self.cookies).content
|
|
self.statenum = getStateNum(first_req)
|
|
|
|
self.statenum = getStateNum(requests.post(searchurl,
|
|
data=courseCodes1.format(self.statenum, self.semester),
|
|
cookies=self.cookies,
|
|
headers=custom_headers).content)
|
|
|
|
return getCodes(requests.post(searchurl,
|
|
data=courseCodes2.format(self.statenum, letter, self.semester),
|
|
cookies=self.cookies,
|
|
allow_redirects=False,
|
|
headers=custom_headers).content)
|
|
@property
|
|
def codes(self):
|
|
if not self.codes_:
|
|
self.codes_ = list(chain.from_iterable(
|
|
map((lambda l:
|
|
self.getCodes(chr(l))),
|
|
range(65, 91))))
|
|
return self.codes_
|
|
|
|
def request(codes, lists, semester):
|
|
requester = MosReq(semester)
|
|
while not codes.empty():
|
|
code = codes.get()
|
|
try:
|
|
lists.put(requester.classes(code))
|
|
except exception:
|
|
print(exception)
|
|
codes.task_done()
|
|
return
|
|
codes.task_done()
|
|
|
|
class CourseInfo(object):
|
|
def __init__(self, threadcount, semester):
|
|
self._codes = False
|
|
self.threadcount = threadcount
|
|
self.semester = semester
|
|
|
|
@property
|
|
def codes(self):
|
|
if not self._codes:
|
|
req = MosReq(self.semester)
|
|
self._codes = req.codes
|
|
return self._codes
|
|
|
|
def classes(self):
|
|
qcodes = q.Queue()
|
|
for code in self.codes:
|
|
qcodes.put(code)
|
|
lists = q.Queue()
|
|
threads = []
|
|
thread = None
|
|
for i in range(self.threadcount):
|
|
thread = thd.Thread(group=None, target=request, args=(qcodes, lists, self.semester))
|
|
threads.append(thread)
|
|
thread.start()
|
|
qcodes.join()
|
|
for t in threads:
|
|
t.join()
|
|
|
|
sections = []
|
|
while not lists.empty():
|
|
sections.append(lists.get())
|
|
|
|
for cl in chain.from_iterable(sections):
|
|
new_sections = []
|
|
for sec in cl[1]:
|
|
print(sec)
|
|
if len(sec.day) > 1:
|
|
for day in sec.day:
|
|
new_sections.append(copy.deepcopy(sec))
|
|
new_sections[-1]._day = day
|
|
else:
|
|
sec._day = sec.day[0]
|
|
new_sections.append(sec)
|
|
yield Class(cl[0][0], sub("\xa0+", "", cl[0][1]), sorted(new_sections))
|
|
|
|
def getCourses(semester, threadcount=10):
|
|
return CourseInfo(threadcount, semester).classes()
|
|
|
|
def allCourses():
|
|
return chain.from_iterable(
|
|
(getCourses(sem, threadcount=25)
|
|
for sem in (fall, winter, spring_summer)))
|
|
|
|
if __name__ == "__main__":
|
|
for course in allCourses():
|
|
sys.stdout.write("%s, %s, %s, %s\n" % (course.title, course.code, course.dept, course.books))
|
|
print(course.sections)
|
|
|