2 changed files with 18 additions and 385 deletions
@ -1,358 +0,0 @@ |
|||
#! /usr/bin/python2 |
|||
|
|||
from sys import argv |
|||
from itertools import chain, islice |
|||
from re import search, sub |
|||
from functools import total_ordering |
|||
|
|||
from sylla2 import textbookInfo |
|||
from collections import MutableMapping |
|||
|
|||
import datetime as dt |
|||
import lxml.html as lxh |
|||
import requests |
|||
import logging |
|||
import sys |
|||
import copy |
|||
|
|||
fall = "2169" |
|||
spring_summer = "2175" |
|||
winter = "2171" |
|||
|
|||
# threading stuff |
|||
import Queue as q |
|||
import threading as thd |
|||
|
|||
baseurl = "https://applicants.mcmaster.ca/psp/prepprd/EMPLOYEE/PSFT_LS/c/COMMUNITY_ACCESS.CLASS_SEARCH.GBL" |
|||
|
|||
searchurl = "https://csprd.mcmaster.ca/psc/prcsprd/EMPLOYEE/PSFT_LS/c/COMMUNITY_ACCESS.CLASS_SEARCH.GBL" |
|||
|
|||
custom_headers = { |
|||
"User-Agent" : "Mozilla/5.0 (X11; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0", |
|||
"Content-Type" : "application/x-www-form-urlencoded; charset=UTF-8", |
|||
} |
|||
|
|||
courseCodes1 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=CLASS_SRCH_WRK2_SSR_PB_SUBJ_SRCH%240&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=5tq9x%2Fjt42mf62Sh5z%2BrjxT0gT15kiIyQ2cecCSmRB4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={1}" |
|||
|
|||
courseCodes2 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=SSR_CLSRCH_WRK2_SSR_ALPHANUM_{1}&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=vIUgl6ZXw045S07EPbQw4RDzv7NmKCDdJFdT4CTRQNM%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={2}" |
|||
|
|||
payload2 = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=%23ICSave&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=aWx3w6lJ6d2wZui6hwRVSEnzsPgCA3afYJEFBLLkxe4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&CLASS_SRCH_WRK2_STRM$45$={1}" |
|||
|
|||
payload = "ICAJAX=1&ICNAVTYPEDROPDOWN=1&ICType=Panel&ICElementNum=0&ICStateNum={0}&ICAction=CLASS_SRCH_WRK2_SSR_PB_CLASS_SRCH&ICXPos=0&ICYPos=0&ResponsetoDiffFrame=-1&TargetFrameName=None&FacetPath=None&ICFocus=&ICSaveWarningFilter=0&ICChanged=-1&ICResubmit=0&ICSID=aWx3w6lJ6d2wZui6hwRVSEnzsPgCA3afYJEFBLLkxe4%3D&ICActionPrompt=false&ICFind=&ICAddCount=&ICAPPCLSDATA=&SSR_CLSRCH_WRK_SUBJECT$75$$0={1}&CLASS_SRCH_WRK2_STRM$45$={2}" |
|||
|
|||
|
|||
year = dt.date.today().year |
|||
month = dt.date.today().month |
|||
|
|||
days = { |
|||
"Mo" : 0, |
|||
"Tu" : 1, |
|||
"We" : 2, |
|||
"Th" : 3, |
|||
"Fr" : 4, |
|||
"Sa" : 5, |
|||
"Su" : 6 |
|||
} |
|||
|
|||
day_descs = { |
|||
"Mo" : "Monday Mon Mo", |
|||
"Tu" : "Tuesday Tues Tu Tue", |
|||
"We" : "Wednesday Wed We", |
|||
"Th" : "Thursday Th Thurs", |
|||
"Fr" : "Friday Fr Fri", |
|||
"Sa" : "Saturday Sat Sa", |
|||
"Su" : "Sunday Su Sun", |
|||
"T" : "TBA" |
|||
} |
|||
|
|||
def timeparse(time): |
|||
""" |
|||
Parse the time into numbers |
|||
""" |
|||
if len(time) == 7: |
|||
hour = int(time[0:2]) |
|||
minutes = int(time[3:5]) |
|||
half = time[5:7] |
|||
else: |
|||
hour = int(time[0]) |
|||
minutes = int(time[2:4]) |
|||
half = time[4:6] |
|||
if half == "PM": |
|||
if hour < 12: |
|||
hour = hour + 12 |
|||
|
|||
return (str(hour), str(minutes), half) |
|||
|
|||
class Class(object): |
|||
def __init__(self, dept, title, sections): |
|||
self.title = title.encode("UTF-8") |
|||
self.sections = sections |
|||
self.dept = dept |
|||
|
|||
def __repr__(self): |
|||
return repr((self.title, self.sections)) |
|||
|
|||
def __iter__(self): |
|||
return iter((self.title, sec) for sec in self.sections) |
|||
|
|||
def hasCode(self): |
|||
splitted = self.title.strip().split(" ") |
|||
return ((len(splitted) >= 2) and |
|||
(splitted[0].upper() == splitted[0]) and |
|||
(splitted[1].upper() == splitted[1])) |
|||
|
|||
@property |
|||
def code(self): |
|||
if self.hasCode(): |
|||
return self.title.strip().split(" ")[1].strip() |
|||
return False |
|||
|
|||
@property |
|||
def books(self): |
|||
if self.dept and self.code: |
|||
return textbookInfo(self.dept, self.code, withPrices=True) |
|||
return False |
|||
|
|||
@total_ordering |
|||
class Section(dict): |
|||
def __init__(self, time, loc, prof, sem): |
|||
self.time = time.encode("UTF-8") |
|||
self.loc = loc.encode("UTF-8") |
|||
self.prof = prof.encode("UTF-8") |
|||
self._sem = sem.encode("UTF-8") |
|||
self._date = False |
|||
self._day = False |
|||
|
|||
@property |
|||
def sem(self): |
|||
if self._sem == fall: |
|||
return "Fall" |
|||
elif self._sem == winter: |
|||
return "Winter" |
|||
else: |
|||
return "Spring/Summer" |
|||
|
|||
@property |
|||
def date(self): |
|||
if self.time != "TBA": |
|||
day, start, _, end = self.time.split() |
|||
|
|||
if self._day: |
|||
assert len(self._day) == 2 |
|||
day = self._day |
|||
else: |
|||
day = [day[n:n+2] for n in range(0, len(day)-1, 2)] |
|||
|
|||
self._date = (day, timeparse(start), timeparse(end)) |
|||
|
|||
return self._date |
|||
|
|||
return self.time |
|||
|
|||
@property |
|||
def day(self): |
|||
return self.date[0] |
|||
|
|||
@property |
|||
def start(self): |
|||
return self.date[1][0] + self.date[1][1] |
|||
|
|||
def __repr__(self): |
|||
return (""" |
|||
Time = %s, Location = %s, Instructor = %s, Semester Running = %s |
|||
""" % (self.date, self.loc, self.prof, self.sem)) |
|||
def __gt__(self, x): |
|||
if isinstance(self.day, list): |
|||
raise NotImplementedError |
|||
|
|||
if (self.date == "TBA" or |
|||
x.date == "TBA"): |
|||
return False |
|||
|
|||
return ((days[self.day] > days[x.day]) or |
|||
((self.day == x.day) and |
|||
(self.start > x.start))) |
|||
|
|||
def __eq__(self, x): |
|||
return (x.date == self.date and |
|||
x.prof == self.prof and |
|||
x.loc == self.loc and |
|||
x.sem == self.sem) |
|||
|
|||
|
|||
def getStateNum(html): |
|||
""" |
|||
Get the state num from Mosaic |
|||
This is unique to each requester |
|||
""" |
|||
parsed = lxh.fromstring(html) |
|||
return parsed.xpath(".//input[@name=\"ICStateNum\"]")[0].value |
|||
|
|||
def parseSection(section): |
|||
cols = section.xpath(".//td") |
|||
assert len(cols) == 4 |
|||
time, loc, prof, sem = [col.text_content().encode("UTF-8").strip() for col in cols] |
|||
|
|||
classinfo = Section(time, loc, prof, sem) |
|||
return classinfo |
|||
|
|||
def getSectionInfo(table): |
|||
trs = table.xpath(".//tr") |
|||
for tr in trs: |
|||
if tr.xpath("@id") and search(r"SSR_CLSRCH", tr.xpath("@id")[0]): |
|||
yield parseSection(tr) |
|||
|
|||
def parseColumns(subject, html): |
|||
parsed = lxh.fromstring(html) |
|||
|
|||
classInfo = (list(getSectionInfo(table)) for table in |
|||
islice((table for table in parsed.xpath(".//table") |
|||
if table.xpath("@id") and |
|||
search(r"ICField[0-9]+\$scroll", table.xpath("@id")[0])), 1, sys.maxsize)) |
|||
|
|||
classNames = ((subject, span.text_content().strip()) for span in parsed.xpath(".//span") |
|||
if span.xpath("@id") and |
|||
search(r"DERIVED_CLSRCH_DESCR", span.xpath("@id")[0])) |
|||
|
|||
return list(zip(classNames, classInfo)) |
|||
|
|||
def getCodes(html): |
|||
parsed = lxh.fromstring(html) |
|||
|
|||
return (code.text_content().encode("UTF-8") for code in |
|||
parsed.xpath("//span") |
|||
if code.xpath("@id") and |
|||
search(r"SSR_CLSRCH_SUBJ_SUBJECT\$[0-9]+", code.xpath("@id")[0])) |
|||
|
|||
class MosReq(object): |
|||
def __init__(self, semester): |
|||
self.semester = semester |
|||
s = requests.Session() |
|||
resp = s.get(baseurl, allow_redirects=True, headers=custom_headers).content |
|||
|
|||
# Let the server set some cookies before doing the searching |
|||
cookies = {} |
|||
for key, val in s.cookies.items(): |
|||
cookies[key] = val |
|||
self.cookies = cookies |
|||
self.statenum = False |
|||
self.codes_ = [] |
|||
|
|||
def getlist(self, subject): |
|||
sys.stderr.write("Getting %s\n" % subject.decode("UTF-8")) |
|||
first_req = requests.get(searchurl, cookies=self.cookies) |
|||
# for some reason Mosaic wants us to request it twice, ?????????????????? |
|||
self.statenum = getStateNum(first_req.content) |
|||
first_req = requests.post(searchurl, |
|||
data=payload.format(self.statenum, subject, self.semester), |
|||
cookies=self.cookies, |
|||
allow_redirects=False, |
|||
headers=custom_headers) |
|||
print("HEADERS") |
|||
print(first_req.request.headers) |
|||
# we make a first request to get the ICStateNum in case it thinks there are too many results |
|||
try: |
|||
self.statenum = getStateNum(first_req.content) |
|||
except IndexError: |
|||
pass |
|||
if b"Your search will return over" in first_req: |
|||
|
|||
return requests.post(searchurl, |
|||
data=payload2.format(self.statenum, self.semester), |
|||
cookies=self.cookies, |
|||
allow_redirects=False, |
|||
headers=custom_headers).content |
|||
else: |
|||
return first_req |
|||
|
|||
def classes(self, subject): |
|||
return list(parseColumns(subject, self.getlist(subject))) |
|||
|
|||
def getCodes(self, letter): |
|||
sys.stderr.write("Getting letter " + letter + "\n") |
|||
first_req = requests.get(searchurl, cookies=self.cookies).content |
|||
self.statenum = getStateNum(first_req) |
|||
|
|||
self.statenum = getStateNum(requests.post(searchurl, |
|||
data=courseCodes1.format(self.statenum, self.semester), |
|||
cookies=self.cookies, |
|||
headers=custom_headers).content) |
|||
|
|||
return getCodes(requests.post(searchurl, |
|||
data=courseCodes2.format(self.statenum, letter, self.semester), |
|||
cookies=self.cookies, |
|||
allow_redirects=False, |
|||
headers=custom_headers).content) |
|||
@property |
|||
def codes(self): |
|||
if not self.codes_: |
|||
self.codes_ = list(chain.from_iterable( |
|||
list(map((lambda l: |
|||
self.getCodes(chr(l))), |
|||
range(65, 91))))) |
|||
return self.codes_ |
|||
|
|||
def request(codes, lists, semester): |
|||
requester = MosReq(semester) |
|||
while not codes.empty(): |
|||
code = codes.get() |
|||
lists.put(requester.classes(code)) |
|||
codes.task_done() |
|||
|
|||
class CourseInfo(object): |
|||
def __init__(self, threadcount, semester): |
|||
self._codes = False |
|||
self.threadcount = threadcount |
|||
self.semester = semester |
|||
|
|||
@property |
|||
def codes(self): |
|||
if not self._codes: |
|||
req = MosReq(self.semester) |
|||
self._codes = req.codes |
|||
return self._codes |
|||
|
|||
def classes(self): |
|||
qcodes = q.Queue() |
|||
for code in self.codes: |
|||
qcodes.put(code) |
|||
lists = q.Queue() |
|||
threads = [] |
|||
thread = None |
|||
for i in range(self.threadcount): |
|||
thread = thd.Thread(group=None, target=request, args=(qcodes, lists, self.semester)) |
|||
threads.append(thread) |
|||
thread.start() |
|||
qcodes.join() |
|||
for t in threads: |
|||
t.join() |
|||
|
|||
sections = [] |
|||
while not lists.empty(): |
|||
sections.append(lists.get()) |
|||
|
|||
for cl in chain.from_iterable(sections): |
|||
new_sections = [] |
|||
for sec in cl[1]: |
|||
print(sec) |
|||
if len(sec.day) > 1: |
|||
for day in sec.day: |
|||
new_sections.append(copy.deepcopy(sec)) |
|||
new_sections[-1]._day = day |
|||
else: |
|||
sec._day = sec.day[0] |
|||
new_sections.append(sec) |
|||
yield Class(cl[0][0], sub("\xa0+", "", cl[0][1]), sorted(new_sections)) |
|||
|
|||
def getCourses(semester, threadcount=10): |
|||
return CourseInfo(threadcount, semester).classes() |
|||
|
|||
def allCourses(): |
|||
return chain.from_iterable( |
|||
(getCourses(sem, threadcount=25) |
|||
for sem in (fall, winter, spring_summer))) |
|||
|
|||
if __name__ == "__main__": |
|||
for course in allCourses(): |
|||
sys.stdout.write("%s, %s, %s, %s\n" % (course.title, course.code, course.dept, list(chain.from_iterable(course.books) if course.books else []))) |
|||
print(course.sections) |
|||
Loading…
Reference in new issue