|
@ -1,12 +1,10 @@ |
|
|
from .archive import archiver |
|
|
from .archive import archiver |
|
|
from ..types import Conversation, Message |
|
|
from ..types import Conversation, Message |
|
|
from .common import SELENIUM_EXCEPTIONS, logger, click_button |
|
|
from .common import SELENIUM_EXCEPTIONS, logger, click_button, wait_xpath |
|
|
from selenium.webdriver.common.by import By |
|
|
|
|
|
from selenium.webdriver.support import expected_conditions as EC |
|
|
|
|
|
from selenium.webdriver.support.ui import WebDriverWait |
|
|
|
|
|
from selenium.webdriver.common.action_chains import ActionChains |
|
|
from selenium.webdriver.common.action_chains import ActionChains |
|
|
from pendulum import now |
|
|
from pendulum import now |
|
|
from json import loads |
|
|
from json import loads |
|
|
|
|
|
from time import sleep |
|
|
|
|
|
|
|
|
import lxml.html as lxh |
|
|
import lxml.html as lxh |
|
|
|
|
|
|
|
@ -17,17 +15,7 @@ def get_conversations(driver): |
|
|
Get a list of conversations |
|
|
Get a list of conversations |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
actions = ActionChains(driver) |
|
|
wait_xpath(driver, "//div[@id=\"threadlist_rows\"]") |
|
|
|
|
|
|
|
|
wait = WebDriverWait(driver, 20) |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
wait.until( |
|
|
|
|
|
EC.presence_of_element_located((By.XPATH, "//div[@id=\"threadlist_rows\"]")) |
|
|
|
|
|
) |
|
|
|
|
|
except SELENIUM_EXCEPTIONS: |
|
|
|
|
|
LOG.exception("No conversations") |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
# This function *cannot* be a generator |
|
|
# This function *cannot* be a generator |
|
|
# Otherwise elements will become stale |
|
|
# Otherwise elements will become stale |
|
@ -98,14 +86,7 @@ def get_convo(driver, convo): |
|
|
""" |
|
|
""" |
|
|
driver.get(convo.url) |
|
|
driver.get(convo.url) |
|
|
|
|
|
|
|
|
wait = WebDriverWait(driver, 20) |
|
|
wait_xpath(driver, "//*[contains(text(), 'See Older Messages')]") |
|
|
try: |
|
|
|
|
|
wait.until( |
|
|
|
|
|
EC.presence_of_element_located((By.XPATH, "//*[contains(text(), 'See Older Messages')]")) |
|
|
|
|
|
) |
|
|
|
|
|
except SELENIUM_EXCEPTIONS: |
|
|
|
|
|
LOG.exception("Could not load more messages") |
|
|
|
|
|
return |
|
|
|
|
|
|
|
|
|
|
|
# Expand conversation until we've reached the beginning |
|
|
# Expand conversation until we've reached the beginning |
|
|
while True: |
|
|
while True: |
|
@ -131,6 +112,12 @@ def delete_conversation(driver, convo): |
|
|
Deletes a conversation |
|
|
Deletes a conversation |
|
|
""" |
|
|
""" |
|
|
|
|
|
|
|
|
|
|
|
actions = ActionChains(driver) |
|
|
|
|
|
|
|
|
|
|
|
delete_button = driver.find_element_by_xpath("//select/option[contains(text(), 'Delete')]") |
|
|
|
|
|
|
|
|
|
|
|
actions.move_to_element(delete_button).click().perform() |
|
|
|
|
|
|
|
|
return |
|
|
return |
|
|
|
|
|
|
|
|
def extract_convo(driver, convo): |
|
|
def extract_convo(driver, convo): |
|
@ -150,8 +137,7 @@ def extract_convo(driver, convo): |
|
|
|
|
|
|
|
|
return convo |
|
|
return convo |
|
|
|
|
|
|
|
|
|
|
|
def traverse_conversations(driver, year=None, delete=False): |
|
|
def traverse_conversations(driver, year=None): |
|
|
|
|
|
""" |
|
|
""" |
|
|
Remove all conversations within a specified range |
|
|
Remove all conversations within a specified range |
|
|
""" |
|
|
""" |
|
@ -169,9 +155,13 @@ def traverse_conversations(driver, year=None): |
|
|
if convo.date.year == int(year): |
|
|
if convo.date.year == int(year): |
|
|
extract_convo(driver, convo) |
|
|
extract_convo(driver, convo) |
|
|
archive_convo.archive(convo) |
|
|
archive_convo.archive(convo) |
|
|
|
|
|
if delete: |
|
|
|
|
|
delete_conversation(driver, convo) |
|
|
|
|
|
|
|
|
# Otherwise we're looking at all convos |
|
|
# Otherwise we're looking at all convos |
|
|
elif not year: |
|
|
elif not year: |
|
|
extract_convo(driver, convo) |
|
|
extract_convo(driver, convo) |
|
|
archive_convo.archive(convo) |
|
|
archive_convo.archive(convo) |
|
|
|
|
|
if delete: |
|
|
|
|
|
delete_conversation(driver, convo) |
|
|
|
|
|
|
|
|