Merged master.

README.md

@@ -19,13 +19,15 @@ Personally, I did this so I would feel less attached to my Facebook profile
 
 ## Installation
 You have several options to run it.
-1) Install from PyPI with `pip install --user delete-facebook-posts`
-2) Clone this repo and run `pip install --user .` or do `pip install --user
+1) Install from PyPI with `pip3 install --user delete-facebook-posts`
+2) Clone this repo and run `pip3 install --user .` or do `pip3 install --user
 git+https://github.com/weskerfoot/DeleteFB.git`
-3) Set up a Python virtualenv, activate it, and run `pip install -r requirements.txt`, then you can just run `python -m deletefb.deletefb` in the DeleteFB directory.
+3) Set up a Python virtualenv, activate it, and run `pip3 install -r requirements.txt`, then you can just run `python -m deletefb.deletefb` in the DeleteFB directory.
 
 ## How To Use It
 
+* Make sure that you have a recent version of Python 3.x installed (preferably
+  3.6 or greater)
 * Make sure that you have Google Chrome installed and that it is up to date
 * Also install the chromedriver for Selenium. See [here](https://sites.google.com/a/chromium.org/chromedriver/home) for an explanation of what the chromedriver does.
   * On Linux, it will be called something like `chromium-chromedriver` or just
@@ -37,15 +39,18 @@ git+https://github.com/weskerfoot/DeleteFB.git`
     brew cask install chromedriver
     ```
 
-* Run `deletefb -E "youremail@example.org" -P "yourfacebookpassword" -U "https://www.facebook.com/your.profile.url"`
+* Run `deletefb -E 'youremail@example.org' -P 'yourfacebookpassword' -U 'https://www.facebook.com/your.profile.url'`
 * The script will log into your Facebook account, go to your profile page, and
   start deleting posts. If it cannot delete something, then it will "hide" it
   from your timeline instead.
 * Be patient as it will take a very long time, but it will eventually clear
   everything. You may safely minimize the chrome window without breaking it.
 
+## Login
+* The tool will log in using the credentials passed to it. It will wait until the page `https://www.facebook.com/` is loaded in order to avoid any issues with logging in. If you pass a 2FA token explicitly with the `-F` option, then it will try to enter that for you. If there are any issues, it simply pauses indefinitely to allow the user to resolve the problems, and then continues execution.
+
 ## 2FA
-* It is recommended that you disable Two-Factor Authentication tempoprarily
+* It is recommended that you disable Two-Factor Authentication temporarily
   while you are running the script, in order to get the best experience.
 
 * If you run into issues with Facebook complaining about your browser,
@@ -63,7 +68,8 @@ git+https://github.com/weskerfoot/DeleteFB.git`
 ## Unlike Pages
 * You may use `-M unlike_pages` to unlike all of your pages. The names of the
   pages will be archived (unless archival is turned off), and this option
-  conflicts with the year option.
+  conflicts with the year option. This will only unlike your *pages* that you
+  have liked. It will *not* unlike anything else (like books or movies).
 
 ## Archival
 * The tool will archive everything being deleted by default in `.log` files.

deletefb/deletefb.py

@@ -1,19 +1,16 @@
 #!/usr/bin/env python
+from .tools.common import logger
+from .tools.config import settings
+from .tools.likes import unlike_pages
+from .tools.login import login
+from .tools.wall import delete_posts
 
 import argparse
 import getpass
-import json
-import os
 import sys
 
-from .tools.common import logger
-from .tools.login import login
-from .tools.wall import delete_posts
-from .tools.likes import unlike_pages
-
 LOG = logger("deletefb")
 
-
 def run_delete():
     parser = argparse.ArgumentParser()
 
@@ -92,11 +89,7 @@ def run_delete():
 
     args = parser.parse_args()
 
-    if args.archive_off:
-        os.environ["DELETEFB_ARCHIVE"] = "false"
-    else:
-        os.environ["DELETEFB_ARCHIVE"] = "true"
-
+    settings["ARCHIVE"] = not args.archive_off
 
     if args.year and args.mode != "wall":
         parser.error("The --year option is only supported in wall mode")
@@ -111,11 +104,14 @@ def run_delete():
     )
 
     if args.mode == "wall":
-        delete_posts(driver,
-                     args.profile_url,
-                     year=args.year)
+        delete_posts(
+            driver,
+            args.profile_url,
+            year=args.year
+        )
+
     elif args.mode == "unlike_pages":
-        unlike_pages(driver)
+        unlike_pages(driver, args.profile_url)
     else:
         print("Please enter a valid mode")
         sys.exit(1)

deletefb/logging_conf.json

@@ -0,0 +1,62 @@
+{
+  "logging": {
+    "version": 1,
+    "disable_existing_loggers": true,
+    "formatters": {
+      "brief": {
+        "class": "logging.Formatter",
+        "style": "{",
+        "datefmt": "%H:%M:%S",
+        "format": "{name:s}-{levelname:s}-{asctime:s}-{message:s}"
+      },
+      "verbose": {
+        "class": "logging.Formatter",
+        "style": "{",
+        "datefmt": "%Y-%m-%dT%H:%M:%S",
+        "format": "{name:s}:{levelname:s}:L{lineno:d} {asctime:s} {message:s}"
+      }
+    },
+    "handlers": {
+      "console": {
+        "level": "DEBUG",
+        "class": "logging.StreamHandler",
+        "formatter": "brief",
+        "stream": "ext://sys.stdout"
+      },
+      "file_handler": {
+        "level": "INFO",
+        "class": "logging.handlers.WatchedFileHandler",
+        "formatter": "verbose",
+        "filename": "./deletefb.log",
+        "mode": "a",
+        "encoding": "utf-8"
+      }
+    },
+    "loggers": {
+      "root": {
+        "level": "DEBUG",
+        "handlers": ["console", "file_handler"]
+      },
+      "deletefb": {
+        "level": "DEBUG",
+        "handlers": ["console"],
+        "propagate": false
+      },
+      "login": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      },
+      "likes": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      },
+      "wall": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      }
+    }
+  }
+}

deletefb/tools/archive.py

@@ -0,0 +1,54 @@
+from .config import settings
+from contextlib import contextmanager
+from pathlib import Path
+
+import attr
+import json
+
+# Used to avoid duplicates in the log
+from pybloom_live import BloomFilter
+
+def make_filter():
+    return BloomFilter(
+        capacity=settings["MAX_POSTS"],
+        error_rate=0.001
+    )
+
+@attr.s
+class Archive:
+    archive_type = attr.ib()
+
+    # We give the Archive class a file handle
+    archive_file = attr.ib()
+
+    _bloom_filter = attr.ib(factory=make_filter)
+
+    def archive(self, content):
+        """
+        Archive an object
+        """
+        print("Archiving {0}".format(content))
+
+        if content.name not in self._bloom_filter:
+            self.archive_file.write(json.dumps(attr.asdict(content)) + "\n")
+            self._bloom_filter.add(content.name)
+        return
+
+@contextmanager
+def archiver(archive_type):
+
+    archive_file = open(
+        (Path(".") / Path(archive_type).name).with_suffix(".log"),
+        mode="ta",
+        buffering=1
+    )
+
+    archiver_instance = Archive(
+        archive_type=archive_type,
+        archive_file=archive_file
+    )
+
+    try:
+        yield archiver_instance
+    finally:
+        archive_file.close()

deletefb/tools/common.py

@@ -1,16 +1,14 @@
-import json
-import logging
-import logging.config
-import os
-from os.path import abspath, relpath, split, isfile
-import time
-
+from os.path import isfile
 from selenium.common.exceptions import (
     NoSuchElementException,
     StaleElementReferenceException,
     TimeoutException
 )
 
+import json
+import logging
+import logging.config
+import os
 
 SELENIUM_EXCEPTIONS = (
     NoSuchElementException,
@@ -18,13 +16,15 @@ SELENIUM_EXCEPTIONS = (
     TimeoutException
 )
 
-def try_move(actions, el):
-    for _ in range(10):
-        try:
-            actions.move_to_element(el).perform()
-        except StaleElementReferenceException:
-            time.sleep(5)
-            continue
+def click_button(driver, el):
+    """
+    Click a button using Javascript
+    Args:
+        driver: seleniumrequests.Chrome Driver instance
+    Returns:
+        None
+    """
+    driver.execute_script("arguments[0].click();", el)
 
 def logger(name):
     """
@@ -34,7 +34,10 @@ def logger(name):
     Returns:
         logging.Logger
     """
-    config_path = "deletefb/logging_conf.json"
+
+    # Make sure the path always points to the correct directory
+    config_path = os.path.dirname(os.path.realpath(__file__)) + "/../logging_conf.json"
+
     if not isfile(config_path):  # called from file (deletefb.py)
         os.chdir("..")
     with open(config_path, "r", encoding="utf-8") as config_file:
@@ -42,34 +45,6 @@ def logger(name):
         logging.config.dictConfig(config["logging"])
     return logging.getLogger(name)
 
-def archiver(category):
-    """
-     Log content to file. Call using `archive("some content")`
-
-    Args:
-        category: str The category of logs you want to log
-
-    Returns:
-        (log_file_handle, archiver)
-    """
-    log_path = "{0}.log".format(abspath(relpath(split(category)[-1], ".")))
-
-    log_file = open(log_path, mode="ta", buffering=1)
-
-    def log(content, timestamp=False):
-        if os.environ.get("DELETEFB_ARCHIVE", "true") == "false":
-            return
-        structured_content = {
-            "category" : category,
-            "content" : content,
-            "timestamp" : timestamp
-        }
-
-        log_file.write("{0}\n".format(json.dumps(structured_content)))
-
-    return (log_file, log)
-
-
 NO_CHROME_DRIVER = """
 You need to install the chromedriver for Selenium\n
 Please see this link https://github.com/weskerfoot/DeleteFB#how-to-use-it\n

deletefb/tools/config.py

@@ -0,0 +1,4 @@
+settings = {
+    "ARCHIVE" : True,
+    "MAX_POSTS" : 5000
+}

deletefb/tools/likes.py

@@ -1,14 +1,13 @@
+from .archive import archiver
+from ..types import Page
+from .common import SELENIUM_EXCEPTIONS, logger, click_button
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.action_chains import ActionChains
-from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-
-from .common import SELENIUM_EXCEPTIONS, archiver, logger
+from selenium.webdriver.support.ui import WebDriverWait
 
 LOG = logger(__name__)
 
-
-def load_likes(driver):
+def load_likes(driver, profile_url):
     """
     Loads the page that lists all pages you like
 
@@ -18,69 +17,98 @@ def load_likes(driver):
     Returns:
         None
     """
-    driver.get("https://www.facebook.com/pages/?category=liked")
+
+    driver.get("{0}/likes_all".format(profile_url))
 
     wait = WebDriverWait(driver, 20)
 
     try:
         wait.until(
-            EC.presence_of_element_located((By.XPATH, "//button/div/div[text()='Liked']"))
-        )
-
-        wait.until(
-            EC.presence_of_element_located((By.XPATH, "//button/div/i[@aria-hidden=\"true\"]"))
+            EC.presence_of_element_located((By.CSS_SELECTOR, ".PageLikeButton"))
         )
     except SELENIUM_EXCEPTIONS:
         LOG.exception("Traceback of load_likes")
         return
 
-def unlike_pages(driver):
+def get_page_links(driver):
     """
-    Unlike all pages
+    Gets all of the links to the pages you like
 
     Args:
         driver: seleniumrequests.Chrome Driver instance
+    Returns:
+        List of URLs to pages
+    """
+    pages = driver.find_elements_by_xpath("//li//div/div/a[contains(@class, 'lfloat')]")
+
+    return [page.get_attribute("href").replace("www", "mobile") for page in pages]
+
+def unlike_page(driver, url, archive=None):
+    """
+    Unlikes a page given the URL to it
+    Args:
+        driver: seleniumrequests.Chrome Driver instance
+        url: url string pointing to a page
+        archive: archiver instance
 
     Returns:
         None
+
     """
 
-    like_log, archive_likes = archiver("likes")
+    driver.get(url)
+
+    print("Unliking {0}".format(url))
+
+    wait = WebDriverWait(driver, 20)
 
-    actions = ActionChains(driver)
+    try:
+        wait.until(
+            EC.presence_of_element_located((By.XPATH, "//*[text()='Liked']"))
+        )
+    except SELENIUM_EXCEPTIONS:
+        # Something went wrong with this page, so skip it
+        return
 
-    load_likes(driver)
+    button = driver.find_element_by_xpath("//*[text()='Liked']")
 
-    pages_list = driver.find_element_by_css_selector("#all_liked_pages")
+    # Click the "Liked" button to open up "Unlike"
+    click_button(driver, button)
 
-    actions.move_to_element(pages_list).perform()
+    wait.until(
+        EC.presence_of_element_located((By.XPATH, "//*[text()='Unlike']"))
+    )
 
-    unlike_buttons = pages_list.find_elements_by_xpath("//button/div/div[text()='Liked']/../..")
+    # There should be an "Unlike" button now, click it
+    unlike_button = driver.find_element_by_xpath("//*[text()='Unlike']/..")
 
-    while unlike_buttons:
-        for button in unlike_buttons:
-            try:
-                if "Liked" in button.text:
-                    page_name = button.find_element_by_xpath("./../..").text.split("\n")[0]
+    click_button(driver, unlike_button)
 
-                    driver.execute_script("arguments[0].click();", button)
+    if archive:
+        archive(Page(name=url))
 
-                    archive_likes(page_name)
+def unlike_pages(driver, profile_url):
+    """
+    Unlike all pages
 
-                    print("{0} was unliked".format(page_name))
+    Args:
+        driver: seleniumrequests.Chrome Driver instance
+
+    Returns:
+        None
+    """
 
+    with archiver("likes") as archive_likes:
+        load_likes(driver, profile_url)
+
+        urls = get_page_links(driver)
+
+        while urls:
+            for url in urls:
+                unlike_page(driver, url, archive=archive_likes.archive)
+            try:
+                load_likes(driver, profile_url)
+                urls = get_page_links(driver)
             except SELENIUM_EXCEPTIONS:
-                continue
-
-        load_likes(driver)
-        try:
-            pages_list = driver.find_element_by_css_selector("#all_liked_pages")
-            actions.move_to_element(pages_list).perform()
-            unlike_buttons = pages_list.find_elements_by_xpath("//button")
-            if not unlike_buttons:
+                # We're done
                 break
-        except SELENIUM_EXCEPTIONS:
-            break
-
-    # Explicitly close the log file when we're done with it
-    like_log.close()

deletefb/tools/login.py

@@ -1,12 +1,10 @@
-import time
-import sys
-
+from .common import NO_CHROME_DRIVER
 from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.chrome.options import Options
 from seleniumrequests import Chrome
 
-from .common import NO_CHROME_DRIVER
-
+import sys
+import time
 
 def login(user_email_address,
           user_password,
@@ -47,7 +45,7 @@ def login(user_email_address,
 
     driver.implicitly_wait(10)
 
-    driver.get("https://facebook.com")
+    driver.get("https://www.facebook.com/login/device-based/regular/login/?login_attempt=1&lwv=110")
 
     email = "email"
     password = "pass"
@@ -99,4 +97,10 @@ def login(user_email_address,
     else:
         pass
 
+    # block until we have reached the main page
+    # print a message warning the user
+    while driver.current_url != "https://www.facebook.com/":
+        print("Execution blocked: Please navigate to https://www.facebook.com to continue")
+        time.sleep(5)
+
     return driver

deletefb/tools/wall.py

@@ -1,12 +1,13 @@
-import time
-
+from ..types import Post
+from .archive import archiver
+from .common import SELENIUM_EXCEPTIONS, click_button
+from .config import settings
 from selenium.webdriver.common.action_chains import ActionChains
 
-from .common import SELENIUM_EXCEPTIONS, archiver
+import time
 
 # Used as a threshold to avoid running forever
-MAX_POSTS = 15000
-
+MAX_POSTS = settings["MAX_POSTS"]
 
 def delete_posts(driver,
                  user_profile_url,
@@ -31,42 +32,54 @@ def delete_posts(driver,
         post_content_sel = "userContent"
         post_timestamp_sel = "timestampContent"
 
-        wall_log, archive_wall_post = archiver("wall")
+        button_types = ["FeedDeleteOption", "HIDE_FROM_TIMELINE", "UNTAG"]
 
-        while True:
-            try:
-                timeline_element = driver.find_element_by_class_name(post_button_sel)
+        with archiver("wall") as archive_wall_post:
+            while True:
+                try:
+                    timeline_element = driver.find_element_by_class_name(post_button_sel)
 
-                post_content_element = driver.find_element_by_class_name(post_content_sel)
-                post_content_ts = driver.find_element_by_class_name(post_timestamp_sel)
+                    post_content_element = driver.find_element_by_class_name(post_content_sel)
+                    post_content_ts = driver.find_element_by_class_name(post_timestamp_sel)
 
-                archive_wall_post(post_content_element.text, timestamp=post_content_ts.text)
 
-                actions = ActionChains(driver)
-                actions.move_to_element(timeline_element).click().perform()
+                    # Archive the post
+                    archive_wall_post.archive(
+                        Post(
+                            content=post_content_element.text,
+                            date=post_content_ts.text
+                        )
+                    )
 
-                menu = driver.find_element_by_css_selector("#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
-                actions.move_to_element(menu).perform()
+                    actions = ActionChains(driver)
+                    actions.move_to_element(timeline_element).click().perform()
+
+                    menu = driver.find_element_by_css_selector("#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
+                    actions.move_to_element(menu).perform()
+
+                    delete_button = None
+
+                    for button_type in button_types:
+                        try:
+                            delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"{0}\"]".format(button_type))
+                            break
+                        except SELENIUM_EXCEPTIONS:
+                            continue
+
+                    if not delete_button:
+                        print("Could not find anything to delete")
+                        break
+
+                    actions.move_to_element(delete_button).click().perform()
+                    confirmation_button = driver.find_element_by_class_name("layerConfirm")
+
+                    click_button(driver, confirmation_button)
 
-                try:
-                    delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"FeedDeleteOption\"]")
                 except SELENIUM_EXCEPTIONS:
-                    try:
-                        delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
-                    except SELENIUM_EXCEPTIONS:
-                        delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"UNTAG\"]")
-
-                actions.move_to_element(delete_button).click().perform()
-                confirmation_button = driver.find_element_by_class_name("layerConfirm")
-
-                # Facebook would not let me get focus on this button without some custom JS
-                driver.execute_script("arguments[0].click();", confirmation_button)
-            except SELENIUM_EXCEPTIONS:
-                continue
-            else:
-                break
-        wall_log.close()
-
-        # Required to sleep the thread for a bit after using JS to click this button
-        time.sleep(5)
-        driver.refresh()
+                    continue
+                else:
+                    break
+
+            # Required to sleep the thread for a bit after using JS to click this button
+            time.sleep(5)
+            driver.refresh()

deletefb/types.py

@@ -0,0 +1,29 @@
+import attr
+import uuid
+import datetime
+
+def timestamp_now():
+    """
+    Returns: a timestamp for this instant, in ISO 8601 format
+    """
+    return datetime.datetime.isoformat(datetime.datetime.now())
+
+# Data type definitions of posts and comments
+@attr.s
+class Post:
+    content = attr.ib()
+    comments = attr.ib(default=[])
+    date = attr.ib(factory=timestamp_now)
+    name = attr.ib(factory=lambda: uuid.uuid4().hex)
+
+@attr.s
+class Comment:
+    commenter = attr.ib()
+    content = attr.ib()
+    date = attr.ib(factory=timestamp_now)
+    name = attr.ib(factory=lambda: uuid.uuid4().hex)
+
+@attr.s
+class Page:
+    name = attr.ib()
+    date = attr.ib(factory=timestamp_now)

requirements.txt

@@ -1,6 +1,9 @@
+attrs==19.1.0
+bitarray==0.9.3
 certifi==2018.11.29
 chardet==3.0.4
 idna==2.8
+pybloom-live==3.0.0
 requests==2.22.0
 requests-file==1.4.3
 selenium==3.141.0

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="delete-facebook-posts",
-    version="1.1.1",
+    version="1.1.7",
     author="Wesley Kerfoot",
     author_email="wes@wesk.tech",
     description="A Selenium Script to Delete Facebook Posts",
@@ -13,10 +13,18 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://github.com/weskerfoot/DeleteFB",
     packages=setuptools.find_packages(),
+    include_package_data=True,
+    requires_python=">=3.6",
+    package_data={
+        # Include *json files in the package:
+        '': ['*.json'],
+    },
     install_requires = [
         "selenium",
         "selenium-requests",
-        "requests"
+        "requests",
+        "pybloom-live",
+        "attrs"
     ],
     classifiers= [
         "Programming Language :: Python :: 3",