Merged master.

README.md

@@ -19,13 +19,15 @@ Personally, I did this so I would feel less attached to my Facebook profile
 
 ## Installation
 You have several options to run it.
-1) Install from PyPI with `pip install --user delete-facebook-posts`
+1) Install from PyPI with `pip3 install --user delete-facebook-posts`
-2) Clone this repo and run `pip install --user .` or do `pip install --user
+2) Clone this repo and run `pip3 install --user .` or do `pip3 install --user
 git+https://github.com/weskerfoot/DeleteFB.git`
-3) Set up a Python virtualenv, activate it, and run `pip install -r requirements.txt`, then you can just run `python -m deletefb.deletefb` in the DeleteFB directory.
+3) Set up a Python virtualenv, activate it, and run `pip3 install -r requirements.txt`, then you can just run `python -m deletefb.deletefb` in the DeleteFB directory.
 
 ## How To Use It
 
+* Make sure that you have a recent version of Python 3.x installed (preferably
+  3.6 or greater)
 * Make sure that you have Google Chrome installed and that it is up to date
 * Also install the chromedriver for Selenium. See [here](https://sites.google.com/a/chromium.org/chromedriver/home) for an explanation of what the chromedriver does.
   * On Linux, it will be called something like `chromium-chromedriver` or just
@@ -37,15 +39,18 @@ git+https://github.com/weskerfoot/DeleteFB.git`
     brew cask install chromedriver
     ```
 
-* Run `deletefb -E "youremail@example.org" -P "yourfacebookpassword" -U "https://www.facebook.com/your.profile.url"`
+* Run `deletefb -E 'youremail@example.org' -P 'yourfacebookpassword' -U 'https://www.facebook.com/your.profile.url'`
 * The script will log into your Facebook account, go to your profile page, and
   start deleting posts. If it cannot delete something, then it will "hide" it
   from your timeline instead.
 * Be patient as it will take a very long time, but it will eventually clear
   everything. You may safely minimize the chrome window without breaking it.
 
+## Login
+* The tool will log in using the credentials passed to it. It will wait until the page `https://www.facebook.com/` is loaded in order to avoid any issues with logging in. If you pass a 2FA token explicitly with the `-F` option, then it will try to enter that for you. If there are any issues, it simply pauses indefinitely to allow the user to resolve the problems, and then continues execution.
+
 ## 2FA
-* It is recommended that you disable Two-Factor Authentication tempoprarily
+* It is recommended that you disable Two-Factor Authentication temporarily
   while you are running the script, in order to get the best experience.
 
 * If you run into issues with Facebook complaining about your browser,
@@ -63,7 +68,8 @@ git+https://github.com/weskerfoot/DeleteFB.git`
 ## Unlike Pages
 * You may use `-M unlike_pages` to unlike all of your pages. The names of the
   pages will be archived (unless archival is turned off), and this option
-  conflicts with the year option.
+  conflicts with the year option. This will only unlike your *pages* that you
+  have liked. It will *not* unlike anything else (like books or movies).
 
 ## Archival
 * The tool will archive everything being deleted by default in `.log` files.

deletefb/deletefb.py

@@ -1,19 +1,16 @@
 #!/usr/bin/env python
+from .tools.common import logger
+from .tools.config import settings
+from .tools.likes import unlike_pages
+from .tools.login import login
+from .tools.wall import delete_posts
 
 import argparse
 import getpass
-import json
-import os
 import sys
 
-from .tools.common import logger
-from .tools.login import login
-from .tools.wall import delete_posts
-from .tools.likes import unlike_pages
-
 LOG = logger("deletefb")
 
-
 def run_delete():
     parser = argparse.ArgumentParser()
 
@@ -92,11 +89,7 @@ def run_delete():
 
     args = parser.parse_args()
 
-    if args.archive_off:
+    settings["ARCHIVE"] = not args.archive_off
-        os.environ["DELETEFB_ARCHIVE"] = "false"
-    else:
-        os.environ["DELETEFB_ARCHIVE"] = "true"
-
 
     if args.year and args.mode != "wall":
         parser.error("The --year option is only supported in wall mode")
@@ -111,11 +104,14 @@ def run_delete():
     )
 
     if args.mode == "wall":
-        delete_posts(driver,
+        delete_posts(
+            driver,
             args.profile_url,
-                     year=args.year)
+            year=args.year
+        )
+
     elif args.mode == "unlike_pages":
-        unlike_pages(driver)
+        unlike_pages(driver, args.profile_url)
     else:
         print("Please enter a valid mode")
         sys.exit(1)

deletefb/logging_conf.json

@@ -0,0 +1,62 @@
+{
+  "logging": {
+    "version": 1,
+    "disable_existing_loggers": true,
+    "formatters": {
+      "brief": {
+        "class": "logging.Formatter",
+        "style": "{",
+        "datefmt": "%H:%M:%S",
+        "format": "{name:s}-{levelname:s}-{asctime:s}-{message:s}"
+      },
+      "verbose": {
+        "class": "logging.Formatter",
+        "style": "{",
+        "datefmt": "%Y-%m-%dT%H:%M:%S",
+        "format": "{name:s}:{levelname:s}:L{lineno:d} {asctime:s} {message:s}"
+      }
+    },
+    "handlers": {
+      "console": {
+        "level": "DEBUG",
+        "class": "logging.StreamHandler",
+        "formatter": "brief",
+        "stream": "ext://sys.stdout"
+      },
+      "file_handler": {
+        "level": "INFO",
+        "class": "logging.handlers.WatchedFileHandler",
+        "formatter": "verbose",
+        "filename": "./deletefb.log",
+        "mode": "a",
+        "encoding": "utf-8"
+      }
+    },
+    "loggers": {
+      "root": {
+        "level": "DEBUG",
+        "handlers": ["console", "file_handler"]
+      },
+      "deletefb": {
+        "level": "DEBUG",
+        "handlers": ["console"],
+        "propagate": false
+      },
+      "login": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      },
+      "likes": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      },
+      "wall": {
+        "level": "DEBUG",
+        "handlers": ["file_handler"],
+        "propagate": false
+      }
+    }
+  }
+}

deletefb/tools/archive.py

@@ -0,0 +1,54 @@
+from .config import settings
+from contextlib import contextmanager
+from pathlib import Path
+
+import attr
+import json
+
+# Used to avoid duplicates in the log
+from pybloom_live import BloomFilter
+
+def make_filter():
+    return BloomFilter(
+        capacity=settings["MAX_POSTS"],
+        error_rate=0.001
+    )
+
+@attr.s
+class Archive:
+    archive_type = attr.ib()
+
+    # We give the Archive class a file handle
+    archive_file = attr.ib()
+
+    _bloom_filter = attr.ib(factory=make_filter)
+
+    def archive(self, content):
+        """
+        Archive an object
+        """
+        print("Archiving {0}".format(content))
+
+        if content.name not in self._bloom_filter:
+            self.archive_file.write(json.dumps(attr.asdict(content)) + "\n")
+            self._bloom_filter.add(content.name)
+        return
+
+@contextmanager
+def archiver(archive_type):
+
+    archive_file = open(
+        (Path(".") / Path(archive_type).name).with_suffix(".log"),
+        mode="ta",
+        buffering=1
+    )
+
+    archiver_instance = Archive(
+        archive_type=archive_type,
+        archive_file=archive_file
+    )
+
+    try:
+        yield archiver_instance
+    finally:
+        archive_file.close()

deletefb/tools/common.py

@@ -1,16 +1,14 @@
-import json
+from os.path import isfile
-import logging
-import logging.config
-import os
-from os.path import abspath, relpath, split, isfile
-import time
-
 from selenium.common.exceptions import (
     NoSuchElementException,
     StaleElementReferenceException,
     TimeoutException
 )
 
+import json
+import logging
+import logging.config
+import os
 
 SELENIUM_EXCEPTIONS = (
     NoSuchElementException,
@@ -18,13 +16,15 @@ SELENIUM_EXCEPTIONS = (
     TimeoutException
 )
 
-def try_move(actions, el):
+def click_button(driver, el):
-    for _ in range(10):
+    """
-        try:
+    Click a button using Javascript
-            actions.move_to_element(el).perform()
+    Args:
-        except StaleElementReferenceException:
+        driver: seleniumrequests.Chrome Driver instance
-            time.sleep(5)
+    Returns:
-            continue
+        None
+    """
+    driver.execute_script("arguments[0].click();", el)
 
 def logger(name):
     """
@@ -34,7 +34,10 @@ def logger(name):
     Returns:
         logging.Logger
     """
-    config_path = "deletefb/logging_conf.json"
+
+    # Make sure the path always points to the correct directory
+    config_path = os.path.dirname(os.path.realpath(__file__)) + "/../logging_conf.json"
+
     if not isfile(config_path):  # called from file (deletefb.py)
         os.chdir("..")
     with open(config_path, "r", encoding="utf-8") as config_file:
@@ -42,34 +45,6 @@ def logger(name):
         logging.config.dictConfig(config["logging"])
     return logging.getLogger(name)
 
-def archiver(category):
-    """
-     Log content to file. Call using `archive("some content")`
-
-    Args:
-        category: str The category of logs you want to log
-
-    Returns:
-        (log_file_handle, archiver)
-    """
-    log_path = "{0}.log".format(abspath(relpath(split(category)[-1], ".")))
-
-    log_file = open(log_path, mode="ta", buffering=1)
-
-    def log(content, timestamp=False):
-        if os.environ.get("DELETEFB_ARCHIVE", "true") == "false":
-            return
-        structured_content = {
-            "category" : category,
-            "content" : content,
-            "timestamp" : timestamp
-        }
-
-        log_file.write("{0}\n".format(json.dumps(structured_content)))
-
-    return (log_file, log)
-
-
 NO_CHROME_DRIVER = """
 You need to install the chromedriver for Selenium\n
 Please see this link https://github.com/weskerfoot/DeleteFB#how-to-use-it\n

deletefb/tools/config.py

@@ -0,0 +1,4 @@
+settings = {
+    "ARCHIVE" : True,
+    "MAX_POSTS" : 5000
+}

deletefb/tools/likes.py

@@ -1,14 +1,13 @@
+from .archive import archiver
+from ..types import Page
+from .common import SELENIUM_EXCEPTIONS, logger, click_button
 from selenium.webdriver.common.by import By
-from selenium.webdriver.common.action_chains import ActionChains
-from selenium.webdriver.support.ui import WebDriverWait
 from selenium.webdriver.support import expected_conditions as EC
-
+from selenium.webdriver.support.ui import WebDriverWait
-from .common import SELENIUM_EXCEPTIONS, archiver, logger
 
 LOG = logger(__name__)
 
-
+def load_likes(driver, profile_url):
-def load_likes(driver):
     """
     Loads the page that lists all pages you like
 
@@ -18,69 +17,98 @@ def load_likes(driver):
     Returns:
         None
     """
-    driver.get("https://www.facebook.com/pages/?category=liked")
+
+    driver.get("{0}/likes_all".format(profile_url))
 
     wait = WebDriverWait(driver, 20)
 
     try:
         wait.until(
-            EC.presence_of_element_located((By.XPATH, "//button/div/div[text()='Liked']"))
+            EC.presence_of_element_located((By.CSS_SELECTOR, ".PageLikeButton"))
-        )
-
-        wait.until(
-            EC.presence_of_element_located((By.XPATH, "//button/div/i[@aria-hidden=\"true\"]"))
         )
     except SELENIUM_EXCEPTIONS:
         LOG.exception("Traceback of load_likes")
         return
 
-def unlike_pages(driver):
+def get_page_links(driver):
     """
-    Unlike all pages
+    Gets all of the links to the pages you like
+
+    Args:
+        driver: seleniumrequests.Chrome Driver instance
+    Returns:
+        List of URLs to pages
+    """
+    pages = driver.find_elements_by_xpath("//li//div/div/a[contains(@class, 'lfloat')]")
+
+    return [page.get_attribute("href").replace("www", "mobile") for page in pages]
 
+def unlike_page(driver, url, archive=None):
+    """
+    Unlikes a page given the URL to it
     Args:
         driver: seleniumrequests.Chrome Driver instance
+        url: url string pointing to a page
+        archive: archiver instance
 
     Returns:
         None
+
     """
 
-    like_log, archive_likes = archiver("likes")
+    driver.get(url)
+
+    print("Unliking {0}".format(url))
+
+    wait = WebDriverWait(driver, 20)
+
+    try:
+        wait.until(
+            EC.presence_of_element_located((By.XPATH, "//*[text()='Liked']"))
+        )
+    except SELENIUM_EXCEPTIONS:
+        # Something went wrong with this page, so skip it
+        return
+
+    button = driver.find_element_by_xpath("//*[text()='Liked']")
 
-    actions = ActionChains(driver)
+    # Click the "Liked" button to open up "Unlike"
+    click_button(driver, button)
 
-    load_likes(driver)
+    wait.until(
+        EC.presence_of_element_located((By.XPATH, "//*[text()='Unlike']"))
+    )
 
-    pages_list = driver.find_element_by_css_selector("#all_liked_pages")
+    # There should be an "Unlike" button now, click it
+    unlike_button = driver.find_element_by_xpath("//*[text()='Unlike']/..")
 
-    actions.move_to_element(pages_list).perform()
+    click_button(driver, unlike_button)
 
-    unlike_buttons = pages_list.find_elements_by_xpath("//button/div/div[text()='Liked']/../..")
+    if archive:
+        archive(Page(name=url))
 
-    while unlike_buttons:
+def unlike_pages(driver, profile_url):
-        for button in unlike_buttons:
+    """
-            try:
+    Unlike all pages
-                if "Liked" in button.text:
-                    page_name = button.find_element_by_xpath("./../..").text.split("\n")[0]
 
-                    driver.execute_script("arguments[0].click();", button)
+    Args:
+        driver: seleniumrequests.Chrome Driver instance
 
-                    archive_likes(page_name)
+    Returns:
+        None
+    """
 
-                    print("{0} was unliked".format(page_name))
+    with archiver("likes") as archive_likes:
+        load_likes(driver, profile_url)
 
-            except SELENIUM_EXCEPTIONS:
+        urls = get_page_links(driver)
-                continue
 
-        load_likes(driver)
+        while urls:
+            for url in urls:
+                unlike_page(driver, url, archive=archive_likes.archive)
             try:
-            pages_list = driver.find_element_by_css_selector("#all_liked_pages")
+                load_likes(driver, profile_url)
-            actions.move_to_element(pages_list).perform()
+                urls = get_page_links(driver)
-            unlike_buttons = pages_list.find_elements_by_xpath("//button")
-            if not unlike_buttons:
-                break
             except SELENIUM_EXCEPTIONS:
+                # We're done
                 break
-
-    # Explicitly close the log file when we're done with it
-    like_log.close()

deletefb/tools/login.py

@@ -1,12 +1,10 @@
-import time
+from .common import NO_CHROME_DRIVER
-import sys
-
 from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.chrome.options import Options
 from seleniumrequests import Chrome
 
-from .common import NO_CHROME_DRIVER
+import sys
-
+import time
 
 def login(user_email_address,
           user_password,
@@ -47,7 +45,7 @@ def login(user_email_address,
 
     driver.implicitly_wait(10)
 
-    driver.get("https://facebook.com")
+    driver.get("https://www.facebook.com/login/device-based/regular/login/?login_attempt=1&lwv=110")
 
     email = "email"
     password = "pass"
@@ -99,4 +97,10 @@ def login(user_email_address,
     else:
         pass
 
+    # block until we have reached the main page
+    # print a message warning the user
+    while driver.current_url != "https://www.facebook.com/":
+        print("Execution blocked: Please navigate to https://www.facebook.com to continue")
+        time.sleep(5)
+
     return driver

deletefb/tools/wall.py

@@ -1,12 +1,13 @@
-import time
+from ..types import Post
-
+from .archive import archiver
+from .common import SELENIUM_EXCEPTIONS, click_button
+from .config import settings
 from selenium.webdriver.common.action_chains import ActionChains
 
-from .common import SELENIUM_EXCEPTIONS, archiver
+import time
 
 # Used as a threshold to avoid running forever
-MAX_POSTS = 15000
+MAX_POSTS = settings["MAX_POSTS"]
-
 
 def delete_posts(driver,
                  user_profile_url,
@@ -31,8 +32,9 @@ def delete_posts(driver,
         post_content_sel = "userContent"
         post_timestamp_sel = "timestampContent"
 
-        wall_log, archive_wall_post = archiver("wall")
+        button_types = ["FeedDeleteOption", "HIDE_FROM_TIMELINE", "UNTAG"]
 
+        with archiver("wall") as archive_wall_post:
             while True:
                 try:
                     timeline_element = driver.find_element_by_class_name(post_button_sel)
@@ -40,7 +42,14 @@ def delete_posts(driver,
                     post_content_element = driver.find_element_by_class_name(post_content_sel)
                     post_content_ts = driver.find_element_by_class_name(post_timestamp_sel)
 
-                archive_wall_post(post_content_element.text, timestamp=post_content_ts.text)
+
+                    # Archive the post
+                    archive_wall_post.archive(
+                        Post(
+                            content=post_content_element.text,
+                            date=post_content_ts.text
+                        )
+                    )
 
                     actions = ActionChains(driver)
                     actions.move_to_element(timeline_element).click().perform()
@@ -48,24 +57,28 @@ def delete_posts(driver,
                     menu = driver.find_element_by_css_selector("#globalContainer > div.uiContextualLayerPositioner.uiLayer > div")
                     actions.move_to_element(menu).perform()
 
+                    delete_button = None
+
+                    for button_type in button_types:
                         try:
-                    delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"FeedDeleteOption\"]")
+                            delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"{0}\"]".format(button_type))
-                except SELENIUM_EXCEPTIONS:
+                            break
-                    try:
-                        delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"HIDE_FROM_TIMELINE\"]")
                         except SELENIUM_EXCEPTIONS:
-                        delete_button = menu.find_element_by_xpath("//a[@data-feed-option-name=\"UNTAG\"]")
+                            continue
+
+                    if not delete_button:
+                        print("Could not find anything to delete")
+                        break
 
                     actions.move_to_element(delete_button).click().perform()
                     confirmation_button = driver.find_element_by_class_name("layerConfirm")
 
-                # Facebook would not let me get focus on this button without some custom JS
+                    click_button(driver, confirmation_button)
-                driver.execute_script("arguments[0].click();", confirmation_button)
+
                 except SELENIUM_EXCEPTIONS:
                     continue
                 else:
                     break
-        wall_log.close()
 
             # Required to sleep the thread for a bit after using JS to click this button
             time.sleep(5)

deletefb/types.py

@@ -0,0 +1,29 @@
+import attr
+import uuid
+import datetime
+
+def timestamp_now():
+    """
+    Returns: a timestamp for this instant, in ISO 8601 format
+    """
+    return datetime.datetime.isoformat(datetime.datetime.now())
+
+# Data type definitions of posts and comments
+@attr.s
+class Post:
+    content = attr.ib()
+    comments = attr.ib(default=[])
+    date = attr.ib(factory=timestamp_now)
+    name = attr.ib(factory=lambda: uuid.uuid4().hex)
+
+@attr.s
+class Comment:
+    commenter = attr.ib()
+    content = attr.ib()
+    date = attr.ib(factory=timestamp_now)
+    name = attr.ib(factory=lambda: uuid.uuid4().hex)
+
+@attr.s
+class Page:
+    name = attr.ib()
+    date = attr.ib(factory=timestamp_now)

requirements.txt

@@ -1,6 +1,9 @@
+attrs==19.1.0
+bitarray==0.9.3
 certifi==2018.11.29
 chardet==3.0.4
 idna==2.8
+pybloom-live==3.0.0
 requests==2.22.0
 requests-file==1.4.3
 selenium==3.141.0

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="delete-facebook-posts",
-    version="1.1.1",
+    version="1.1.7",
     author="Wesley Kerfoot",
     author_email="wes@wesk.tech",
     description="A Selenium Script to Delete Facebook Posts",
@@ -13,10 +13,18 @@ setuptools.setup(
     long_description_content_type="text/markdown",
     url="https://github.com/weskerfoot/DeleteFB",
     packages=setuptools.find_packages(),
+    include_package_data=True,
+    requires_python=">=3.6",
+    package_data={
+        # Include *json files in the package:
+        '': ['*.json'],
+    },
     install_requires = [
         "selenium",
         "selenium-requests",
-        "requests"
+        "requests",
+        "pybloom-live",
+        "attrs"
     ],
     classifiers= [
         "Programming Language :: Python :: 3",