path: root/pyblog

#!/usr/bin/env python3

"""A simple blog generator with Pandoc as backend."""

# TODO: auto retouch: prompt for git commit amend after touching
# (display commit message to avoid amending the wrong commit)

# pylint: disable=too-many-lines

import argparse
from contextlib import contextmanager
import copy
import curses
import datetime
import email.utils
import fileinput
import io
import http.client
import http.server
import multiprocessing
import os
import re
import shutil
import signal
import string
import subprocess
import sys
import tempfile
import time
import urllib.parse

import blessed
import bs4
import colorama
import dateutil.parser
import dateutil.tz
import lxml.etree as ET

from bs4 import UnicodeDammit
from pprint import pprint
import requests

import toml

from rss import *

from utils import utils

from config.config import *

from generators import generators

from cli import cli


def sanitize(string):
    """Sanitize string (title) for URI consumption."""
    if isinstance(string, bytes):
        string = string.decode('utf-8')
    # to lowercase
    string = string.lower()
    # strip all non-word, non-hyphen and non-whitespace characters
    string = re.sub(r"[^\w\s-]", "", string)
    # replace consecutive whitespaces with a single hyphen
    string = re.sub(r"\s+", "-", string)
    # percent encode the result
    return urllib.parse.quote(string)


def edit_post_with_editor(path):
    """Launch text editor to edit post at a given path.

    Text editor is $VISUAL, then if empty, $EDITOR, then if still empty,
    vi.

    """
    if "VISUAL" in os.environ:
        editor = os.environ["VISUAL"]
    elif "EDITOR" in os.environ:
        editor = os.environ["EDITOR"]
    else:
        editor = "vi"
    subprocess.call([editor, path])


def new_post(title):
    """Create a new post with metadata pre-filled.

    The path to the new post is printed to stdout.

    Returns
    -------
    0
        On success.

    """
    date = utils.current_datetime()
    filename_date = date.strftime("%Y-%m-%d")
    iso_date = date.isoformat()
    display_date = "%s %d, %d" % (date.strftime("%B"), date.day, date.year)
    title_sanitized = sanitize(title)
    filename = "%s-%s.md" % (filename_date, title_sanitized)
    fullpath = os.path.join(POSTSDIR, filename)
    if not os.path.isdir(POSTSDIR):
        if os.path.exists(POSTSDIR):
            os.remove(POSTSDIR)
        os.mkdir(POSTSDIR, mode=0o755)
    if os.path.exists(fullpath):
        sys.stderr.write("%serror: '%s' already exists, please pick a different title%s\n" %
                         (RED, fullpath, RESET))
        return 1
    with open(fullpath, 'w', encoding='utf-8') as newpost:
        newpost.write("---\n")
        newpost.write('title: "%s"\n' % title)
        newpost.write("date: %s\n" % iso_date)
        newpost.write("date_display: %s\n" % display_date)
        newpost.write("---\n\n")
    sys.stderr.write("New post created in:\n")
    print(fullpath)
    edit_post_with_editor(fullpath)

    return 0


def new_post_cli(args):
    """CLI wrapper around new_post."""
    new_post(args.title)


def touch(filename):
    """Update the timestamp of a post to the current time."""
    filename = os.path.basename(filename)
    fullpath = os.path.join(POSTSDIR, filename)
    if not os.path.exists(fullpath):
        sys.stderr.write("%serror: post %s not found %s\n" %
                         (RED, fullpath, RESET))
        return 1
    filename_prefix_re = re.compile(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}")
    if not filename_prefix_re.match(filename):
        sys.stderr.write(RED)
        sys.stderr.write("error: post %s is not a valid post\n" % filename)
        sys.stderr.write("error: the filename of a valid post begins with "
                         "a date in the form xxxx-xx-xx\n")
        sys.stderr.write(RESET)
        return 1

    # update timestamp in the metadata section of the post
    whatchanged = io.StringIO()
    date = utils.current_datetime()
    iso_date = date.isoformat()
    display_date = "%s %d, %d" % (date.strftime("%B"), date.day, date.year)
    filename_date = date.strftime("%Y-%m-%d")
    with fileinput.input(files=(fullpath), inplace=True) as lines:
        meta_fences = 0
        for line in lines:
            if line.startswith("---"):
                meta_fences += 1
                sys.stdout.write(line)
                continue
            if meta_fences >= 2:
                # already went past the metadata section
                sys.stdout.write(line)
                continue

            if line.startswith("date: "):
                updated_line = "date: %s\n" % iso_date
                sys.stdout.write(updated_line)
                whatchanged.write("-%s+%s\n" % (line, updated_line))
                continue

            if line.startswith("date_display: "):
                updated_line = "date_display: %s\n" % display_date
                sys.stdout.write(updated_line)
                whatchanged.write("-%s+%s\n" % (line, updated_line))
                continue

            sys.stdout.write(line)

    sys.stderr.write("\n%schangeset:%s\n\n%s" %
                     (YELLOW, RESET, whatchanged.getvalue()))
    whatchanged.close()

    # check if the file needs to be renamed
    new_filename = filename_prefix_re.sub(filename_date, filename)
    if new_filename != filename:
        new_fullpath = os.path.join(POSTSDIR, new_filename)
        os.rename(fullpath, new_fullpath)
        sys.stderr.write("%srenamed to %s%s\n" % (YELLOW, new_filename, RESET))
    return 0


def touch_cli(args):
    """CLI wrapper around touch."""
    touch(args.filename)


def deploy(args):
    """Deploys build directory to origin/master without regenerating.

    Returns
    -------
    0
        On success. Exit early with nonzero status otherwise.

    """

    # pylint: disable=unused-argument,too-many-statements

    # check whether root is dirty
    os.chdir(ROOTDIR)
    dirty = subprocess.check_output(["git", "status", "--porcelain"])
    if dirty:
        sys.stderr.write(YELLOW)
        sys.stderr.write("Project root is dirty.\n")
        sys.stderr.write("You may want to commit in your changes "
                         "to the source branch, since the SHA and title "
                         "of the latest commit on the source branch will be "
                         "incorporated into the commit message on "
                         "the deployment branch. Type s[hell] on the "
                         "next prompt to open an interactive shell.\n")
        sys.stderr.write(RESET)
        while True:
            sys.stderr.write("Continue? [yNs] ")
            answer = input()
            if not answer:
                # default
                abort = True
                break
            elif answer.startswith(('y', 'Y')):
                abort = False
                break
            elif answer.startswith(('n', 'N')):
                abort = True
                break
            elif answer.startswith(('s', 'S')):
                shell = (os.environ['SHELL'] if 'SHELL' in os.environ and os.environ['SHELL']
                         else 'zsh')
                subprocess.call(shell)
                stilldirty = subprocess.check_output(["git", "status", "--porcelain"])
                if stilldirty:
                    sys.stderr.write(YELLOW)
                    sys.stderr.write("Project root is still dirty.\n")
                    sys.stderr.write(RESET)
            else:
                sys.stderr.write("Please answer yes or no.\n")
        if abort:
            sys.stderr.write("%saborting deployment%s\n" % (RED, RESET))
            return 1

    # extract latest commit on the source branch
    source_commit = subprocess.check_output(
        ["git", "log", "-1", "--pretty=oneline", "source", "--"]).decode('utf-8').strip()

    # cd into BUILDDIR and assemble commit message
    sys.stderr.write("%scommand: cd '%s'%s\n" % (BLUE, BUILDDIR, RESET))
    os.chdir(BUILDDIR)

    # extract updated time from atom.xml
    if not os.path.exists("atom.xml"):
        sys.stderr.write("atom.xml not found, cannot deploy\naborting\n")
        return 1
    atomxml = ET.parse("atom.xml").getroot()
    updated = atomxml.find('{http://www.w3.org/2005/Atom}updated').text

    commit_message = ("Site updated at %s\n\nsource branch was at:\n%s\n" %
                      (updated, source_commit))

    # commit changes in BUILDDIR
    sys.stderr.write("%scommand: git add --all%s\n" % (BLUE, RESET))
    subprocess.check_call(["git", "add", "--all"])
    sys.stderr.write("%scommand: git commit --no-verify --gpg-sign --message='%s'%s\n" %
                     (BLUE, commit_message, RESET))
    try:
        subprocess.check_call(["git", "commit", "--gpg-sign",
                               "--message=%s" % commit_message])
    except subprocess.CalledProcessError:
        sys.stderr.write("\n%serror: git commit failed%s\n" % (RED, RESET))
        return 1

    # check dirty status
    dirty = subprocess.check_output(["git", "status", "--porcelain"])
    if dirty:
        sys.stderr.write(RED)
        sys.stderr.write("error: failed to commit all changes; "
                         "build directory still dirty\n")
        sys.stderr.write("error: please manually inspect what was left out\n")
        sys.stderr.write(RESET)
        return 1

    # push to origin/master
    sys.stderr.write("%scommand: git push origin master%s\n" % (BLUE, RESET))
    try:
        subprocess.check_call(["git", "push", "origin", "master"])
    except subprocess.CalledProcessError:
        sys.stderr.write("\n%serror: git push failed%s\n" % (RED, RESET))
        return 1
    return 0


def gen_deploy(args):
    """Regenerate and deploy."""
    # pylint: disable=unused-argument,too-many-branches

    # try to smartly determine the latest post, and prompt to touch it
    current_time = time.time()
    latest_post = None
    latest_postdate = 0
    latest_mtime = 0
    for name in os.listdir(POSTSDIR):
        matchobj = re.match(r"^([0-9]{4})-([0-9]{2})-([0-9]{2})-.*\.md", name)
        if not matchobj:
            continue
        fullpath = os.path.join(POSTSDIR, name)
        mtime = os.path.getmtime(fullpath)
        # get post date from the date metadata field of the post
        postdate = 0
        with open(fullpath) as postobj:
            for line in postobj:
                dateregex = r"^date: (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}-\d{2}:?\d{2})"
                datematch = re.match(dateregex, line.rstrip())
                if datematch:
                    postdate = dateutil.parser.parse(datematch.group(1)).timestamp()
                    break
        # skip the post if it is dated more than three days ago
        if current_time - postdate > 3 * 24 * 3600:
            continue
        if mtime > latest_mtime:
            latest_post = name
            latest_postdate = postdate
            latest_mtime = mtime
    # prompt for touching if the latest post determined above was
    # modified within the last hour but the date registered in the post
    # isn't within the last ten minutes
    if ((latest_post is not None and current_time - latest_mtime < 3600 and
         current_time - latest_postdate > 600)):
        sys.stderr.write("%sIt appears that %s might be a new post.\n"
                         "Do you want to touch its timestamp?%s\n" %
                         (GREEN, latest_post, RESET))
        while True:
            yesnoquit = input("[ynq]: ")
            if yesnoquit.startswith(("Y", "y")):
                yesno = True
                break
            elif yesnoquit.startswith(("N", "n")):
                yesno = False
                break
            elif yesnoquit.startswith(("Q", "q")):
                sys.stderr.write("%saborting gen_deploy%s\n" % (RED, RESET))
                return 1
            else:
                sys.stderr.write("Please answer yes, no, or quit.\n")
        if yesno:
            sys.stderr.write("%stouching %s%s\n" % (BLUE, latest_post, RESET))
            touch(latest_post)
            sys.stderr.write("\n")

    generators.generate_blog(fresh=True)
    deploy(None)


class HTTPServerProcess(multiprocessing.Process):
    """This class can be used to run an HTTP server."""

    def __init__(self, rootdir):
        """Initialize the HTTPServerProcess class.

        Parameters
        ----------
        rootdir : str
            The root directory to serve from.

        """

        super().__init__()
        self.rootdir = rootdir

    def run(self):
        """Create an HTTP server and serve forever.

        Runs on localhost. The default port is 8000; if it is not
        available, a random port is used instead.
        """

        os.chdir(self.rootdir)
        # pylint: disable=invalid-name
        HandlerClass = http.server.SimpleHTTPRequestHandler
        try:
            httpd = http.server.HTTPServer(("", 8001), HandlerClass)
        except OSError:
            httpd = http.server.HTTPServer(("", 0), HandlerClass)
        _, portnumber = httpd.socket.getsockname()
        sys.stderr.write("server serving on http://localhost:%d\n" % portnumber)
        try:
            httpd.serve_forever()
        except KeyboardInterrupt:
            httpd.shutdown()


def preview(args):
    """Serve the blog and auto regenerate upon changes."""

    # pylint: disable=unused-argument

    server_process = HTTPServerProcess(BUILDDIR)
    server_process.start()
    sys.stderr.write("watching for changes\n")
    sys.stderr.write("send SIGINT to stop\n")

    # install a SIGINT handler only for this process
    sigint_raised = False

    def sigint_mitigator(signum, frame):
        """Translate SIGINT to setting the sigint_raised flag."""
        nonlocal sigint_raised
        sigint_raised = True

    signal.signal(signal.SIGINT, sigint_mitigator)

    # Watch and auto-regen.
    # No need to actually implement watch separately, since
    # generate_blog(fresh=False, report_total_errors=False) already
    # watches for modifications and only regens upon changes, and it is
    # completely silent when there's no change.
    while not sigint_raised:
        generators.generate_blog(fresh=False, report_total_errors=False)
        time.sleep(0.5)

    sys.stderr.write("\nSIGINT received, cleaning up...\n")
    server_process.join()
    return 0


def list_posts():
    """List all posts, with date, title, and path to source file.

    This function only lists posts that has been built (since it reads
    metadata from HTML rather than Markdown).

    Returns
    -------
    posts : list
        A list of posts, in reverse chronological order, where each
        element is a tuple of (date, title, path to source file).

    """
    posts = []
    for name in os.listdir(os.path.join(BUILDDIR, "blog")):
        if not re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}.*\.html", name):
            continue

        htmlpath = os.path.join(BUILDDIR, "blog", name)
        entry = AtomEntry()
        item = RssItem()
        try:
            with open(htmlpath, encoding="utf-8") as htmlfile:
                soup = bs4.BeautifulSoup(htmlfile.read(), "lxml")
                title = soup.title.text
                date = dateutil.parser.parse(soup.find("meta", attrs={"name": "date"})["content"])
                source_path = os.path.join(POSTSDIR, re.sub(r'.html$', '.md', name))
                posts.append((date, title, source_path))
        except Exception:
            sys.stderr.write("error: failed to read metadata from HTML file %s\n" % name)
            with open(htmlpath, encoding="utf-8") as htmlfile:
                sys.stderr.write("dumping HTML:%s\n\n" % htmlfile.read())
            raise

    posts.sort(key=lambda post: post[0], reverse=True)
    return posts


class PostSelector:

    def __init__(self, term, posts):
        self._term = term
        self.posts_per_page = term.height - 2
        self.pages = [posts[i:i+self.posts_per_page]
                      for i in range(0, len(posts), self.posts_per_page)]
        self.num_pages = len(self.pages)
        self.pagepos = 0
        self.postpos = 0
        self.inserting = False  # True if in the middle of inserting a post #, False otherwise
        term.enter_fullscreen()
        print(term.clear(), end="")
        sys.stdout.flush()
        self.selection = ""
        self.quit = False

        self.display_page()

    def _clear_to_eol(self):
        term = self._term
        print(term.clear_eol, end="")
        sys.stdout.flush()

    def _print_line(self, line, linenum, highlight=False):
        term = self._term
        width = term.width
        with term.location(0, linenum):
            if highlight:
                print(term.reverse(line[:width]), end="")
            else:
                print(line[:width], end="")
            self._clear_to_eol()

    def _print_post(self, page, pos, highlight=False):
        if pos >= len(page):
            # if position out of range, just clear the line
            self._print_line("", pos + 1, highlight)
        else:
            date, title, path = page[pos]
            line = "%3d: %s %s" % (pos, date.strftime("%m/%d/%y"), title)
            self._print_line(line, pos + 1, highlight)

    def display_page(self):
        term = self._term
        page = self.pages[self.pagepos]

        with term.hidden_cursor():
            topline = "  PAGE %d/%d  POST %d" % (self.pagepos + 1, self.num_pages, self.postpos)
            if self.inserting:
                topline += term.blink("_")
            self._print_line(topline, 0, highlight=True)

            for i in range(self.posts_per_page):
                self._print_post(page, i)
            # highlight selected post
            self._print_post(page, self.postpos, highlight=True)

            bottomline = "  Press h for help."
            self._print_line(bottomline, term.height - 1, highlight=True)

    def dispatch(self, key):
        term = self._term
        if key in string.digits:
            # insert
            if self.inserting:
                newpostpos = 10 * self.postpos + int(key)
                if newpostpos < len(self.pages[self.pagepos]):
                    self.postpos = newpostpos
            else:
                self.postpos = int(key)
            self.inserting = True
        elif key.name == "KEY_DELETE":
            self.postpos //= 10
            self.inserting = True
        else:
            self.inserting = False
            if key.name == "KEY_ENTER":
                self.selection = self.pages[self.pagepos][self.postpos][2]
            if key in {"q", "Q"}:
                self.quit = True
            elif key.name == "KEY_DOWN" or key in {"n", "N"}:
                if self.postpos + 1 < len(self.pages[self.pagepos]):
                    self.postpos += 1
            elif key.name == "KEY_UP" or key in {"p", "P"}:
                if self.postpos > 0:
                    self.postpos -= 1
            elif key.name == "KEY_RIGHT" or key in {".", ">"}:
                if self.pagepos + 1 < self.num_pages:
                    self.pagepos += 1
                    self.postpos = 0
            elif key.name == "KEY_LEFT" or key in {",", "<"}:
                if self.pagepos > 0:
                    self.pagepos -= 1
                    self.postpos = 0
            elif key in {"h", "H"}:
                print(term.clear_eol, end="")
                sys.stdout.flush()
                help_text_lines = [
                    "Next post:         n or <down>",
                    "Previous post:     p or <up>",
                    "Next page:         . or > or <right>",
                    "Previous page:     , or < or <left>",
                    "Select post:       <enter> or <return>",
                    "Select by number:  type number as shown (delete or backspace to edit)",
                    "Get help:          h",
                    "Quit program:      q",
                ]
                for i in range(term.height - 1):
                    self._print_line(help_text_lines[i] if i < len(help_text_lines) else "", i)
                bottomline = "  Press any key to continue."
                self._print_line(bottomline, term.height - 1, highlight=True)

                with term.raw():
                    term.inkey()

    def restore(self):
        term = self._term
        term.exit_fullscreen()
        print(term.clear(), end="")
        sys.stdout.flush()

    def select(self):
        term = self._term
        try:
            while True:
                with term.raw():
                    self.dispatch(term.inkey())
                if self.selection or self.quit:
                    break
                self.display_page()
        except Exception:
            raise
        finally:
            self.restore()

        return self.selection


def edit_existing_post(args):
    selector = PostSelector(blessed.Terminal(), list_posts())
    selection = selector.select()
    if selection:
        print(selection)
        edit_post_with_editor(selection)
    else:
        return 1


def main():
    """CLI interface."""
    description = "Simple blog generator in Python with Pandoc as backend."
    parser = argparse.ArgumentParser(description=description)
    subparsers = parser.add_subparsers(dest="action")
    subparsers.required = True

    parser_new_post = subparsers.add_parser(
        "new_post", aliases=["n", "new"],
        description="Create a new post with metadata pre-filled.")
    parser_new_post.add_argument("title", help="title of the new post")
    parser_new_post.set_defaults(func=new_post_cli)

    parser_new_post = subparsers.add_parser(
        "touch", aliases=["t", "tou"],
        description="""Touch an existing post, i.e., update its
        timestamp to current time.  Why is this ever useful? Well, the
        timestamp filled in by new_post is the time of creation, but one
        might spend several hours after the creation of the file to
        finish the post. Sometimes the post is even created on one day
        and finished on another (say created at 11pm and finished at
        1am). Therefore, one may want to retouch the timestamp before
        publishing.""")
    parser_new_post.add_argument("filename",
                                 help="path or basename of the source file, "
                                 "e.g., 2015-05-05-new-blog-new-start.md")
    parser_new_post.set_defaults(func=touch_cli)

    parser_generate = subparsers.add_parser(
        "generate", aliases=["g", "gen"],
        description="Generate new or changed objects.")
    parser_generate.set_defaults(func=cli.generate)

    parser_regenerate = subparsers.add_parser(
        "regenerate", aliases=["r", "regen"],
        description="Regenerate the entire blog afresh.")
    parser_regenerate.set_defaults(func=cli.regenerate)

    parser_new_post = subparsers.add_parser(
        "preview", aliases=["p", "pre"],
        description="Serve the blog locally and auto regenerate upon changes.")
    parser_new_post.set_defaults(func=preview)

    parser_new_post = subparsers.add_parser(
        "deploy", aliases=["d", "dep"],
        description="Deploy build/ to origin/master without regenerating.")
    parser_new_post.set_defaults(func=deploy)

    parser_new_post = subparsers.add_parser(
        "gen_deploy", aliases=["gd", "gendep"],
        description="Rebuild entire blog and deploy build/ to origin/master.")
    parser_new_post.set_defaults(func=gen_deploy)

    parser_new_post = subparsers.add_parser(
        "edit", aliases=["e", "ed"],
        description="Bring up post selector to select post for editing.")
    parser_new_post.set_defaults(func=edit_existing_post)

    with utils.init_colorama():
        args = parser.parse_args()
        returncode = args.func(args)
    exit(returncode)


if __name__ == '__main__':
    main()