generate

#!/usr/bin/python3

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2, as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

# Variables
CONTENT_PATH = "content"
TARGET_PATH = "output"

# Required imports
import bs4
import jinja2
import json
import markdown
import markdown.extensions.codehilite
import pygments.formatters
import os
import shutil
import subprocess


def pretty_print(html):
    """
        Returns a cleanly indented version of the provided html code.
    """

    soup = bs4.BeautifulSoup(html)
    return soup.prettify(formatter="html")


def content_path(filename):
    """
        Returns the path to a file from the content directory
    """
    return os.path.join(CONTENT_PATH, filename)


def load_json(filename):
    """
        Load the provided JSON file from the content directory.
    """

    with open(content_path(filename), "r") as fd:
        return json.load(fd)


def gen_menu(structure, override, prefix):
    """
        Generate a list representing the menu.
        If there are sub-menus, the content of the entry will be a
        nested list.

        The function takes the structure list and an optional override
        dict used to handle translations.

        When all internal links must be prefixed, prefix should be set.
    """

    menu = []
    sub_menu = None
    sub_menu_title = None

    for entry in structure:
        item = dict(entry)

        # Apply the override (if any)
        item.update(override.get(entry['path'], {}))

        if prefix:
            # Prepend the prefix to all internal links
            item['path'] = "%s%s" % (prefix, item['path'])

        if "generator" not in item:
            # Menu splitter
            item['path'] = ""
        elif item['generator'] == "link":
            # External link
            item['path'] = item['meta']['url']

        if not "menu" in item:
            continue

        if len(item['menu']) == 1:
            if "generator" in item and item['generator'] == "manpages":
                # Dynamic manpage menu
                man_menu = []
                man_path = item['meta']['dir'].lstrip("/")

                for entry in sorted(os.listdir(man_path)):
                    if not os.path.isfile("%s/%s" % (man_path, entry)):
                        continue

                    entry_link = "/%s/%s" % (item['path'].lstrip("/"), entry)
                    man_menu.append((entry_link, entry))

                menu.append((man_menu, item['menu'][0]))
            else:
                # Simple menu entry
                menu.append((item['path'], item['menu'][0]))
        else:
            # Drop-down menu
            if sub_menu_title != item['menu'][0]:
                if sub_menu:
                    menu.append((sub_menu, sub_menu_title))
                sub_menu = []
                sub_menu_title = item['menu'][0]

            if "generator" in item and item['generator'] == "manpages":
                # Dynamic manpage menu
                man_menu = []
                man_path = item['meta']['dir'].lstrip("/")

                for entry in sorted(os.listdir(man_path)):
                    if not os.path.isfile("%s/%s" % (man_path, entry)):
                        continue

                    entry_link = "/%s/man%s/%s.html" % (
                        item['path'].lstrip("/"), entry.split(".")[-1], entry)
                    man_menu.append((entry_link, entry))

                sub_menu.append((man_menu, item['menu'][-1]))
            else:
                # Simple menu entry
                sub_menu.append((item['path'], item['menu'][-1]))

    # Process the last drop-down menu
    if structure and sub_menu_title:
        if sub_menu:
            menu.append((sub_menu, sub_menu_title))

    return menu


def gen_pages(structure, override, prefix, **variables):
    for entry in structure:
        item = dict(entry)

        # Apply the override (if any)
        item.update(override.get(entry['path'], {}))

        # Store the original path (required for translations)
        page_raw_path = item['path']

        if prefix:
            # Prepend the prefix to all internal links
            item['path'] = "%s%s" % (prefix, item['path'])

        if "generator" not in item or item['generator'] == "link":
            # Skip the virtual entries (external links, splitters)
            continue

        # Generate the target path and create any missing directory
        output_path = "%s%s/index.html" % (TARGET_PATH, item['path'])
        if not os.path.exists(os.path.dirname(output_path)):
            os.makedirs(os.path.dirname(output_path))

        # Generate the page content
        content = ""
        template = "page.tpl.html"
        hilite_config = [('linenums', None),
                         ('guess_lang', False),
                         ('noclasses', False)]
        codehilite = markdown.extensions.codehilite.makeExtension(
            hilite_config)

        if item['generator'] == "html":
            with open(content_path(item['meta']['input']), "r") as fd:
                content = fd.read()
        elif item['generator'] == "markdown":
            with open(content_path(item['meta']['input']), "r") as fd:
                content = markdown.markdown(fd.read(), extensions=[codehilite])
        elif item['generator'] == "downloads":
            # Support a markdown description before the download table
            if "input" in item['meta']:
                with open(content_path(item['meta']['input']), "r") as fd:
                    content = markdown.markdown(fd.read(),
                                                extensions=[codehilite])

            downloads = []
            for entry in sorted(os.listdir(item['meta']['dir'].lstrip("/")),
                                reverse=True):
                if entry.endswith(".asc"):
                    continue

                fs_path = "%s/%s" % (item['meta']['dir'].lstrip("/"), entry)

                download = {}
                download['filename'] = entry
                download['path'] = "%s/%s" % (item['meta']['dir'], entry)
                download['signame'] = None
                download['sigpath'] = None
                if os.path.exists("%s.asc" % fs_path):
                    download['signame'] = "%s.asc" % download['filename']
                    download['sigpath'] = "%s.asc" % download['path']
                download['size'] = "%sK" % \
                    (round(os.stat(fs_path).st_size / 1024, 2))

                downloads.append(download)

            variables['downloads'] = downloads
            template = "downloads.tpl.html"
        elif item['generator'] == "manpages":
            man_path = item['meta']['dir'].lstrip("/")

            for entry in sorted(os.listdir(man_path)):
                if not os.path.isfile("%s/%s" % (man_path, entry)):
                    continue

                section = entry.split(".")[-1]

                output_man_path = "%s%s/man%s/%s.html" % (TARGET_PATH,
                                                          item['path'],
                                                          section, entry)

                if not os.path.exists(os.path.dirname(output_man_path)):
                    os.makedirs(os.path.dirname(output_man_path))

                man2html = subprocess.Popen(["man2html", "-r",
                                             "%s/%s" % (man_path, entry)],
                                            stdout=subprocess.PIPE,
                                            universal_newlines=True)
                man2html.wait()
                soup = bs4.BeautifulSoup(man2html.stdout.read())

                # Remove all broken links
                for link in soup.findAll("a"):
                    if not link.get("href", None):
                        continue

                    if link['href'] in ("../index.html",):
                        continue

                    if link['href'].startswith("mailto"):
                        continue

                    if link['href'].startswith("#"):
                        continue

                    filename = link['href'].split("/")[-1].split(".html")[0]
                    if filename in os.listdir(man_path):
                        continue

                    link.replace_with_children()

                contents = soup.findAll("body")[0].contents
                content = "".join([str(entry)
                                   for entry in contents[1:]])

                content = "<div class='manpage'>%s</div>" % content

                entry_link = "/%s/man%s/%s.html" % (
                    item['path'].lstrip("/"), section, entry)

                template = env.get_template(template)
                with open(output_man_path, "w+") as fd:
                    fd.write(pretty_print(
                        template.render(page_path=entry_link,
                                        page_raw_path="%s/man%s/%s.html" % (
                                            page_raw_path, section, entry),
                                        page_title="%s - %s" % (item['title'],
                                                                entry),
                                        page_menu=item['menu'] + [entry],
                                        content=content,
                                        **variables)))

            content = "<ul>"
            for entry in sorted(os.listdir(man_path)):
                if not os.path.isfile("%s/%s" % (man_path, entry)):
                    continue

                section = entry.split(".")[-1]
                entry_link = "/%s/man%s/%s.html" % (
                    item['path'].lstrip("/"), section, entry)

                content += "<li><a href=\"%s\">%s</a></li>" % (entry_link,
                                                               entry)
            content += "</ul>"

        elif item['generator'] == "directory":
            os.rmdir("%s/%s" % (TARGET_PATH, item['path']))
            shutil.copytree(content_path(item['meta']['input']),
                            "%s/%s" % (TARGET_PATH, item['path']))
            continue

        # Pass all the variables to the template and generate the html
        template = env.get_template(template)
        with open(output_path, "w+") as fd:
            fd.write(pretty_print(
                template.render(page_path=item['path'],
                                page_raw_path=page_raw_path,
                                page_title=item['title'],
                                page_menu=item['menu'],
                                content=content,
                                **variables)))


# Load the configuration and structure
env = jinja2.Environment(loader=jinja2.FileSystemLoader('templates/'))
config = load_json("CONFIG.json")
structure = load_json("STRUCTURE.json")

# Wipe the output directory clean
if os.path.exists(TARGET_PATH):
    shutil.rmtree(TARGET_PATH)
os.mkdir(TARGET_PATH)
shutil.copytree("static", "%s/static" % TARGET_PATH)
shutil.copytree("downloads", "%s/downloads" % TARGET_PATH)
os.symlink("static/img/favicon.ico", "%s/favicon.ico" % TARGET_PATH)

with open("%s/static/css/pygments.css" % TARGET_PATH, "w+") as fd:
    fd.write(pygments.formatters.HtmlFormatter().get_style_defs())

# Start generating the website
for language in config['languages']:
    # Load a translation override
    override = {}
    try:
        override = {entry['path']: entry
                    for entry in load_json("STRUCTURE.%s.json" % language[0])}
    except FileNotFoundError:
        pass

    # Figure out the translation prefix
    lang_prefix = ""
    if language[0]:
        lang_prefix = "/%s" % language[0]

    # Generate the menu
    menu = gen_menu(structure, override, lang_prefix)

    # Generate all the pages
    gen_pages(structure, override, lang_prefix,
              menu=menu, page_language=language,
              languages=config['languages'])