#!/usr/bin/env python3
# publish_cloud - Publish cloud documents
# Author: Perette Barella
# Copyright 2018 Devious Fish.  All rights reserved.
# vim: ts=4:sw=4:expandtab
# $Id: publish_cloud.py 65 2022-03-10 16:10:24Z perette $

import sys, os, re, functools, argparse

# Given dictionaries of user options and default options, flesh
# out the user's options with any missing values.
# Also sanity check user provided types by validating against defaults.
# @return The fleshed out user defaults.
def complete_options (user_options, option_defaults):
    assert (user_options is None or isinstance (user_options, dict))
    assert (isinstance (option_defaults, dict))

    # If user didn't provide options, use the defaults wholesale
    if user_options is None:
        return option_defaults

    # Merge missing values
    for value in option_defaults:
        if value not in user_options or user_options [value] is None:
            user_options [value] = option_defaults [value]
        elif option_defaults [value] is not None:
            assert (type (user_options [value]) == type (option_defaults [value]))
    return user_options


def split_filename (path):
    assert (isinstance (path, str))

    filename = os.path.basename (path)
    return (os.path.splitext (os.path.basename (path)))


def entity_encode (*text):
    message = "";
    for bit in text:
        message += (bit.replace ('&', '&amp;').
                    replace ('<', '&lt;').
                    replace ('>', '&gt;')) + ' '
    return message.strip()



# Determine if a program is available.
# @param program The program name or path.
# @param path A path to search in.  If None, defaults to $PATH.
# If `program` includes a path, then a path search is not done.
# @return True if found, false otherwise.
def whence (program, path = None):
    def is_executable (test_path):
        return os.path.isfile (test_path) and os.access (test_path, os.X_OK)

    prog_path, prog_file = os.path.split (program)
    if (prog_path == ''):
        if path == None:
            path = os.environ ['PATH']
        for trypath in path.split (':'):
            test = trypath + '/' + program
            if is_executable (test):
                return test
    else:
        if (is_executable (program)):
            return program
    return None
            
    

# Create a list of available file translations
# 'how' contains program that performs conversion.
# None wraps content in HTML using built-in function.
def get_translators ():
    # Convert text files to HTML using built-in routine.
    trans = {
        'cpt': {
            'target': 'not_generated',
            'how': "echo '{source}' '{destination}' >/dev/null",
            'publish': False
        },
        'txt': {
            'target': 'html',
            'how': None
        },
        '.file': {
            'target': None,
            'how': "ln -f '{source}' '{destination}'",
            'publish': False
        },
        'Makefile': {
            'target': None,
            'how': "ln -f '{source}' '{destination}'",
            'publish': False
        },
        'index': {
            'target': None,
            'how': None
        }
    }

    # For graphics files or PDF, publish as-is.
    for filetype in [ 'pdf', 'html', 'gif', 'png', 'jpeg' ]:
        trans [filetype] = {
            'target': filetype,
            'how': "ln -f '{source}' '{destination}'"
        }

    # If we have multimarkdown or mmd, use those.
    # Otherwise, treat as a text file.
    if whence ('multimarkdown'):
        trans ['md'] = {
            'target': 'html',
            'how': "multimarkdown --full '{source}' > '{destination}'"
        }
    elif whence ('mmd'):
        trans ['md'] = {
            'target': 'html',
            'how': "mmd --full '{source}' > '{destination}'"
        }
    else:
        trans ['md'] = trans ['txt']

    # If reStructuredText is installed, use that.
    # Otherwise, treat as a text file.
    rsthandler = None
    if whence ('rst2html'):
        rsthandler = 'rst2html'
    elif whence ('rst2html.py'):
        rsthandler = 'rst2html.py'
    if rsthandler != None:
        trans ['rst'] = {
            'target': 'html',
            'how': rsthandler + " --input-encoding='UTF-8' --input-encoding-error-handler=replace --output-encoding='UTF-8:strict' --smart-quotes=yes --no-doc-info '{source}' '{destination}'"
        }
    else:
        trans ['rst'] = trans ['txt']

    # If we have `unrtf`, we can process RTFs.
    if whence ('unrtf'):
        trans ['rtf'] =  {
            'target': 'html',
            'how': "unrtf --nopict --quiet --html '{source}' > '{destination}'"
        }
    return trans;


# Create a nice, friendly title from a filename.
def get_name_from_filename (path):
    filename, extension = split_filename (path)
    name = filename.replace ('_', ' ').replace ('-', ' ').title();
    return re.sub ('[a-z][A-Z]', lambda match: match.group(0)[0:1] + ' ' +
                                               match.group(0)[1:], name)


# Determine if a string is a valid restructured text heading over/underline.
# @param text The potential over/underline.
# @param char The character the over/underline should be composed of.
# If none, any permitted characters are accepted.
def is_reStructuredTextLine (text, char):
    assert (isinstance (text, str))
    if (text == ''):
        return False
    if not char:
        char = text [0]
    if not char in '=-`:\'"~^_*+#<>"':
        return False
    return text == char * len (text)
    

# Class that retrieves and stores data about files
class FileInfo:
    info = { }
    metadata = { }

    is_metadata = re.compile ("([A-Za-z][A-Za-z0-9-_]*):[ \t]*(\\S.*?)[ \t]*$")
    is_rstmetadata = re.compile (":([A-Za-z][A-Za-z0-9-_]*):[ \t]*(\\S.*?)$[ \t]*")
    is_heading = re.compile ("#+[ \t]*([^# \t].+)$")
    is_underline = re.compile ("(---*|===*)[ \t]*$")


    def getTextMetadata (self, source, filetype):
        assert (filetype in [None, 'md', 'rst', 'txt'])
        state = 'iniz'
        between = True
        metadata = { }
        field = None;
        lastline = None;
        restructured_heading_char = None
        debug = False
        if debug:
               print ("Starting metadata extraction")
        for line in source:
            line = line.strip();
            if debug:
                print (line)
            # Successive blank lines are ignored.
            if line == '' and between:
                continue
            # Blank lines have meaning.
            if line == '':
                if not metadata and state == 'para' and lastline:
                    # Line on its own, leading the document, is the title
                    metadata ['title'] = lastline
                    break
                between = True
                lastline = None;
                field = None;
                if state == 'para':
                    break
                state = 'para'
                continue
            between = False
            
            # Look for markdown # Header format, if we haven't seen
            # evidence for reStructuredText yet.
            if filetype in [None, 'txt', 'md']:
                head = self.is_heading.match (line)
                if head:
                    metadata ['format'] = 'md'
                    if debug:
                        print ("Found markdown heading:", line)
                    if not 'title' in metadata:
                        metadata ['title'] = head.group (1)
                    return metadata;
    
            # Check for metadata.  If it's in reStructuredText format, set that mode.
            meta = None
            if filetype in [None, 'txt', 'rst']:
                meta = self.is_rstmetadata.match (line)
            if meta:
                if debug:
                    print ("Found reStructuredText metadata: ", line)
                filetype = 'rst'
            if not meta and filetype in [None, 'txt', 'md']:
                meta = self.is_metadata.match (line)
                if meta:
                    if debug:
                        print ("Found markdown metadata: ", line)
                    filetype = 'md'
                    
            # Stash any metadata
            if meta:
                if state == 'iniz':
                    state = 'header'
                if state != 'header':
                    if debug:
                        print ("Metadata outside header: ", state, ':', line)
                    break
                field = meta.group(1).lower()
                metadata [field] = meta.group(2)
                continue
    
            if state == 'header':
                assert field != None
                if debug:
                    print ("Metadata continuation: ", line)
                metadata [field] += ' ' + line
                continue
    
            assert state == 'para' or state == 'iniz'
            assert field == None
            state = 'para'
    
            # Look for heading marker lines.
            # * reStructuredText is identified by over and underline, or just underline
            #   if the header metadata indicated reStructuredText.
            # * Markdown is identified by underline, or the # header format (above).
            if filetype in [None, 'txt', 'md']:
                under = self.is_underline.match (line)
                if not under and filetype != 'md':
                    under = is_reStructuredTextLine (line, restructured_heading_char)
                    if under:
                        if debug:
                            print ("Found restructured text heading delimiter")
                        filetype = 'rst'
                elif under and debug:
                    print ("Found common heading delimiter")
            else:
                under = is_reStructuredTextLine (line, restructured_heading_char)
                if under:
                    if debug:
                        print ("Found restructured text heading delimiter")
                    filetype = 'rst'
            if under:
                restructured_heading_char = line [0]
            if under and not lastline:
                if filetype == 'md':
                    if debug:
                        print ("Found underline without heading, not multimarkdown.")
                    break
                filetype = 'rst'
                continue
            if not under and lastline:
                if debug:
                    print ("bare text encountered, plain text file: ", line)
                break
            if under: # and lastline
                metadata ['format'] = 'rst' if filetype == 'rst' else 'md'
                if not 'title' in metadata:
                    metadata ['title'] = lastline
                if debug:
                    print ("Heading delimiter accepted, filetype: ", metadata ['format'])
                return metadata;
            # Neither under nor lastline
            if debug:
                print ("Tentative title: ", line)
            lastline = line
        # If we get here, the file is neither markdown nor restructured text.
        if debug:
            print ("Probing reached end of file.", metadata)
        metadata ['format'] = 'txt' if filetype == None else filetype
        return metadata


    # Determine metadata for a file.
    # Minimal metadata is 'format' and 'title'.
    # Depending on file type, this may involve reading the file;
    # if this is not possible, they are determined from the file's name.
    # The resulting data is cached in case it is needed again.
    def probeDoc (self, path, doctype):
        if path in self.metadata:
            return self.metadata [path]
        if (doctype in [ None, 'txt', 'md', 'rst' ]):
            with open (path, encoding='utf-8') as source:
                info = self.getTextMetadata (source, doctype)
            if not 'title' in info:
                info ['title'] = get_name_from_filename (path)
        else:
            info = {
                'title': get_name_from_filename (path),
                'format': doctype
            }
        self.metadata [path] = info
        return info

 
    # Determine the document type either from the file extension,
    # or by probing the file contents.
    def determineDocumentType (self, path):
        filename, extension = split_filename (path)
        if (filename [0] == '.'):
            # Allow for .htaccess and such
            return '.file'
        extension = extension.lower()
        type_remap = {
            '.jpg': 'jpeg',
            '.htm': 'html'
        }
        if extension in type_remap:
            extension = type_remap [extension]
        elif extension != '':
            extension = extension [1:]
        if extension != '' and extension != 'txt':
            return extension
        if filename == 'Makefile' or filename == 'makefile':
            return 'Makefile'
        info = self.probeDoc (path, None)
        # print (path, ":", info ['format'])
        return info ['format']
    
    
    def collectFileData (self, path):
        doc_type = self.determineDocumentType (path)
        try:
            meta = self.probeDoc (path, doc_type)
        except:
            self.info [path] = None
            raise
        doc = {
            'type': doc_type,
            'title': meta ['title'],
            'metadata': meta
        }
        doc ['filename'], doc ['extension'] = split_filename (path)
        self.info [path] = doc
        return doc

    def getInfo (self, path):
        if not path in self.info:
            return self.collectFileData (path)
        if not files [path]:
            raise ValueError ('Invalid document (cached result)')
        return files [path];



# Determine if a directory should be ignored.
def ignored_directory (name):
    return (name == 'RCS' or name == 'CVS' or name[0] == '.')


# Given a path, make a hierarchical list of all publishable
# documents contained therein.
def get_document_list (path, offset = ''):
        file_info = FileInfo ()
        documents = []
        for entry in os.scandir (path):
            if (entry.is_dir()):
                if ignored_directory (entry.name):
                    continue;
                doc = {
                    'type': 'index',
                    'title': get_name_from_filename (entry.name),
                    'filename': entry.name,
                    'extension': None,
                    'contents': get_document_list (entry.path, offset + entry.name + '/')
                }
            else:
                try:
                    doc = file_info.getInfo (entry.path)
                except ValueError as err:
                    print (entry.path, ': ', err)
                    continue
            doc ['source'] = entry.path
            doc ['path'] = offset
            documents.append (doc)
        return documents



# Convert a text document to HTML
def publish_text_to_html (source, destination, metadata):
    with open (source, encoding='utf-8') as src:
        with open (destination, "w", encoding='utf-8') as dest:
            dest.write ("<!DOCTYPE html>\n"
                        "  <html>\n"
                        "    <head>\n"
                        "      <meta charset='UTF-8'>\n"
                        "      <title>" + entity_encode (metadata ['title']) + "</title>"
                        "    </head>\n"
                        "  <body>\n"
                        "    <pre>")
            for line in src:
                dest.write (entity_encode (line))
                dest.write ('\n')
            dest.write (    "</pre>\n"
                        "  </body>\n"
                        "</html>\n")


# Indicate if the first file is last modified more recently than a second file.
# An existent file is considered newer than a non-existent file
def file_newer_than_file (patha, pathb):
    try:
        amtime = os.path.getmtime (patha)
        try:
            bmtime = os.path.getmtime (pathb)
            return amtime > bmtime
        except os.error:
            return True
    except os.error:
        return False

def sanitize_filename (name):
    return name.replace (" ", "_")

# Publish all documents in a collection
def publish_documents (translators, documents, destination):
    if not destination.endswith ('/'):
        destination += '/'
    if not os.path.isdir (destination):
        os.mkdir (destination)
    for doc in documents:
        if not doc ['type'] in translators:
            print ("Cannot transform:", doc ['source'])
            doc ['publish'] = False
            continue
        xlate = translators [doc ['type']]
        dest = doc ['path'] + sanitize_filename (doc ['filename'])
        dest += "" if xlate ['target'] == None else ('.' + xlate ['target'])
        doc ['url'] = dest
        dest = destination + dest
        doc ['destination'] = dest
        if doc ['type'] == 'index':
            doc ['publish'] = True
            if not os.path.isdir (dest):
                os.mkdir (dest)
            publish_documents (translators, doc ['contents'], destination)
        else:
            doc ['publish'] = True if 'publish' not in xlate else xlate ['publish']
            if file_newer_than_file (dest, doc ['source']):
                continue
            if xlate ['how'] == None:
                publish_text_to_html (doc ['source'], dest, doc ['metadata'])
            else:
                transform = xlate ['how'].format (destination = dest,
                                                  source = doc ['source'])
                exit_status = os.system (transform);
                if (exit_status != 0):
                    os.remove (dest)
                    print ("{command}: Exited non-cleanly; not publishing."
                            .format (command=transform), file=sys.stderr)
                    doc ['publish'] = False
    return 0


# Sort alphabetically, regardless of type.
def sort_pure_alphabetic (a, b):
    av = a['title'].lower();
    bv = b['title'].lower();
    if av < bv:
        return -1
    if av > bv:
        return 1
    return 0


# Sort directories/files, alphabetically therein.
def sort_by_type (a, b):
    av = a['type']
    bv = b['type']
    if av == 'index' and bv == 'index':
        return sort_pure_alphabetic (a, b)
    if av != 'index' and bv != 'index':
        return sort_pure_alphabetic (a, b)
    return -1 if av == 'index' else 1


# Write an index of all documents in a collection
def write_index (dest, documents, sort_order, indent=''):
    assert (sort_order == 'type' or sort_order == 'name')

    dest.write ("{indent}<ul>\n".format (indent=indent))
    sort = sort_by_type if sort_order == 'type' else sort_pure_alphabetic
    for doc in sorted (documents, key = functools.cmp_to_key (sort)):
        if not doc ['publish']:
            continue
        title = doc ['title']
        if doc ['type'] == 'index':
            dest.write ("{indent}  <li>{title}...\n".format (indent=indent,
                        title = entity_encode (title)))
            write_index (dest, doc ['contents'], sort_order, indent + '    ')
        else:
            dest.write ("{indent}  <li><a href='{url}'>{title}</a>\n".format (indent=indent,
                        url=doc ['url'], title = entity_encode (title)))
    dest.write ("{indent}</ul>\n".format (indent=indent))


def get_published_file_list (documents):
    assert (isinstance (documents, list))

    files = []
    for doc in documents:
        if 'destination' in doc:
            files.append (doc ['destination'])
            if doc ['type'] == 'index':
                files += get_published_file_list (doc ['contents'])
    return files



    
# Purge lingering files from prior runs
# @param path The path to remove files at
# @param exceptions A list of file paths that should remain.
# @param really If false, reports what would be done instead of doing it.
# @return None
def scrub_directory (path, exceptions, really = True):
    assert (isinstance (path, str))
    assert (isinstance (exceptions, list))

    # Make a hash able so we go faster
    retain = {}

    def recurse (path):
        assert (isinstance (path, str))

        for entry in os.scandir (path):
            if entry.is_dir():
                recurse (entry.path)
            if entry.path not in retain:
                if not really:
                    print ("{path}: remove".format (path=entry.path))
                elif entry.is_dir():
                    os.rmdir (entry.path)
                else:
                    os.remove (entry.path)
            elif not really:
                print ("{doc}: retain".format (doc=entry.path))
                
    for doc in exceptions:
        retain [doc] = True
    recurse (path)
    


def publish_to_cloud (source, destination, options = None):
    assert (isinstance (source, str))
    assert (isinstance (destination, str))
    options = complete_options (options, {
        'sort_order': 'type',
        'index_format': 'none',
    })
    assert (options ['index_format'] in ['html', 'm4', 'none'])
        
    # Get the document list
    if publish_to_cloud.translators == None:
        publish_to_cloud.translators = get_translators()
    documents = get_document_list (source)

    # Process the documents
    publish_documents (publish_to_cloud.translators,
                       documents, destination)
    manifest = get_published_file_list (documents)
    scrub_directory (destination, manifest)

    # Write the index
    if options ['index_format'] == 'none':
        return
    index_file = destination + '/index.' + options ['index_format'];
    with open (index_file, "w", encoding='utf-8') as target:
        if options ['index_format'] == 'html':
            target.write ("<!DOCTYPE html>\n"
                          "  <html>\n"
                          "    <head>\n"
                          "      <meta charset='UTF-8'>\n"
                          "      <title>Documents</title>"
                          "    </head>\n"
                          "  <body>\n")
        write_index (target, documents,
                     options ['sort_order'] if 'sort_order' in options else 'type')
        if options ['index_format'] == 'html':
            target.write ("  </body>\n"
                          "</html>\n")



# Static variable for above function
publish_to_cloud.translators = None



def validate_directory (path):
    if not os.path.exists (path):
        sys.exit ("{dir}: Does not exist".format (dir=path))
    if not os.path.isdir (path):
        sys.exit ("{dir}: Not a directory".format (dir=path))


# Fix the goddamn argument parser to respect newlines in strings.  Dumbass thing.
class FixedArgumentFormatter(argparse.HelpFormatter):
    def _split_lines(self, text, width):
        results = []
        for paragraph in text.splitlines ():
            results += argparse.HelpFormatter._split_lines(self, paragraph, width)
        return results

    def _format_text (self, text):
        if text == None:
            return argparse.HelpFormatter.add_text (self, text)
        results = ''
        for paragraph in text.splitlines ():
            results += argparse.HelpFormatter._format_text (self, paragraph)
            results = results [:-1]
        return results + '\n'




if __name__ == "__main__":
    parser = argparse.ArgumentParser (formatter_class = FixedArgumentFormatter,
        description="%(prog)s - Prepare documents for web publication.  ",
        epilog="%(prog)s prepares documents for web publication.  "
        "All documents in the source directory that have a known publication "
        "scheme are processed into a duplicate file hierarchy in the destination "
        "directory.  An index is produced at the top of the destination hierarchy "
        "linking to the published files.\n\n"
        "Copyright 2018 Devious Fish.\nAuthor: Perette Barella")
    
    parser.add_argument ("-s", "--sort", action="store", dest="sort_order",
                       choices=[ "type", "name" ], default="type",
                       help="Order may be:\n"
                            "- type:  by file type, then alphabetical\n"
                            "- name:  purely alphabetical")
    parser.add_argument ("-i", "--index-format", action="store", dest="index_format",
                         choices=[ "html", "m4", "none"], default="html",
                         help="Format for document index")
    home = os.environ ['HOME'] if 'HOME' in os.environ else None
    parser.add_argument ("source", nargs = '?',
             default = None if home == None else home + '/Documents/Cloud',
             help="Path to original documents")
    parser.add_argument ("target", nargs = '?',
             default = None if home == None else home + '/Sites/Cloud/Documents',
             help="Path to publication directory")

    args = parser.parse_args ()                      
    if (args.target == None):
        sys.exit ("{prog}: Source and destination must be specified.".format (
                 prog = os.path.basename (__file__)))
    validate_directory (args.source)
    validate_directory (args.target)

    publish_to_cloud (args.source, args.target, {
                        'sort_order': args.sort_order,
                        'index_format': args.index_format
                     })



