beancount.web

def internal_redirect(app, path_depth):
    """A version of bottle's mountpoint_wrapper() that we call explicitly.

    Bottle supports a mount() method that allows on to install an application on
    a subpath of the main application. However, it does this on a fixed path. We
    want to manually intercept the lazy creation or fetching of a view and call
    for a redirect explicitly (via bottle's mountpoint_wrapper() function).
    However, this function is hidden within the scope of a the Bottle.mount()
    method; if it were defined globally we would just use it, but it is not. So
    we copy if here. This is directly lifted from Bottle.mount() and edited
    minimally.

    Args:
      app: A Bottle instance.
      path_depth: The number of request path components to skip for the mount.
        For example, if our subapplication is mount on /view/all, then the path
        depth is 2.
    Returns:
      A Bottle HTTPResponse object.
    Raises:
      Exception: Any exception, depending on the callback.
    """
    # pylint: disable=invalid-name
    try:
        request.path_shift(path_depth)
        rs = bottle.HTTPResponse([])
        def start_response(status, headerlist, exc_info=None):
            if exc_info:
                try:
                    _raise(*exc_info)
                finally:
                    exc_info = None
            rs.status = status
            for name, value in headerlist: rs.add_header(name, value)
            return rs.body.append
        body = app(request.environ, start_response)
        if body and rs.body: body = itertools.chain(rs.body, body)
        rs.body = body or rs.body
        return rs
    finally:
        request.path_shift(-path_depth)

def iterlinks(html, html_path):
    """Find links targets in HTML text.

    This deals with both absolute and relative links, and it external links to
    external sites.

    Args:
      html: An lxml document node.
      html_path: The URL of the document node.
    Yields:
      URL strings, where found.

    """
    html_dir = path.dirname(html_path)
    for element, attribute, link, pos in lxml.html.iterlinks(html):
        url = urllib.parse.urlparse(link)
        if url.scheme or url.netloc:
            continue  # Skip external urls.
        link = url.path
        if not link:
            continue
        if not path.isabs(link):
            link = path.join(html_dir, link)
        yield link

def scrape_urls(url_format, callback, ignore_regexp=None):
    """Recursively scrape pages from a web address.

    Args:
      url_format: The pattern for building links from relative paths.
      callback: A callback function to invoke on each page to validate it.
        The function is called with the response and the url as arguments.
        This function should trigger an error on failure (via an exception).
      ignore_regexp: A regular expression string, the urls to ignore.
    Returns:
      A set of all the processed URLs and a set of all the skipped URLs.
    """
    # The set of all URLs seen so far.
    seen = set()

    # The list of all URLs to process. We use a list here so we have
    # reproducible order if we repeat the test.
    process_list = ["/"]

    # A set of all the URLs processed and skipped everywhere.
    all_processed_urls = set()
    all_skipped_urls = set()

    # Loop over all URLs remaining to process.
    while process_list:
        url = process_list.pop()

        logging.debug("Processing: %s", url)
        all_processed_urls.add(url)

        # Fetch the URL and check its return status.
        response = urllib.request.urlopen(url_format.format(url))

        # Generate errors on redirects.
        redirected_url = urllib.parse.urlparse(response.geturl()).path
        if redirected_url != url:
            logging.error("Redirected: %s -> %s", url, redirected_url)

        # Read the contents. This can only be done once.
        response_contents = response.read()

        skipped_urls = set()
        content_type = response.info().get_content_type()
        if content_type == 'text/html':
            # Process all the links in the page and register all the unseen links to
            # be processed.
            html_root = lxml.html.document_fromstring(response_contents)
            for link in iterlinks(html_root, url):

                # Skip URLs to be ignored.
                if ignore_regexp and re.match(ignore_regexp, link):
                    logging.debug("Skipping: %s", link)
                    skipped_urls.add(link)
                    all_skipped_urls.add(link)
                    continue

                # Check if link has already been seen.
                if link in seen:
                    logging.debug('Seen: "%s"', link)
                    continue

                # Schedule the link for scraping.
                logging.debug('Scheduling: "%s"', link)
                process_list.append(link)
                seen.add(link)

        else:
            html_root = None

        # Call back for processing.
        callback(url, response, response_contents, html_root, skipped_urls)

    return all_processed_urls, all_skipped_urls

def validate_local_links(filename):
    """Open and parse the given HTML filename and verify all local targets exist.

    This checks that all the files pointed to by the file we're processing are
    files that exist on disk. This can be used to validate that a baked output
    does not have links to files that do not exist, that all the links are valid.

    Args:
      filename: A string, the name of the HTML file to process.
    Returns:
      A pair of:
        missing: A set of strings, the names of links to files that do not exist.
        empty: A boolean, true if this file is empty.
    """
    filedir = path.dirname(filename)
    contents = open(filename, 'rb').read()

    empty = len(contents) == 0
    missing = set()
    if not empty:
        html = lxml.html.document_fromstring(contents)
        if html is not None:
            for element, attribute, link, pos in lxml.html.iterlinks(html):
                urlpath = urllib.parse.urlparse(link)
                if urlpath.scheme or urlpath.netloc:
                    continue
                if path.isabs(urlpath.path):
                    continue
                target = path.normpath(path.join(filedir, urlpath.path))
                if not path.exists(target):
                    missing.add(target)

    return missing, empty

def validate_local_links_in_dir(directory):
    """Find all the files under the given directory and validate all their links.

    Args:
      directory: A string, the root directory whose files to process.
    Returns:
      A tuple of:
        files: A list of all the filenames found and processed.
        missing: A set of strings, the names of links to files that do not exist.
        empty: A boolean, true if this file is empty.
    """
    logging.basicConfig(level=logging.INFO,
                        format='%(levelname)-8s: %(message)s')
    allfiles = []
    missing, empty = set(), set()
    for root, dirs, files in os.walk(directory):
        for filename in files:
            afilename = path.join(root, filename)
            allfiles.append(afilename)
            logging.info("Validating: '%s'", afilename)
            missing, is_empty = validate_local_links(afilename)
            if is_empty:
                empty.add(afilename)
    return allfiles, missing, empty

@viewapp.route('/activity', name='activity')
def activity():
    "Render the update activity."
    return render_global(
        pagetitle="Update Activity",
        contents=render_real_report(misc_reports.ActivityReport,
                                    request.view.real_accounts,
                                    app.price_map,
                                    request.view.price_date,
                                    leaf_only=False))

def auto_reload_input_file(callback):
    """A plugin that automatically reloads the input file if it changed since the
    last page was loaded."""
    def wrapper(*posargs, **kwargs):
        filename = app.args.filename

        if loader.needs_refresh(app.options):
            logging.info('Reloading...')

            # Save the source for later, to render.
            with open(filename, encoding='utf8') as f:
                app.source = f.read()

            # Parse the beancount file.
            entries, errors, options_map = loader.load_file(filename)

            # Print out the list of errors.
            if errors:
                # pylint: disable=unsupported-assignment-operation
                request.params['render_overlay'] = True
                print(',----------------------------------------------------------------')
                printer.print_errors(errors, file=sys.stdout)
                print('`----------------------------------------------------------------')

            # Save globals in the global app.
            app.entries = entries
            app.errors = errors
            app.options = options_map
            app.account_types = options.get_account_types(options_map)

            # Pre-compute the price database.
            app.price_map = prices.build_price_map(entries)

            # Pre-compute the list of active years.
            app.active_years = list(getters.get_active_years(entries))

            # Reset the view cache.
            app.views.clear()

        else:
            # For now, the overlay is a link to the errors page. Always render
            # it on the right when there are errors.
            if app.errors:
                # pylint: disable=unsupported-assignment-operation
                request.params['render_overlay'] = True

        return callback(*posargs, **kwargs)
    return wrapper

@viewapp.route('/balsheet', name='balsheet')
def balsheet():
    "Balance sheet."
    return render_view(pagetitle="Balance Sheet",
                       contents=render_real_report(balance_reports.BalanceSheetReport,
                                                   request.view.closing_real_accounts,
                                                   app.price_map,
                                                   request.view.price_date,
                                                   leaf_only=True))

@viewapp.route('/commodities', name='commodities')
def commodities():
    "Render a list commodities with list their prices page."
    html_table = render_report(price_reports.CommoditiesReport, request.view.entries,
                               [],
                               css_id='price-index')
    return render_view(
        pagetitle="Commodities",
        contents=html_table)

@app.route('/context/<ehash:re:[a-fA-F0-9]*>', name='context')
def context_(ehash=None):
    "Render the before & after context around a transaction entry."

    matching_entries = [entry
                        for entry in app.entries
                        if ehash == compare.hash_entry(entry)]

    oss = io.StringIO()
    if len(matching_entries) == 0:
        print("ERROR: Could not find matching entry for '{}'".format(ehash),
              file=oss)

    elif len(matching_entries) > 1:
        print("ERROR: Ambiguous entries for '{}'".format(ehash),
              file=oss)
        print(file=oss)
        dcontext = app.options['dcontext']
        printer.print_entries(matching_entries, dcontext, file=oss)

    else:
        entry = matching_entries[0]

        # Render the context.
        oss.write("<pre>\n")
        oss.write(context.render_entry_context(app.entries, app.options, entry))
        oss.write("</pre>\n")

        # Render the filelinks.
        if FILELINK_PROTOCOL:
            meta = entry.meta
            uri = FILELINK_PROTOCOL.format(filename=meta.get('filename'),
                                           lineno=meta.get('lineno'))
            oss.write('<div class="filelink"><a href="{}">{}</a></div>'.format(uri, 'Open'))

    return render_global(
        pagetitle="Context: {}".format(ehash),
        contents=oss.getvalue())

@viewapp.route('/conversions', name='conversions')
def conversions_():
    "Render the list of transactions with conversions."
    return render_view(
        pagetitle="Conversions",
        contents=render_report(journal_reports.ConversionsReport,
                               request.view.entries, leaf_only=False))

@app.route('/doc/<filename:re:.*>', name=doc_name)
def doc(filename=None):
    "Serve static filenames for documents directives."

    filename = '/' + filename

    # Check that there is a document directive that has this filename.
    # This is for security; we don't want to be able to serve just any file.
    for entry in misc_utils.filter_type(app.entries, data.Document):
        if entry.filename == filename:
            break
    else:
        raise bottle.HTTPError(404, "Not found")

    # Just serve the file ourselves.
    return bottle.static_file(path.basename(filename),
                              path.dirname(filename))

@viewapp.route('/documents', name='documents')
def documents():
    "Render a tree with all the documents found."
    return render_view(
        pagetitle="Documents",
        contents=render_report(journal_reports.DocumentsReport,
                               request.view.entries, leaf_only=False))

@app.route('/errors', name='errors')
def errors():
    "Report error encountered during parsing, checking and realization."
    return render_global(
        pagetitle="Errors",
        contents=render_report(misc_reports.ErrorReport, [], leaf_only=False))

@viewapp.route(r'/event/<event:re:([A-Za-z0-9\-_/.]+)?>', name='event')
def event(event=None):
    "Render all values of a particular event."
    if not event:
        bottle.redirect(app.get_url('event_index'))
    return render_view(
        pagetitle="Event: {}".format(event),
        contents=render_report(misc_reports.EventsReport, app.entries,
                               ['--expr', event]))

@viewapp.route('/event', name='event_index')
def event_index():
    "Render the latest values of all events and an index."
    return render_view(
        pagetitle="Events Index",
        contents=render_report(misc_reports.CurrentEventsReport,
                               request.view.entries))

def get_all_view(app):
    """Return a view of all transactions.

    Returns:
      An instance of AllView, that covers all transactions.
    """
    return views.AllView(app.entries, app.options, 'All Transactions')

def handle_view(path_depth):
    """A decorator for handlers which create views lazily.
    If you decorate a method with this, the wrapper does the redirect
    handling and your method is just a factory for a View instance,
    which is cached.

    Args:
      path_depth: An integer, the number of components that form the view id.
    Returns:
      A decorator function, used to wrap view handlers.
    """
    def view_populator(callback):
        def wrapper(*args, **kwargs):
            components = request.path.split('/')
            viewid = '/'.join(components[:path_depth+1])
            try:
                # Try fetching the view from the cache.
                view = app.views[viewid]
            except KeyError:
                # We need to create the view.
                view = app.views[viewid] = callback(*args, **kwargs)

            # Save the view for the subrequest and redirect. populate_view()
            # picks this up and saves it in request.view.
            request.environ['VIEW'] = view
            return bottle_utils.internal_redirect(viewapp, path_depth)
        return wrapper
    return view_populator

@viewapp.route('/equity/holdings', name='holdings')
def holdings_():
    "Render a detailed table of all holdings."
    html_table = render_report(holdings_reports.HoldingsReport, request.view.entries,
                               [],
                               css_class='holdings detail-table sortable', center=True)
    return render_view(
        pagetitle="Holdings",
        contents=html_table,
        scripts='<script src="/third_party/sorttable.js"></script>')

@viewapp.route('/equity/holdings_byaccount', name='holdings_byaccount')
def holdings_byaccount():
    "Render a table of holdings by account."

    html_table = render_report(holdings_reports.HoldingsReport, request.view.entries,
                               ['--by', 'account'],
                               css_class='holdings detail-table sortable', center=True)
    return render_view(
        pagetitle="Holdings by Account",
        contents=html_table,
        scripts='<script src="/third_party/sorttable.js"></script>')

@viewapp.route('/equity/holdings_bycommodity', name='holdings_bycommodity')
def holdings_bycommodity():
    "Render a table of holdings by commodity."

    html_table = render_report(holdings_reports.HoldingsReport, request.view.entries,
                               ['--by', 'commodity'],
                               css_class='holdings detail-table sortable', center=True)
    return render_view(
        pagetitle="Holdings by Commodity",
        contents=html_table,
        scripts='<script src="/third_party/sorttable.js"></script>')

@viewapp.route('/equity/holdings_bycurrency', name='holdings_bycurrency')
def holdings_bycurrency():
    "Render a table of holdings by currency."

    html_table = render_report(holdings_reports.HoldingsReport, request.view.entries,
                               ['--by', 'currency'],
                               css_class='holdings detail-table sortable', center=True)
    return render_view(
        pagetitle="Holdings by Currency",
        contents=html_table,
        scripts='<script src="/third_party/sorttable.js"></script>')

@viewapp.route('/equity/holdings_byrootaccount', name='holdings_byrootaccount')
def holdings_byrootaccount():
    "Render a table of holdings by account."

    html_table = render_report(holdings_reports.HoldingsReport, request.view.entries,
                               ['--by', 'root-account'],
                               css_class='holdings detail-table sortable', center=True)
    return render_view(
        pagetitle="Holdings by Account",
        contents=html_table,
        scripts='<script src="/third_party/sorttable.js"></script>')

def incognito(callback):
    """A plugin that converts all numbers rendered into X's, in order
    to hide the actual values in the ledger. This is used for doing
    public demos using my real ledger, where I don't necessarily
    want to share the detail of my financial life with the viewers
    but when I still want an interesting ledger, with enough
    detail that looks realistic."""

    def wrapper(*posargs, **kwargs):
        contents = callback(*posargs, **kwargs)
        # pylint: disable=bad-continuation
        if (response.content_type in ('text/html', '') and
            isinstance(contents, str)):
            contents = text_utils.replace_numbers(contents)
        return contents

    return wrapper

@viewapp.route('/income', name='income')
def income():
    "Income statement."
    return render_view(pagetitle="Income Statement",
                       contents=render_real_report(balance_reports.IncomeStatementReport,
                                                   request.view.real_accounts,
                                                   app.price_map,
                                                   request.view.price_date,
                                                   leaf_only=True))

@viewapp.route('/index', name='index')
def index():
    "Index of all pages, so that navigation need not have all links."

    oss = io.StringIO()
    oss.write('<ul>\n')
    for title, page in [
            ("Balances", "trial"),
            ("Balance Sheet", "balsheet"),
            ("Opening Balances", "openbal"),
            ("Income Statement", "income"),
            ("General Journal", "journal_all"),
            ("Conversions", "conversions"),
            ("Documents", "documents"),
            ("Holdings (Full Detail)", "holdings"),
            ("Holdings by Account", "holdings_byaccount"),
            ("Holdings by Root Account", "holdings_byrootaccount"),
            ("Holdings by Commodity", "holdings_bycommodity"),
            ("Holdings by Currency", "holdings_bycurrency"),
            ("Net Worth", "networth"),
            ("Commodities", "commodities"),
            ("Events", "event_index"),
            ("Activity/Update", "activity"),
            ("Statistics (Types)", "stats_types"),
            ("Statistics (Postings)", "stats_postings"),
    ]:
        oss.write('<li><a href={}>{}</a></li>\n'.format(
            request.app.get_url(page), title))
    oss.write('</ul>\n')

    return render_view(
        pagetitle="Index",
        contents=oss.getvalue())

@viewapp.route('/journal/<account_name:re:.*>', name='journal')
def journal_(account_name=None):
    "A list of all the entries for this account realization."
    account_name = app.account_xform.parse(account_name)

    # Figure out which account to render this from.
    real_accounts = request.view.real_accounts
    if account_name:
        if account_name and account_types.is_balance_sheet_account(account_name,
                                                                   app.account_types):
            real_accounts = request.view.closing_real_accounts

    # Render the report.
    args = []
    if account_name:
        args.append('--account={}'.format(account_name))

    render_postings = request.params.get('postings', True)
    if isinstance(render_postings, str):
        render_postings = render_postings.lower() in ('1', 'true')
    if render_postings:
        args.append('--verbose')

    try:
        html_journal = render_real_report(journal_reports.JournalReport,
                                          real_accounts,
                                          app.price_map,
                                          request.view.price_date,
                                          args, leaf_only=False)
    except KeyError as e:
        raise bottle.HTTPError(404, '{}'.format(e))

    return render_view(pagetitle='{}'.format(account_name or
                                             'General Ledger (All Accounts)'),
                       contents=html_journal)

@viewapp.route('/journal/all', name='journal_all')
def journal_all():
    "A list of all the entries in this realization."
    bottle.redirect(request.app.get_url('journal', account_name=''))

@app.route('/link/<link:re:.*>', name='link')
def link(link=None):
    "Serve journals for links."

    linked_entries = basicops.filter_link(link, app.entries)

    oss = io.StringIO()
    formatter = HTMLFormatter(app.options['dcontext'],
                              request.app.get_url, False, app.account_xform,
                              view_links=False)
    journal_html.html_entries_table_with_balance(oss, linked_entries, formatter)
    return render_global(
        pagetitle="Link: {}".format(link),
        contents=oss.getvalue())

def main():
    """Main web service runner. This runs the event loop and blocks indefinitely."""

    argparser = version.ArgumentParser(description=__doc__.strip())
    add_web_arguments(argparser)
    args = argparser.parse_args()

    run_app(args)

def month_request(year, month):
    """Render a URL to a particular month of the context's request.
    """
    if year < app.active_years[0] or year > app.active_years[-1]:
        return ''
    month = list(calendar.month_abbr).index(month)
    month = "{:0>2d}".format(month)
    return app.router.build('month', year=year, month=month,
            path=request.path[1:])

@viewapp.route('/equity/networth', name='networth')
def networth():
    "Render a table of the net worth for this filter."

    html_table = render_report(holdings_reports.NetWorthReport, request.view.entries)
    return render_view(
        pagetitle="Net Worth",
        contents=html_table)

@viewapp.route('/openbal', name='openbal')
def openbal():
    "Opening balances."
    return render_view(pagetitle="Opening Balances",
                       contents=render_real_report(balance_reports.BalanceSheetReport,
                                                   request.view.opening_real_accounts,
                                                   app.price_map,
                                                   request.view.price_date,
                                                   leaf_only=True))

def populate_view(callback):
    """A plugin that will populate the request with the current view instance.

    Args:
      callback: A continuation function to call to handle the request.
    Returns:
      A function to call to install view-specific parameters on the request.
    """
    def wrapper(*args, **kwargs):
        request.view = request.environ['VIEW']
        return callback(*args, **kwargs)
    return wrapper

@viewapp.route('/prices'
               r'/<base:re:[A-Z][A-Z0-9\'\.\_\-]{0,22}[A-Z0-9]>'
               r'/<quote:re:[A-Z][A-Z0-9\'\.\_\-]{0,22}[A-Z0-9]>', name='prices')
def prices_values(base=None, quote=None):
    "Render all the values for a particular price pair."
    html_table = render_report(price_reports.CommodityPricesReport, request.view.entries,
                               ['--commodity', '{}/{}'.format(base, quote)],
                               css_id='price-index')
    return render_view(
        pagetitle="Price: {} / {}".format(base, quote),
        contents=html_table)

def render_global(*args, **kw):
    """Render the title and contents in our standard template for a global page.

    Args:
      *args: A tuple of values for the HTML template.
      *kw: A dict of optional values for the HTML template.
    Returns:
      An HTML string of the rendered template.
    """
    response.content_type = 'text/html'
    kw['A'] = A # Application mapper
    kw['V'] = V # View mapper
    kw['title'] = app.options['title']
    kw['view_title'] = ''
    kw['navigation'] = GLOBAL_NAVIGATION
    kw['scripts'] = kw.get('scripts', '')

    kw['overlay'] = (
        render_overlay('<li><a href="{}">Errors</a></li>'.format(
            app.router.build('errors')))
        if request.params.pop('render_overlay', True)
        else '')
    return template.render(*args, **kw)

def render_overlay(contents):
    """Render an overlay of the navigation with the current errors.

    This is used to bring up new errors on any page when they occur.

    Returns:
      A string of HTML for the contents of the errors overlay.
    """
    return '''
      <div class="navigation" id="nav-right">
        <ul>
          {}
        </ul>
      </div>'''.format(contents)

    # It would be nice to have a fancy overlay here, that automatically appears
    # after parsing if there are errors and that automatically smoothly fades
    # out.

def render_real_report(report_class, real_root, price_map, price_date,
                       args=None, leaf_only=False):
    # pylint: disable=too-many-arguments
    """Instantiate a report and rendering it to a string.

    This is intended to be called in the context of a Bottle view app request
    (it uses 'request').

    Args:
      report_class: A class, the type of the report to render.
      real_root: An instance of RealAccount to render.
      price_map: A price map as built by build_price_map().
      price_date: The date at which to evaluate the prices.
      args: A list of strings, the arguments to initialize the report with.
      leaf_only: A boolean, whether to render the leaf names only.
    Returns:
      A string, the rendered report.
    """
    formatter = HTMLFormatter(app.options['dcontext'],
                              request.app.get_url, leaf_only, app.account_xform)
    oss = io.StringIO()
    report_ = report_class.from_args(args, formatter=formatter)
    report_.render_real_htmldiv(real_root, price_map, price_date, app.options, oss)
    return oss.getvalue()

def render_report(report_class, entries, args=None,
                  css_id=None, css_class=None, center=False, leaf_only=True):
    """Instantiate a report and rendering it to a string.

    Args:
      report_class: A class, the type of the report to render.
      real_root: An instance of RealAccount to render.
      args: A list of strings, the arguments to initialize the report with.
      css_id: An optional string, the CSS id for the div to render.
      css_class: An optional string, the CSS class for the div to render.
      center: A boolean flag, if true, wrap the results in a <center> tag.
      leaf_only: A boolean, whether to render the leaf names only.
    Returns:
      A string, the rendered report.
    """
    formatter = HTMLFormatter(app.options['dcontext'],
                              request.app.get_url, leaf_only, app.account_xform)
    oss = io.StringIO()
    if center:
        oss.write('<center>\n')
    report_ = report_class.from_args(args,
                                     formatter=formatter,
                                     css_id=css_id,
                                     css_class=css_class)
    report_.render_htmldiv(entries, app.errors, app.options, oss)
    if center:
        oss.write('</center>\n')
    return oss.getvalue()

def render_view(*args, **kw):
    """Render the title and contents in our standard template for a view page.

    Args:
      *args: A tuple of values for the HTML template.
      *kw: A dict of optional values for the HTML template.
    Returns:
      An HTML string of the rendered template.
    """
    response.content_type = 'text/html'
    kw['A'] = A # Application mapper
    kw['V'] = V # View mapper
    kw['title'] = app.options['title']
    kw['view_title'] = ' - ' + request.view.title

    overlays = []
    if request.params.pop('render_overlay', False):
        overlays.append(
            '<li><a href="{}">Errors</a></li>'.format(app.router.build('errors')))

    # Render navigation, with monthly navigation option.
    oss = io.StringIO()
    oss.write(APP_NAVIGATION.render(A=A, V=V, view_title=request.view.title))
    if request.view.monthly is views.MonthNavigation.COMPACT:
        overlays.append(
            '<li><a href="{}">Monthly</a></li>'.format(M.Jan))
    elif request.view.monthly is views.MonthNavigation.FULL:
        annual = app.router.build('year',
                                  path=DEFAULT_VIEW_REDIRECT,
                                  year=request.view.year)
        oss.write(APP_NAVIGATION_MONTHLY_FULL.render(M=M, Mp=Mp, Mn=Mn, V=V, annual=annual))
    kw['navigation'] = oss.getvalue()
    kw['overlay'] = render_overlay(' '.join(overlays))

    kw['scripts'] = kw.get('scripts', '')

    return template.render(*args, **kw)

@app.route('/', name='root')
def root():
    "Redirect the root page to the home page."
    bottle.redirect(app.get_url('toc'))

def scrape_webapp(webargs, callback, ignore_regexp):
    """Run a web server on a Beancount file and scrape it.

    This is the main entry point of this module.

    Args:
      webargs: An argparse.Namespace container of the arguments provided in
        web.add_web_arguments().
      callback: A callback function to invoke on each page to validate it.
        The function is called with the response and the url as arguments.
        This function should trigger an error on failure (via an exception).
      ignore_regexp: A regular expression string, the urls to ignore.
    Returns:
      A set of all the processed URLs and a set of all the skipped URLs.
    """
    url_format = 'http://localhost:{}{{}}'.format(webargs.port)

    thread = thread_server_start(webargs)

    # Skips:
    # - Docs cannot be read for external files.
    #
    # - Components views... well there are just too many, makes the tests
    #   impossibly slow. Just keep the A's so some are covered.
    url_lists = scrape.scrape_urls(url_format, callback, ignore_regexp)

    thread_server_shutdown(thread)

    return url_lists

def setup_monkey_patch_for_server_shutdown():
    """Setup globals to steal access to the server reference.
    This is required to initiate shutdown, unfortunately.
    (Bottle could easily remedy that.)"""

    # Save the original function.
    # pylint: disable=import-outside-toplevel
    from wsgiref.simple_server import make_server

    # Create a decorator that will save the server upon start.
    def stealing_make_server(*args, **kw):
        global server
        server = make_server(*args, **kw)
        return server

    # Patch up wsgiref itself with the decorated function.
    import wsgiref.simple_server
    wsgiref.simple_server.make_server = stealing_make_server

def shutdown():
    """Request for the server to shutdown."""
    server.shutdown()

@app.route('/source', name='source')
def source():
    "Render the source file, allowing scrolling at a specific line."

    contents = io.StringIO()
    if app.args.no_source:
        contents.write("Source hidden.")
    else:
        contents.write('<div id="source">')
        for i, line in enumerate(app.source.splitlines()):
            lineno = i+1
            contents.write(
                '<pre id="{lineno}">{lineno}  {line}</pre>\n'.format(
                    lineno=lineno, line=line.rstrip()))
        contents.write('</div>')

    return render_global(
        pagetitle="Source",
        contents=contents.getvalue()
        )

@viewapp.route('/stats_postings', name='stats_postings')
def stats_postings():
    "Compute and render statistics about the input file."
    return render_view(
        pagetitle="Postings Statistics",
        contents=render_report(misc_reports.StatsPostingsReport,
                               request.view.entries))

@viewapp.route('/stats_types', name='stats_types')
def stats_types():
    "Compute and render statistics about the input file."
    return render_view(
        pagetitle="Directives Statistics",
        contents=render_report(misc_reports.StatsDirectivesReport,
                               request.view.entries))

@app.route('/resources/web.css', name='style')
def style():
    "Stylesheet for the entire document."
    response.content_type = 'text/css'
    if app.args.debug:
        with open(path.join(path.dirname(__file__), 'web.css')) as f:
            global STYLE; STYLE = f.read()
    return STYLE

def thread_server_shutdown(thread):
    """Shutdown the server running in the given thread.

    Unfortunately, in the meantime this has a side-effect on all servers.
    This returns after waiting that the thread has stopped.

    Args:
      thread: A threading.Thread instance.
    """
    # Clean shutdown: request to stop, then join the thread.
    # Note that because we daemonize, we could forego this elegant detail.
    shutdown()
    thread.join()

def thread_server_start(web_args, **kwargs):
    """Start a server in a new thread.

    Args:
      argparse_args: An argparse parsed options object, with all the options
        from add_web_arguments().
    Returns:
      A new Thread instance.

    """
    thread = threading.Thread(
        target=run_app,
        args=(web_args,),
        kwargs=kwargs)
    thread.daemon = True # Automatically exit if the process comes dwn.
    thread.start()

    # Ensure the server has at least started before running the scraper.
    wait_ready()
    time.sleep(0.1)

    return thread

@viewapp.route('/trial', name='trial')
def trial():
    "Trial balance / Chart of Accounts."
    return render_view(
        pagetitle="Trial Balance",
        contents=render_real_report(balance_reports.BalancesReport,
                                    request.view.real_accounts,
                                    app.price_map,
                                    request.view.price_date,
                                    leaf_only=True))

def url_restrict_generator(url_prefix):
    """Restrict to only a single prefix.

    Args:
      url_prefix: A string, a URL prefix to restrict to.
      callback: The function to wrap.
    Returns:
      A handler decorator.
    """
    # A list of URLs that should always be accepted, even when restricted.
    allowed_regexps = [re.compile(regexp).match
                       for regexp in ['/resources',
                                      '/favicon.ico',
                                      '/index',
                                      '/errors',
                                      '/source',
                                      '/context',
                                      '/third_party']]

    def url_restrict_handler(callback):
        def wrapper(*args, **kwargs):
            if (any(match(request.path) for match in allowed_regexps) or
                request.path.startswith(url_prefix)):
                return callback(*args, **kwargs)
            if request.path == '/':
                bottle.redirect(url_prefix)

            # Note: we issue a "202 Accepted" status in order to satisfy bean-bake,
            # we want to distinguish between an actual 404 error and this case.
            raise bottle.HTTPError(202, "URLs restricted to '{}'".format(url_prefix))

        return wrapper
    return url_restrict_handler

def wait_ready():
    """Wait until the 'server' global has been set.
    This tells us the server is running.
    """
    while server is None:
        time.sleep(0.05)