# --------------------------------------------------
# Copyright The IETF Trust 2011, All Rights Reserved
# --------------------------------------------------

import calendar
import codecs
import datetime
import textwrap
import lxml
import os
import re
import xml2rfc.log
import xml2rfc.util
import xml2rfc.utils
from optparse import Values

try:
    from xml2rfc import debug
    debug.debug = True
except ImportError:
    pass

from xml2rfc import strings, log
from xml2rfc.uniscripts import is_script
from xml2rfc.util.date import extract_date, augment_date, format_date, get_expiry_date
from xml2rfc.util.name import short_author_ascii_name_parts, full_author_name_expansion, short_author_name_parts
from xml2rfc.util.unicode import ( punctuation, unicode_replacements, unicode_content_tags, bare_unicode_tags,
    bare_latin_tags, unicode_attributes, downcode, downcode_punctuation)
from xml2rfc.utils import namespaces, find_duplicate_ids

default_silenced_messages = [
#    ".*[Pp]ostal address",
]

default_options = Values(defaults={
        'accept_prepped': None,
        'add_xinclude': None,
        'basename': None,
        'bom': False,
        'cache': None,
        'clear_cache': False,
        'css': None,
        'date': datetime.date.today(),
        'datestring': None,
        'debug': False,
        'doi_base_url': 'https://doi.org/',
        'dtd': None,
        'expand': False,
        'external_css': False,
        'external_js': True,            # Change to False in 3.x
        'filename': None,
        'first_page_author_org': True,
        'html': False,
        'id_base_url': 'https://tools.ietf.org/html/',
        'id_reference_base_url': 'https://tools.ietf.org/html/',
        'id_is_work_in_progress': True,
        'image_svg': False,
        'indent': 2,
        'info': False,
        'legacy': True,
        'legacy_date_format': False,    # Change to True in 3.x
        'legacy_list_symbols': False,
        'list_symbols': ('*', '-', 'o', '+'),
        'metadata_js_url': 'metadata.min.js',
        'no_css': False,
        'no_dtd': None,
        'no_network': False,
        'nroff': False,
        'omit_headers': None,
        'output_filename': None,
        'output_path': None,
        'pagination': True,
        'pdf': False,
        'preptool': False,
        'quiet': False,
        'remove_pis': False,
        'raw': False,
        'rfc': None,
        'rfc_base_url': 'https://www.rfc-editor.org/rfc/',
        'rfc_reference_base_url': 'https://rfc-editor.org/rfc/',
        'silence': default_silenced_messages,
        'strict': False,
        'text': True,
        'utf8': False,
        'verbose': False,
        'version': False,
        'v2v3': False,
        'vocabulary': 'v2',
    })

class _RfcItem:
    """ A unique ID object for an anchored RFC element.
    
        Anchored elements are the following: Automatic sections, user (middle)
        sections, paragraphs, references, appendices, figures, and tables.
        
        RfcItems are collected into an index list in the base writer
    """
    def __init__(self, autoName, autoAnchor, counter='', title='', anchor='',
                 toc=True, level=1, appendix=False):
        self.counter = str(counter)
        self.autoName = autoName
        self.autoAnchor = autoAnchor
        self.title = title
        self.anchor = anchor
        self.toc = toc
        self.level = level
        self.appendix = appendix
        self.page = 0    # This will be set after buffers are complete!
        self.used = False


class _IrefItem:
    """ A unique ID object for an iref element """
    def __init__(self, anchor=None):
        self.pages = []
        self.subitems = {}
        self.anchor = anchor


class RfcWriterError(Exception):
    """ Exception class for errors during document writing """
    def __init__(self, msg):
        self.msg = msg


class BaseRfcWriter:
    """ Base class for all writers

        All public methods need to be overridden for a writer implementation.
    """

    # -------------------------------------------------------------------------
    # Attribute default values
    #
    # These will mainly come into play if DTD validation was disabled, and
    # processing that happens to rely on DTD populated attributes require a lookup
    defaults = {
        'section_toc':              'default',
        'xref_pageno':              'false',
        'xref_format':              'default',
        'iref_primary':             'false',
        'spanx_style':              'emph',
        'figure_suppress-title':    'false',
        'figure_title':             '',
        'figure_align':             'left',
        'vspace_blanklines':        0,
        'table_suppress-title':     'false',
        'table_title':              '',
        'table_align':              'center',
        'table_style':              'full',
        'ttcol_align':              'left',
        'ipr':                      'trust200902',
        'submissionType':           'IETF',
        'consensus':                'no',
    }

    # -------------------------------------------------------------------------
    # Boilerplate text
    boilerplate = {}

    # The RFC Editor's switchover date for https URLs
    boilerplate_https_date = datetime.date(year=2017, month=8, day=21)

    # Document stream names
    boilerplate['document_stream'] = {}
    boilerplate['document_stream']['IETF'] = \
        'Internet Engineering Task Force (IETF)'
    boilerplate['document_stream']['IAB'] = \
        'Internet Architecture Board (IAB)'
    boilerplate['document_stream']['IRTF'] = \
        'Internet Research Task Force (IRTF)'
    boilerplate['document_stream']['independent'] = \
        'Independent Submission'

    # Draft workgroup name
    boilerplate['draft_workgroup'] = 'Network Working Group'

    # Category names
    boilerplate['std'] = 'Standards Track'
    boilerplate['bcp'] = 'Best Current Practice'
    boilerplate['exp'] = 'Experimental'
    boilerplate['info'] = 'Informational'
    boilerplate['historic'] = 'Historic'

    # Series type
    boilerplate['series_name'] = {}
    boilerplate['series_name']['std'] = 'STD'
    boilerplate['series_name']['bcp'] = 'BCP'
    boilerplate['series_name']['info'] = 'FYI'

    # ISSN
    boilerplate['issn'] = '2070-1721'

    # 'Status of this Memo' boilerplate for RFCs
    boilerplate['status'] = {
        'std': {},
        'bcp': {},
        'exp': {},
        'info': {},
        'historic': {},
    }
    # 'Status of this Memo' boilerplate for RFCs
    boilerplate['status_5741'] = {
    }

    # Paragraph 1
    boilerplate['status']['std']['p1'] = \
        'This is an Internet Standards Track document.'
    boilerplate['status']['bcp']['p1'] = \
        'This memo documents an Internet Best Current Practice.'
    boilerplate['status']['exp']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for examination, experimental implementation, and ' \
        'evaluation.'
    boilerplate['status']['info']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for informational purposes.'
    boilerplate['status']['historic']['p1'] = \
        'This document is not an Internet Standards Track specification; ' \
        'it is published for the historical record.'

    # Paragraph 2 header
    boilerplate['status']['exp']['p2'] = \
        'This document defines an Experimental Protocol for the Internet ' \
        'community.'
    boilerplate['status']['historic']['p2'] = \
        'This document defines a Historic Document for the Internet ' \
        'community.'

    # Paragraph 2 body
    boilerplate['status']['IETF'] = \
        'This document is a product of the Internet Engineering Task Force ' \
        '(IETF).  It has been approved for publication by the Internet ' \
        'Engineering Steering Group (IESG).'
    boilerplate['status']['IETF_consensus'] = \
        'This document is a product of the Internet Engineering Task Force ' \
        '(IETF).  It represents the consensus of the IETF community.  It has ' \
        'received public review and has been approved for publication by ' \
        'the Internet Engineering Steering Group (IESG).'
    boilerplate['status']['IRTF'] = \
        'This document is a product of the Internet Research Task Force ' \
        '(IRTF).  The IRTF publishes the results of Internet-related ' \
        'research and development activities.  These results might not be ' \
        'suitable for deployment.'
    boilerplate['status']['IRTF_workgroup'] = \
        'This RFC represents the individual opinion(s) ' \
        'of one or more members of the %s Research Group of the Internet ' \
        'Research Task Force (IRTF).'
    boilerplate['status']['IRTF_workgroup_consensus'] = \
        'This RFC represents the consensus of the ' \
        '%s Research Group of the Internet Research Task Force (IRTF).'
    boilerplate['status']['IAB'] = \
        'This document is a product of the Internet Architecture Board ' \
        '(IAB) and represents information that the IAB has deemed valuable ' \
        'to provide for permanent record.'
    boilerplate['status']['IAB_consensus'] = (
        'This document is a product of the Internet Architecture Board ' 
        '(IAB) and represents information that the IAB has deemed valuable '
        'to provide for permanent record.  It represents the consensus of '
        'the Internet Architecture Board (IAB).')
    boilerplate['status']['independent'] = \
        'This is a contribution to the RFC Series, independently of any ' \
        'other RFC stream.  The RFC Editor has chosen to publish this ' \
        'document at its discretion and makes no statement about its value ' \
        'for implementation or deployment.'

    # Paragraph 2 last sentence
    boilerplate['status']['p2end_ietf_std'] = \
        'Further information on Internet Standards is available ' \
        'in Section 2 of RFC 7841.'
    boilerplate['status']['p2end_ietf_bcp'] = \
        'Further information on BCPs is available in Section 2 of RFC 7841.'
    boilerplate['status']['p2end_ietf_other'] = \
        'Not all documents approved by the IESG are candidates for any ' \
        'level of Internet Standard; see Section 2 of RFC 7841.'
    boilerplate['status']['p2end_other'] = \
        'Documents approved for publication by the %s are not ' \
        'candidates for any level of Internet Standard; see Section 2 of RFC ' \
        '7841.'

    boilerplate['status_5741']['p2end_ietf_std'] = \
        'Further information on Internet Standards is available ' \
        'in Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_ietf_bcp'] = \
        'Further information on BCPs is available in Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_ietf_other'] = \
        'Not all documents approved by the IESG are candidates for any ' \
        'level of Internet Standard; see Section 2 of RFC 5741.'
    boilerplate['status_5741']['p2end_other'] = \
        'Documents approved for publication by the %s are not ' \
        'candidates for any level of Internet Standard; see Section 2 of RFC ' \
        '5741.'

    # Paragraph 3
    boilerplate['status']['p3'] = (
        'Information about the current status of this document, any errata, '
        'and how to provide feedback on it may be obtained at '
        'http://www.rfc-editor.org/info/rfc%s.')
    boilerplate['status']['p3_s'] = (
        'Information about the current status of this document, any errata, '
        'and how to provide feedback on it may be obtained at '
        'https://www.rfc-editor.org/info/rfc%s.')

    # 'Status of this Memo' boilerplate for drafts
    boilerplate['status']['draft'] = [
       'Internet-Drafts are working documents of the Internet Engineering '
       'Task Force (IETF).  Note that other groups may also distribute '
       'working documents as Internet-Drafts.  The list of current Internet-'
       'Drafts is at http://datatracker.ietf.org/drafts/current/.',
       #
       'Internet-Drafts are draft documents valid for a maximum of six months '
       'and may be updated, replaced, or obsoleted by other documents at any '
       'time.  It is inappropriate to use Internet-Drafts as reference '
       'material or to cite them other than as "work in progress."']
    boilerplate['status']['draft_s'] = [
       'Internet-Drafts are working documents of the Internet Engineering '
       'Task Force (IETF).  Note that other groups may also distribute '
       'working documents as Internet-Drafts.  The list of current Internet-'
       'Drafts is at https://datatracker.ietf.org/drafts/current/.',
       #
       'Internet-Drafts are draft documents valid for a maximum of six months '
       'and may be updated, replaced, or obsoleted by other documents at any '
       'time.  It is inappropriate to use Internet-Drafts as reference '
       'material or to cite them other than as "work in progress."']

    boilerplate['draft_expire'] = \
       'This Internet-Draft will expire on %s.'

    # IPR status boilerplate
    boilerplate['ipr_200902_status'] = \
        'This Internet-Draft is submitted in full conformance ' \
        'with the provisions of BCP 78 and BCP 79.'
    boilerplate['ipr_200811_status'] = \
        'This Internet-Draft is submitted to IETF in full conformance ' \
        'with the provisions of BCP 78 and BCP 79.'
    
    # Copyright boilerplate
    boilerplate['base_copyright_header'] = \
        'Copyright (c) %s IETF Trust and the persons identified as the ' \
        'document authors.  All rights reserved.'
    boilerplate['base_copyright_body'] = (
        'This document is subject to BCP 78 and the IETF Trust\'s Legal '
        'Provisions Relating to IETF Documents '
        '(http://trustee.ietf.org/license-info) in effect on the date of '
        'publication of this document.  Please review these documents '
        'carefully, as they describe your rights and restrictions with respect '
        'to this document.')
    boilerplate['base_copyright_body_s'] = (
        'This document is subject to BCP 78 and the IETF Trust\'s Legal '
        'Provisions Relating to IETF Documents '
        '(https://trustee.ietf.org/license-info) in effect on the date of '
        'publication of this document.  Please review these documents '
        'carefully, as they describe your rights and restrictions with respect '
        'to this document.')

    # IPR values which append things to copyright
    boilerplate['ipr_200902_copyright_ietfbody'] = \
        'Code Components extracted from this document must ' \
        'include Simplified BSD License text as described in Section 4.e of ' \
        'the Trust Legal Provisions and are provided without warranty as ' \
        'described in the Simplified BSD License.'
    boilerplate['ipr_noModification_copyright'] = \
        'This document may not be modified, and derivative works of it may ' \
        'not be created, except to format it for publication as an RFC or ' \
        'to translate it into languages other than English.'
    boilerplate['ipr_noDerivatives_copyright'] = \
        'This document may not be modified, and derivative works of it may ' \
        'not be created, and it may not be published except as an ' \
        'Internet-Draft.'
    boilerplate['ipr_pre5378Trust200902_copyright'] = \
        'This document may contain material from IETF Documents or IETF ' \
        'Contributions published or made publicly available before ' \
        'November 10, 2008.  The person(s) controlling the copyright in some ' \
        'of this material may not have granted the IETF Trust the right to ' \
        'allow modifications of such material outside the IETF Standards ' \
        'Process. Without obtaining an adequate license from the person(s) ' \
        'controlling the copyright in such materials, this document may not ' \
        'be modified outside the IETF Standards Process, and derivative ' \
        'works of it may not be created outside the IETF Standards Process, ' \
        'except to format it for publication as an RFC or to translate it ' \
        'into languages other than English.'

    # Any extra boilerplate
    # Disabled. See issue #123, http://trac.tools.ietf.org/tools/xml2rfc/trac/ticket/123
    ## boilerplate['iprnotified'] = \
    ##     'The IETF has been notified of intellectual property rights ' \
    ##     'claimed in regard to some or all of the specification contained ' \
    ##     'in this document.  For more information consult the online list ' \
    ##     'of claimed rights.'
    
    # Stream approvers
    approvers = {
        'IAB': 'IAB',
        'IRTF': 'IRSG',
        'independent': 'RFC Editor',
    }

    # Valid IPR attributes
    supported_ipr = [
        'trust200902',
        'noModificationTrust200902',
        'noDerivativesTrust200902',
        'pre5378Trust200902',
        'trust200811',
        'noModificationTrust200811',
        'noDerivativesTrust200811',
        'none',
    ]

    # -------------------------------------------------------------------------

    def __init__(self, xmlrfc, quiet=None, options=default_options, date=datetime.date.today()):
        if not quiet is None:
            options.quiet = quiet
        self.options = options
        self.date = date
        self.expire_string = ''
        self.ascii = False
        self.nbws_cond = u'\u00A0'
        self.eref_list = []
        self.xmlrfc = xmlrfc
        self.pis = self.xmlrfc.getpis()

        # We will refer to the XmlRfc document root as 'r'
        self.xmlrfc = xmlrfc
        self.r = xmlrfc.getroot()

        # Document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0

        # Set RFC number and draft flag
        self.rfcnumber = self.r.attrib.get('number', '')
        self.draft = bool(not self.rfcnumber)
        
        # Used for two-pass indexing
        self.indexmode = False

        # Item Indicies
        self._index = []
        self._iref_index = {}

    def _make_iref(self, item, subitem=None, anchor=None):
        """ Create an iref ID object if it doesnt exist yet """
        last = None
        if item not in self._iref_index:
            self._iref_index[item] = _IrefItem()
            last = self._iref_index[item]
        if subitem and subitem not in self._iref_index[item].subitems:
            self._iref_index[item].subitems[subitem] = _IrefItem()
            last = self._iref_index[item].subitems[subitem]
        if last and anchor:
            last.anchor = anchor

    def _add_iref_to_index(self, element):
        item = element.attrib.get('item', None)
        if item:
            subitem = element.attrib.get('subitem', None)
            self._make_iref(item, subitem)
            # Store the buffer position for pagination data later
            pos = len(self.buf)
            if not self.indexmode:
                if pos not in self.iref_marks:
                    self.iref_marks[pos] = []
                self.iref_marks[pos].append((item, subitem))

    def _indexParagraph(self, counter, p_counter, anchor=None, toc=False):
        counter = str(counter)  # This is the section counter
        p_counter = str(p_counter)  # This is the paragraph counter
        autoName = 'Section ' + counter + ', Paragraph ' + p_counter
        autoAnchor = 'rfc.section.' + counter + '.p.' + p_counter
        item = _RfcItem(autoName, autoAnchor, anchor=anchor, toc=toc, counter=p_counter)
        self._index.append(item)
        return item

    def _indexListParagraph(self, p_counter, anchor, toc=False):
        p_counter = str(p_counter)
        autoName = 'Paragraph ' + p_counter
        item = _RfcItem(autoName, '', counter=p_counter, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexSection(self, counter, title=None, anchor=None, toc=True, \
                      level=1, appendix=False, numbered=True):
        counter = str(counter)
        if numbered:
            if appendix:
                autoName = 'Appendix' + self.nbws_cond + counter
                autoAnchor = 'rfc.appendix.' + counter
            else:
                autoName = 'Section' + self.nbws_cond + counter
                autoAnchor = 'rfc.section.' + counter
            item = _RfcItem(autoName, autoAnchor, counter=counter, title=title,
                           anchor=anchor, toc=toc, level=level, appendix=appendix)
        else:
            if not title:
                raise RfcWriterError("No title available when trying to insert index item for anchor %s" % anchor)
            autoAnchor = 'rfc.' + re.sub('[^A-Za-z0-9]+', '_', title).lower()
            item = _RfcItem(title, autoAnchor, title=title)
        self._index.append(item)
        return item

    def _indexReferences(self, counter, title=None, anchor=None, toc=True, \
                         subCounter=0, level=1):
        if subCounter < 1:
            autoName = 'References'
            autoAnchor = 'rfc.references'
        else:
            subCounter = str(subCounter)
            autoName = 'References' + self.nbws_cond + subCounter
            autoAnchor = 'rfc.references.' + subCounter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, \
                       anchor=anchor, toc=toc, level=level)
        self._index.append(item)
        return item
    
    def _indexFigure(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        autoName = 'Figure' + self.nbws_cond + counter
        autoAnchor = 'rfc.figure.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, \
                       toc=toc)
        self._index.append(item)
        return item
        
    def _indexTable(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        autoName = 'Table' + self.nbws_cond + counter
        autoAnchor = 'rfc.table.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexRef(self, counter, title=None, anchor=None, toc=False):
        counter = str(counter)
        if self.pis['symrefs'] == 'yes':
            autoName = '[' + (anchor or counter ) +']'
        else:
            autoName = '['+counter+']'
        autoAnchor = 'rfc.ref.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, title=title, anchor=anchor, toc=toc)
        self._index.append(item)
        return item

    def _indexCref(self, counter, anchor):
        counter = str(counter)
        autoName = 'Comment' + self.nbws_cond + anchor
        autoAnchor = 'rfc.comment.' + counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, anchor=anchor, toc=False)
        self._index.append(item)
        return item

    def _indexAuthor(self, counter, anchor, name):
        autoName = name
        autoAnchor = 'rfc.author.%s' % counter
        item = _RfcItem(autoName, autoAnchor, counter=counter, anchor=anchor, toc=False)
        self._index.append(item)
        return item

    def get_initials(self, author):
        """author is an rfc2629 author element.  Return the author initials,
        fixed up according to current flavour and policy."""
        initials = author.attrib.get('initials', '')
        multiple = author.pis["multiple-initials"] == "yes"
        initials_list = re.split("[. ]+", initials)
        try:
            initials_list.remove('')
        except:
            pass
        if len(initials_list) > 0:
            if multiple:
                # preserve spacing, but make sure all parts have a trailing
                # period
                initials = initials.strip()
                initials += '.' if not initials.endswith('.') else ''
                initials = re.sub('([^.]) ', '\g<1>. ', initials)
            else:
                initials = initials_list[0] + "."
        return initials

    def parse_pi(self, pi):
        return xml2rfc.utils.parse_pi(pi, self.pis)

    def get_numeric_pi(self, key, default):
        num = self.pis.get(key, None)
        if num is None:
            return default
        if not num.isdigit():
            xml2rfc.log.warn('Expected a numeric value for the %s PI, found "%s"' % (key, num))
            return default
        return int(num)

    def _getTocIndex(self):
        return [item for item in self._index if item.toc]
        
    def _getItemByAnchor(self, anchor):
        for item in self._index:
            if item.autoAnchor == anchor or item.anchor == anchor:
                return item
        return None
    
    def _validate_ipr(self):
        """ Ensure the application has boilerplate for the ipr attribute given """
        ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
        if not ipr in self.supported_ipr:
            raise RfcWriterError('No boilerplate text available for '
            'ipr: \'%s\'.  Acceptable values are: ' % ipr + \
            ', '.join(self.supported_ipr))
        
    def is_numbered(self, node):
        attr = node.attrib.get('numbered', 'true')
        return attr in ['yes', 'true', ]
    
    def _format_date(self):
        """ Fix the date data """
        today = self.date
        date = self.r.find('front/date')
        assert date is not None, "Bug in schema validation: no date element in document"
        year = date.attrib.get('year')
        if not year:
            year = str(self.date.year)
            date.set('year', year)
        if not year.isdigit():
            xml2rfc.log.error("Expected a numeric year, found '%s'" % (year, ))
        year = int(year)
        #
        month = date.attrib.get('month')
        if not month:
            if year != today.year:
                xml2rfc.log.error("Cannot handle a <date> with year different than this year, and no month.  Using today's date.")
                year = today.year
            month = today.month
            date.set('month', str(month))
        else:
            if not month.isdigit():
                month = xml2rfc.util.date.normalize_month(month)
            month = int(month)
        #
        day = date.attrib.get('day')
        if day is None:
            temp_date = datetime.date(year=year, month=month, day=1)
            if today.year == year and today.month == month:
                day = today.day
            elif abs(today - temp_date) < datetime.timedelta(days=34):
                if datetime.date(year=year, month=month, day=1) < today:
                    # wrong month, and the first day of that month is earlier
                    # than today.  Use the last day of the month
                    day = calendar.monthrange(year, month)[1]
                else:
                    # wrong month, later than this month.  Use the first day.
                    day = 1
            else:
                day = 1
        else:
            day = int(day)

        self.date = datetime.date(year=year, month=month, day=day)

        # Setup the expiration string for drafts as published date + six months
        if self.draft:
            if not date.get('day'):
                date.set('day', str(day))
            if not date.get('month'):
                date.set('month', str(month))
            expire_date = self.date + datetime.timedelta(185)
            self.expire_string = expire_date.strftime('%B %d, %Y').replace(' 0', ' ')

    def _format_counter(self, text, count, list_length=1):
        """ Return a proper string for a formatted list bullet.  Allowed types:
                %c: Lowercase chars
                %C: Uppercase chars
                %d: Digits
                %i: Lowercase roman numerals
                %I: Uppercase roman numerals
                %o: octal
                %x: Lowercase hex
                %X: Uppercase hex
        """
        import math
        roman_widths = {        1:1,  2:2,  3:3,  4:2,  5:1,  6:2,  7:3,  8:4,  9:2,
                        10:1, 11:2, 12:3, 13:4, 14:3, 15:2, 16:3, 17:4, 18:5, 19:3,
                        20:2, 21:3, 22:4, 23:5, 24:4, 25:3, 26:4, 27:5, 28:6, 29:4, }
        #
        decimal_width = int(math.log(list_length, 10))
        roman_width = roman_widths.get(list_length, 6)
        letter_width = int(math.log(list_length, 26))
        hex_width = int(math.log(list_length, 16))
        octal_width = int(math.log(list_length, 8))
        extra_width = len(text)+1
        if '%d' in text:
            text = text.replace(r'%d', str(count)).ljust(decimal_width+extra_width)
        elif '%c' in text:
            text = text.replace(r'%c', xml2rfc.util.num.int2letter(count)).ljust(letter_width+extra_width)
        elif '%C' in text:
            text = text.replace(r'%C', xml2rfc.util.num.int2letter(count).upper()).ljust(letter_width+extra_width)
        elif '%i' in text:
            text = text.replace(r'%i', xml2rfc.util.num.int2roman(count)).ljust(roman_width+extra_width)
        elif '%I' in text:
            text = text.replace(r'%I', xml2rfc.util.num.int2roman(count).upper()).ljust(roman_width+extra_width)
        elif '%o' in text:
            text = text.replace(r'%o', oct(count).replace("0","",1)).replace("o","",1).ljust(octal_width+extra_width)
        elif '%x' in text:
            text = text.replace(r'%x', hex(count).replace("0x","",1)).ljust(hex_width+extra_width)
        elif '%X' in text:
            text = text.replace(r'%X', hex(count).replace("0x","",1).upper()).ljust(hex_width+extra_width)
        return text

    def _format_author_string(self, authors):
        """ Given a list of <author> elements, return a readable string of names """
        buf = []
        for i, author in enumerate(authors):
            organization = author.find('organization')
            initials, surname = short_author_name_parts(author)
            if i == len(authors) - 1 and len(authors) > 1:
                buf.append('and ')
            if surname:
                initials = self.get_initials(author) or initials or ''
                if i == len(authors) - 1 and len(authors) > 1:
                    # Last author is rendered in reverse
                    if len(initials) > 0:
                        buf.append(initials + ' ' + \
                                     surname)
                    else:
                        buf.append(surname)
                elif len(initials) > 0:
                    buf.append(surname + ', ' + initials)
                else:
                    buf.append(surname)
                if author.attrib.get('role', '') == 'editor':
                    buf.append(', Ed.')
            elif organization is not None and organization.text:
                # Use organization instead of name
                buf.append(organization.text.strip())
            else:
                continue
            if len(authors) == 2 and i == 0:
                buf.append(' ')
            elif i < len(authors) - 1:
                buf.append(', ')
        return ''.join(buf)

    def _prepare_top_left(self):
        """ Returns a lines of lines for the top left header """
        lines = []
        # Document stream / workgroup
        if not self.pis['private']:
            if self.draft:
                workgroup = self.r.find('front/workgroup')
                if workgroup is not None and workgroup.text:
                    lines.append(workgroup.text)
                else:
                    lines.append(self.boilerplate['draft_workgroup'])
            else:
                # Determine 'workgroup' from submissionType
                subtype = self.r.attrib.get('submissionType', 
                                            self.defaults['submissionType'])
                docstream = self.boilerplate['document_stream'].get(subtype)
                lines.append(docstream)

            # RFC number
            if not self.draft:
                lines.append('Request for Comments: ' + self.rfcnumber)
            elif not self.pis['private']:
                lines.append('Internet-Draft')

            # Series number
            category = self.r.attrib.get('category', '')
            seriesNo = self.r.attrib.get('seriesNo')
            if seriesNo is not None and category in self.boilerplate['series_name']:
                lines.append('%s: %s' % (self.boilerplate['series_name'][category], 
                                         seriesNo))

            # RFC relation notice
            approved_text = self.draft and '(if approved)' or ''
            obsoletes = self.r.attrib.get('obsoletes')
            if obsoletes:
                wrapper = textwrap.TextWrapper(width=40, subsequent_indent=' '*len('Obsoletes: '))
                line = 'Obsoletes: %s %s' % (obsoletes, approved_text)
                lines += wrapper.wrap(line)
            updates = self.r.attrib.get('updates')
            if updates:
                wrapper = textwrap.TextWrapper(width=40, subsequent_indent=' '*len('Updates: '))
                line = 'Updates: %s %s' % (updates, approved_text)
                lines += wrapper.wrap(line)

            # Category
            if category:
                cat_text = self.boilerplate[category]
                if self.draft:
                    lines.append('Intended status: ' + cat_text)
                else:
                    lines.append('Category: ' + cat_text)
            else:
                xml2rfc.log.warn('No category specified for document.')

            # Expiration notice for drafts
            if self.expire_string and not self.pis['private']:
                lines.append('Expires: ' + self.expire_string)

            # ISSN identifier
            if not self.draft:
                lines.append('ISSN: %s' % self.boilerplate['issn'])

        # Strip any whitespace from XML to make header as neat as possible
        lines = [ l.rstrip() for l in lines ]
        return lines

    def _prepare_top_right(self):
        """ Returns a list of lines for the top right header """
        lines = []
        # Keep track of previous organization and remove if redundant.
        last_org = None
        last_pos = None
        authors = self.r.findall('front/author')
        authors = [ a for a in authors if a.get('role') != 'contributor' ]
        for author in authors:
            role = author.attrib.get('role', '')
            if role == 'editor':
                role = ', Ed.'
            initials = self.get_initials(author)
            lines.append(initials + ' ' + author.attrib.\
                         get('surname', '') + role)
            organization = author.find('organization')
            org_name = ''
            if self.options.first_page_author_org:
                if organization is not None:
                    abbrev = organization.attrib.get("abbrev", None)
                    if  abbrev != None and abbrev.strip() != '':
                        org_name = abbrev.strip()
                    elif organization.text and organization.text.strip() != '':
                        org_name = organization.text.strip()
                if org_name == '':
                    lines.append('')
                else:
                    if org_name == last_org:
                        # Remove redundant organization
                        del lines[last_pos]
                    lines.append(org_name)
            last_org = org_name
            last_pos = len(lines)-1
        # remove blank lines between authors and date
        if lines[last_pos] == '':
            del lines[last_pos]
            last_pos = len(lines)-1

        date = self.r.find('front/date')
        if date is not None:
            year = date.attrib.get('year', '')
            month = date.attrib.get('month', '')
            day = date.attrib.get('day', '')
            if month:
                if month.isdigit():
                    month = calendar.month_name[int(month)]
                month = month + ' '
            if day:
                day = day + ', '
            lines.append(month + day + year)
            # Strip any whitespace from XML to make header as neat as possible
            lines = [ l.strip() for l in lines ]
        return lines

    def write_figure(self, figure):
        """ Writes <figure> elements """
        figure_align = figure.attrib.get('align', self.defaults['figure_align'])
        anchor = figure.attrib.get('anchor')
        title = figure.attrib.get('title', self.defaults['figure_title'])
        suppress_title = figure.attrib.get('suppress-title', 'false')

        # Keep track of count if there is an anchor, or PI was enabled
        if anchor or self.pis['figurecount'] == 'yes':
            self.figure_count += 1
        
        if anchor:
            # Insert anchor(s) for the figure
            self.insert_anchor('rfc.figure.' + str(self.figure_count))
            self.insert_anchor(anchor)
            if self.indexmode:
                # Add figure to the index, inserting any anchors necessary
                self._indexFigure(self.figure_count, anchor=anchor, title=title)

        # Write preamble
        preamble = figure.find('preamble')
        if preamble is not None:
            self.write_t_rec(preamble, align=figure_align)

        # iref
        for element in figure:
            if element.tag == 'iref':
                self._add_iref_to_index(element)

        # Write figure with optional delimiter
        delimiter = figure.pis['artworkdelimiter']
        artwork = figure.find('artwork')
        artwork_align = artwork.attrib.get('align', figure_align)
        blanklines = int(figure.pis['artworklines'])
        self.write_raw(artwork.text, align=artwork_align,
                       blanklines=blanklines, delimiter=delimiter,
                       source_line=figure.sourceline)

        # Write postamble
        postamble = figure.find('postamble')
        if postamble is not None:
            self.write_t_rec(postamble, align=figure_align)

        # Write label
        title = figure.attrib.get('title', '')
        if anchor or self.pis['figurecount'] == 'yes':
            if suppress_title == 'false':
                if title:
                    title = 'Figure ' + str(self.figure_count) + ': ' + title
                else:
                    title = 'Figure ' + str(self.figure_count)
        if title:
            self.write_label(title, type='figure', source_line=figure.sourceline)

    def write_table(self, table):
        """ Writes <texttable> elements """
        align = table.attrib.get('align', self.defaults['table_align'])
        anchor = table.attrib.get('anchor')
        title = table.attrib.get('title', self.defaults['table_title'])
        suppress_title = table.attrib.get('suppress-title', 'false')

        # Keep track of count if there is an anchor, or PI was enabled
        if anchor or self.pis['tablecount'] == 'yes':
            self.table_count += 1

        if anchor:
            # Insert anchor(s) for the table
            self.insert_anchor('rfc.table.' + str(self.table_count))
            self.insert_anchor(anchor)
            if self.indexmode:
                # Add table to the index, inserting any anchors necessary
                self._indexTable(self.table_count, anchor=anchor, title=title)

        # Write preamble
        preamble = table.find('preamble')
        if preamble is not None:
            self.write_t_rec(preamble, align=align)

        # Write table
        self.draw_table(table, table_num=self.table_count)

        # Write postamble
        postamble = table.find('postamble')
        if postamble is not None:
            self.write_t_rec(postamble, align=align)

        # Write label if anchor is set or PI figurecount = yes
        if anchor or self.pis['tablecount'] == 'yes':
            title = table.attrib.get('title', '')
            if suppress_title == 'false':
                if title:
                    title = 'Table ' + str(self.table_count) + ': ' + title
                else:
                    title = 'Table ' + str(self.table_count)
        if title:
            self.write_label(title, type='table', source_line=table.sourceline)

    def _index_t_rec(self, element):
        """ Traverse a <t> element only performing indexing operations """
        pass
        

    def numToAlpha(self, n):
        str = ""
        az = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
        while n > 0:
            r = (n-1) % 26
            n = int((n-1)/26)
            str = az[r] + str
        return str

    def write_section_rec(self, section, count_str="1.", appendix=False,
                           level=0, numbered=True, emit_numbered="all", s_count=1):
        """ Recursively writes <section> elements 
        
            We use the self.indexmode flag to determine whether or not we
            render actual text at this point or just lookup header information
            to construct the index
        """
        if level > 0:
            anchor = section.attrib.get('anchor')
            title = section.attrib.get('title')
            include_toc = section.attrib.get('toc', self.defaults['section_toc']) != 'exclude' \
                          and (not appendix or self.pis['tocappendix'] == 'yes')
            if level == 1:
                numbered = numbered and self.is_numbered(section)
            else:
                numbered = self.is_numbered(section)
            if level > 1 and not numbered:
                if not self.indexmode:
                    xml2rfc.log.warn('Unnumbered subsections are not permitted: found section "%s" with attribute numbered="no" or "false"' % (title, ))
                numbered = True
            if self.indexmode:
                # Add section to the index
                self._indexSection(count_str, title=title, anchor=anchor,
                                   toc=include_toc, level=level,
                                   appendix=appendix, numbered=numbered)
            else:
                # Write the section heading
                if numbered:
                    autoAnchor = 'rfc.' + ('section.' if not appendix else 'appendix.') + count_str
                else:
                    autoAnchor = 'rfc.' + re.sub('[^A-Za-z0-9]+', '_', title).lower()
                bullet = appendix and level == 1 and 'Appendix %s' % count_str or count_str
                self.write_heading(title, bullet=bullet + '.' if numbered else "",
                                   autoAnchor=autoAnchor,
                                   anchor=anchor, level=level)
        else:
            # Must be <middle> or <back> element -- no title or index.
            count_str = ''
            numbered = True

        p_count = 1  # Paragraph counter
        for element in section:
            # Check for a PI
            if element.tag is lxml.etree.PI:
                pidict = self.parse_pi(element)
                if pidict and "needLines" in pidict:
                    self.needLines(pidict["needLines"])
            # Write elements in XML document order
            if element.tag == 't':
                anchor = element.attrib.get('anchor')
                if self.indexmode:
                    self._indexParagraph(count_str, p_count, anchor=anchor)
                autoAnchor = 'rfc.section.' + count_str + '.p.' + str(p_count)
                self.write_t_rec(element, autoAnchor=autoAnchor)
                p_count += 1
            elif element.tag == 'figure':
                self.write_figure(element)
            elif element.tag == 'texttable':
                self.write_table(element)
            elif element.tag == 'iref':
                self._add_iref_to_index(element)

        # Append a dot to separate sub counters
        if count_str:
            count_str += '.'

        # Recurse on sections
        for child_sec in section.findall('section'):
            if level == 0:
                if numbered:
                    numbered = self.is_numbered(child_sec)
                elif self.is_numbered(child_sec):
                    title = child_sec.attrib.get('title')
                    if not self.indexmode:
                        xml2rfc.log.warn('Numbered sections are not permitted after unnumbered sections: found section "%s" without attribute numbered="no"' % (title,))
                    numbered = False
                if emit_numbered=="only" and not numbered:
                    continue
                if emit_numbered=="no" and numbered:
                    continue
            if appendix == True and not count_str:
                if s_count == 1 and self.pis["rfcedstyle"] == "yes":
                   self.needLines(-1)
                # Use an alphabetic counter for first-level appendix
                self.write_section_rec(child_sec, self.numToAlpha(s_count),
                                        level=level + 1, appendix=True, numbered=numbered)
            else:
                # Use a numeric counter
                self.write_section_rec(child_sec, count_str + str(s_count), 
                                        level=level + 1, appendix=appendix, numbered=numbered)

            s_count += 1

        # Set the ending index number so we know where to begin references
        if count_str == '' and appendix == False:
            self.refs_start = s_count

        return s_count

    def write_status_section(self):
        """ Writes the 'Status of this Memo' section """

        self.write_heading('Status of This Memo', autoAnchor='rfc.status')
        if not self.draft:  #RFC
            # Status boilerplate is determined by category/submissionType/consensus
            category = self.r.attrib.get('category', 'none')
            stream = self.r.attrib.get('submissionType', 
                                       self.defaults['submissionType'])
            # Consensus is assumed 'yes' for Standards Track documents and BCPs
            consensus = category in ['std', 'bcp'] and 'yes' or \
                        self.r.attrib.get('consensus', self.defaults['consensus'])
            workgroup = ''
            wg_element = self.r.find('front/workgroup')
            if wg_element is not None and wg_element.text:
                workgroup = wg_element.text

            # Write first paragraph
            if category in self.boilerplate['status']:
                self.write_paragraph(self.boilerplate['status'][category].get('p1', ''))
            
            # Build second paragraph
            p2 = []
            if category in self.boilerplate['status']:
                p2.append(self.boilerplate['status'][category].get('p2', ''))
            if stream == 'IETF':
                if consensus == 'yes':
                    p2.append(self.boilerplate['status']['IETF_consensus'])
                else:
                    p2.append(self.boilerplate['status']['IETF'])
            elif stream == 'IRTF':
                p2.append(self.boilerplate['status']['IRTF'])
                if workgroup:
                    if consensus == 'yes':
                        p2.append(self.boilerplate['status']['IRTF_workgroup_consensus'] \
                                  % workgroup)
                    else:
                        p2.append(self.boilerplate['status']['IRTF_workgroup'] \
                                  % workgroup)
            elif stream == 'IAB':
                if consensus == 'yes':
                    p2.append(self.boilerplate['status']['IAB_consensus'])
                else:
                    p2.append(self.boilerplate['status']['IAB'])
            else:
                p2.append(self.boilerplate['status'].get(stream, ''))

            # Last sentence of p2
            if self.date < datetime.date(year=2016, month=6, day=1):
                status="status_5741"
            else:
                status="status"         # Current boilerplate
            if stream == 'IETF' and category == 'std':
                p2.append(self.boilerplate[status]['p2end_ietf_std'])
            elif stream == 'IETF' and category == 'bcp':
                p2.append(self.boilerplate[status]['p2end_ietf_bcp'])
            elif stream == 'IETF':
                p2.append(self.boilerplate[status]['p2end_ietf_other'])
            else:
                p2.append(self.boilerplate[status]['p2end_other'] \
                          % self.approvers.get(stream, ''))


            # Write second paragraph
            self.write_paragraph('  '.join(p2))
            
            # Write third paragraph
            key = 'p3' if self.date < self.boilerplate_https_date else 'p3_s'
            self.write_paragraph(self.boilerplate['status'][key] % self.rfcnumber)

        else:  # Draft
            # Start by checking for an ipr header
            ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
            if '200902' in ipr:
                self.write_paragraph(self.boilerplate['ipr_200902_status'])
            elif '200811' in ipr:
                self.write_paragraph(self.boilerplate['ipr_200811_status'])

            # Write the standard draft status
            key = 'draft' if self.date < self.boilerplate_https_date else 'draft_s'
            for par in self.boilerplate['status'][key]:
                self.write_paragraph(par)

            # Write expiration string, if it was generated
            if self.expire_string:
                self.write_paragraph( \
                    self.boilerplate['draft_expire'] % self.expire_string)
    
    def write_copyright(self):
        """ Writes the 'Copyright' section """
        self.write_heading('Copyright Notice', autoAnchor='rfc.copyrightnotice')

        # Write header line with year
        date = self.r.find('front/date')
        year = ''
        if date is not None:
            year = date.attrib.get('year', self.date.year)
        self.write_paragraph(self.boilerplate['base_copyright_header'] % year)

        # Write next paragraph which may be modified by ipr
        ipr = self.r.attrib.get('ipr', self.defaults['ipr'])
        key = 'base_copyright_body' if self.date < self.boilerplate_https_date else 'base_copyright_body_s'
        body = self.boilerplate[key]
        if '200902' in ipr and self.r.attrib.get('submissionType', 
                                   self.defaults['submissionType']) == 'IETF':
            body += '  ' + self.boilerplate['ipr_200902_copyright_ietfbody']
        self.write_paragraph(body)
        
        # Write any additional paragraphs
        if 'noModification' in ipr:
            self.write_paragraph(self.boilerplate['ipr_noModification_copyright'])
        elif 'noDerivatives' in ipr:
            self.write_paragraph(self.boilerplate['ipr_noDerivatives_copyright'])
        elif ipr == 'pre5378Trust200902':
            self.write_paragraph(self.boilerplate['ipr_pre5378Trust200902_copyright'])

    def _build_index(self):
        self.indexmode = True
        # Reset document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0
        self.pis = self.xmlrfc.getpis()

        # Abstract
        abstract = self.r.find('front/abstract')
        if abstract is not None:
            self.write_heading('Abstract', autoAnchor='rfc.abstract')
            for t in abstract.findall('t'):
                self.write_t_rec(t)

        # Authors
        count = 0
        authors = self.r.findall('front/author')
        for author in authors:
            count += 1
            autoName = full_author_name_expansion(author)
            self._indexAuthor(count, author.get('anchor'), autoName)

        # Optional notes
        for note in self.r.findall('front/note'):
            self.write_heading(note.attrib.get('title', 'Note'))
            for t in note.findall('t'):
                self.write_t_rec(t)

        # Middle sections
        middle = self.r.find('middle')
        if middle is not None:
            self.write_section_rec(middle, None)

        # References sections
        # Treat references as nested only if there is more than one
        ref_counter = 0
        refs_counter = str(self.refs_start)
        references = self.r.findall('back/references')
        # Write root level references header
        refs_title = self.pis['refparent']
        if len(references) == 1 and not self.eref_list:
            refs_title = references[0].attrib.get('title', refs_title)

        if len(references) > 0:
            self._indexReferences(refs_counter, title=refs_title)

        if len(references) > 1 or self.eref_list:
            for i, reference_list in enumerate(references):
                refs_newcounter = refs_counter + '.' + str(i + 1)
                refs_title = reference_list.attrib.get('title', self.pis["refparent"])
                self._indexReferences(refs_newcounter, title=refs_title, \
                                      subCounter=i+1, level=2)
            if self.eref_list:
                refs_newcounter = refs_counter + '.' + str(len(references)+1)
                self._indexReferences(refs_newcounter, title="URIs", level=2, subCounter=len(references)+1)


        for ref in self.r.xpath('.//references//reference'):
            if len(ref):
                ref_counter += 1
                title = ref.find("front/title")
                if title != None:
                    if 'anchor' in ref.attrib:
                        self._indexRef(ref_counter, title=title.text, anchor=ref.attrib["anchor"])
                    else:
                        raise RfcWriterError("Reference is missing an anchor: %s" % lxml.etree.tostring(ref))
                        
        # Appendix sections
        back = self.r.find('back')
        if back is not None:
            s_count = self.write_section_rec(back, None, appendix=True, emit_numbered="only")

        # Index section, disable if there are no irefs
        if len(self._iref_index) > 0:
            # Add explicitly to index
            title = 'Index'
            autoAnchor = 'rfc.index'
            item = _RfcItem(title, autoAnchor, title=title)
            self._index.append(item)

        if back is not None:
            self.write_section_rec(back, None, appendix=True, emit_numbered="no", s_count=s_count)

        # Authors addresses section

        if self.pis['authorship'] == 'yes':
            authors = self.r.findall('front/author')
            authors = [ a for a in authors if a.get('role') != 'contributor' ]
            autoAnchor = 'rfc.authors'
            if len(authors) > 1:
                title = "Authors' Addresses"
            else:
                title = "Author's Address"
            # Add explicitly to index
            item = _RfcItem(title, autoAnchor, title=title)
            self._index.append(item)

    def _build_document(self):
        self.indexmode = False
        # Reset document counters
        self.ref_start = 1              # Start of reference counters
        self.refs_start = 1             # Start of references sections
        self.figure_count = 0
        self.table_count = 0
        self.eref_count = 0
        self.pis = self.xmlrfc.getpis()

        # Block header
        topblock = self.pis['topblock']
        if topblock == 'yes':
            self.write_top(self._prepare_top_left(), \
                               self._prepare_top_right())

        # Title & Optional docname
        title = self.r.find('front/title')
        if title is not None:
            docName = self.r.attrib.get('docName', None)
            rfcnum = self.r.attrib.get('number', None)
            if (not docName or not docName.strip()) and not rfcnum:
                xml2rfc.log.warn("No (or empty) 'docName' attribute in the <rfc/> element -- can't insert draft name on first page.")
            if docName and '.' in docName:
                xml2rfc.log.warn("The 'docName' attribute of the <rfc/> element should not contain any filename extension: docName=\"draft-foo-bar-02\".")
            if docName and not rfcnum and not re.search('-\d\d$', docName):
                xml2rfc.log.warn("The 'docName' attribute of the <rfc/> element should have a revision number as the last component: docName=\"draft-foo-bar-02\".")
            self.write_title(title.text, docName, title.sourceline)

        # Abstract
        abstract = self.r.find('front/abstract')
        if abstract is not None:
            self.write_heading('Abstract', autoAnchor='rfc.abstract')
            for t in abstract.findall('t'):
                self.write_t_rec(t)

        # Optional notified boilerplate
        # Disabled. See issue #123, http://trac.tools.ietf.org/tools/xml2rfc/trac/ticket/123
        ## if self.pis['iprnotified'] == 'yes':
        ##    self.write_paragraph(BaseRfcWriter.boilerplate['iprnotified'])

        # Optional notes
        for note in self.r.findall('front/note'):
            self.write_heading(note.attrib.get('title', 'Note'))
            for t in note.findall('t'):
                self.write_t_rec(t)

        if not self.pis['private']:
            # Verify that 'ipr' attribute is valid before continuing
            self._validate_ipr()

            # "Status of this Memo" section
            self.write_status_section()

            # Copyright section
            self.write_copyright()

        # Insert the table of contents marker at this position
        toc_enabled = self.pis['toc']
        if toc_enabled == 'yes':
            self.insert_toc()

        # Middle sections
        middle = self.r.find('middle')
        if middle is not None:
            self.write_section_rec(middle, None)

        # References sections
        # Treat references as nested only if there is more than one
        refs_counter = str(self.refs_start)
        references = self.r.findall('back/references')
        # Write root level references header
        refs_title = self.pis['refparent']
        if len(references) == 1 and not self.eref_list:
            refs_title = references[0].attrib.get('title', refs_title)

        if len(references) > 0 or self.eref_list:
            self.write_heading(refs_title, bullet=refs_counter + '.', \
                               autoAnchor='rfc.references')
        if len(references) > 1:
            for i, reference_list in enumerate(references):
                refs_newcounter = refs_counter + '.' + str(i + 1)
                refs_title = reference_list.attrib.get('title', self.pis['refparent'])
                autoAnchor = 'rfc.references.' + str(i + 1)
                self.write_heading(refs_title, bullet=refs_newcounter + '.',\
                                   autoAnchor=autoAnchor, level=2)
                self.write_reference_list(reference_list)
        elif len(references) == 1:
            if self.eref_list:
                refs_newcounter = refs_counter + '.1'
                refs_title = references[0].attrib.get('title', self.pis['refparent'])
                autoAnchor = 'rfc.references.1'
                self.write_heading(refs_title, bullet=refs_newcounter + '.',\
                                   autoAnchor=autoAnchor, level=2)
            self.write_reference_list(references[0])

        if self.eref_list:
            self.write_erefs(refs_counter, len(references)+1)

        # Appendix sections
        back = self.r.find('back')
        if back is not None:
            s_count = self.write_section_rec(back, None, appendix=True, emit_numbered="only")

        # Index section, disable if there are no irefs
        if len(self._iref_index) > 0:
            self.insert_iref_index()

        if back is not None:
            self.write_section_rec(back, None, appendix=True, emit_numbered="no", s_count=s_count)

        self.write_crefs()

        # Authors addresses section
        if self.pis['authorship'] == 'yes':
            authors = self.r.findall('front/author')
            authors = [ a for a in authors if a.get('role') != 'contributor' ]
            autoAnchor = 'rfc.authors'
            if len(authors) > 1:
                title = "Authors' Addresses"
            else:
                title = "Author's Address"
            self.write_heading(title, autoAnchor=autoAnchor)
            for author in authors:
                self.write_address_card(author)

        self.check_for_unused_references()
        
    def write(self, filename, tmpfile=None):
        """ Public method to write the RFC document to a file. """
        # If the ascii flag is enabled, replace unicode with ascii in the tree
        if self.ascii:
            xml2rfc.utils.safeReplaceUnicode(self.r)

        # Protect any words with slashes that are on the preserve list
        xml2rfc.utils.safeTagSlashedWords(self.r)

        # Do any pre processing necessary, such as inserting metadata
        self.pre_indexing()
        # Make two passes over the document, the first pass we run in
        # 'index mode' to construct the internal index and other things, 
        # the second pass will assemble a buffer and render the actual text
        self._build_index()
        # Format the date properly
        self._format_date()
        # Do any pre-build processing necessary, such as inserting metadata
        self.pre_rendering()
        # Build the document
        self._build_document()
        # Primary buffer is finished -- apply any post processing
        self.post_rendering()

        # Finished processing, write to file
        # Override file with keyword argument if passed in, ignoring filename.
        # Warning -- make sure file is open and ready for writing!
        if not tmpfile:
            if self.ascii:
                file = open(filename, 'w')
            else:
                # Open as unicode
                file = codecs.open(filename, 'w', encoding='utf-8')
            self.write_to_file(file)
            file.close()
        else:
            self.write_to_file(tmpfile)

        if not self.options.quiet and filename:
            xml2rfc.log.write(' Created file', filename)

    def write_erefs(self, refs_counter, refs_subsection):
        """ Only text versions do this so provide a default that does nothing
        """
        pass

    def write_crefs(self):
        """ Only text versions do this so provide a default that does nothing
        """
        pass

    def check_for_unused_references(self):
        """ If this is a reference and it is not used - then warn me
        """
        if not self.indexmode:
            for item in self._index:
                if item.autoAnchor.startswith("rfc.ref.") and not item.used:
                    xml2rfc.log.warn("No <xref> in <rfc> targets <reference anchor='%s'>" % item.anchor)

    def needLines(self, count):
        """ Deal with the needLines PI """
        pass
                
    # -----------------------------------------
    # Base writer interface methods to override
    # -----------------------------------------

    def insert_toc(self):
        """ Marks the current buffer position to insert ToC at """
        raise NotImplementedError('insert_toc() needs to be overridden')
    
    def insert_iref_index(self):
        """ Marks the current buffer position to insert the index at """
        raise NotImplementedError('insert_iref_index() needs to be '
                                  'overridden')

    def write_raw(self, text, indent=3, align='left', blanklines=0,
                  delimiter=None, source_line=None):
        """ Writes a block of text that preserves all whitespace """
        raise NotImplementedError('write_raw() needs to be overridden')

    def write_label(self, text, type='figure', source_line=None):
        """ Writes a table or figure label """
        raise NotImplementedError('write_label() needs to be overridden')

    def write_title(self, title, docName=None, source_line=None):
        """ Writes the document title """
        raise NotImplementedError('write_title() needs to be overridden')

    def write_heading(self, text, bullet='', autoAnchor=None, anchor=None,
                      level=1, breakNeed=0):
        """ Writes a section heading """
        raise NotImplementedError('write_heading() needs to be overridden')

    def write_paragraph(self, text, align='left', autoAnchor=None):
        """ Writes a paragraph of text """
        raise NotImplementedError('write_paragraph() needs to be overridden')

    def write_t_rec(self, t, align='left', autoAnchor=None):
        """ Recursively writes <t> elements """
        raise NotImplementedError('write_t_rec() needs to be overridden')

    def write_top(self, left_header, right_header):
        """ Writes the main document header

            Takes two list arguments, one for each margin, and combines them
            so that they exist on the same lines of text
        """
        raise NotImplementedError('write_top() needs to be overridden')

    def write_address_card(self, author):
        """ Writes the address information for an <author> element """
        raise NotImplementedError('write_address_card() needs to be ' \
                                  'overridden')

    def write_reference_list(self, list):
        """ Writes a <references> element """
        raise NotImplementedError('write_reference_list() needs to be ' \
                                  'overridden')

    def insert_anchor(self, text):
        """ Inserts a document anchor for internal links """
        raise NotImplementedError('insert_anchor() needs to be overridden')

    def draw_table(self, table, table_num=None):
        """ Draws a formatted table from a <texttable> element

            For HTML nothing is really 'drawn' since we can use <table>
        """
        raise NotImplementedError('draw_table() needs to be overridden')

    def pre_indexing(self):
        """ First method that is called before traversing the XML RFC tree for indexing"""
        raise NotImplementedError('pre_indexing() needs to be overridden')

    def pre_rendering(self):
        """ First method that is called before traversing the XML RFC tree for rendering"""
        raise NotImplementedError('pre_rendering() needs to be overridden')

    def post_rendering(self):
        """ Last method that is called after traversing the XML RFC tree """
        raise NotImplementedError('post_rendering() needs to be overridden')

    def write_to_file(self, file):
        """ Writes the finished buffer to a file """
        raise NotImplementedError('write_to_file() needs to be overridden')

# --------------------------------------------------------------------------------------------------
# Sets of various kinds of tags in the v3 schema
        
deprecated_element_tags = set([
    'c',
    'facsimile',
    'format',
    'list',
    'postamble',
    'preamble',
    'spanx',
    'texttable',
    'ttcol',
    'vspace',
])

v3_rng_file = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'data', 'v3.rng')
v3_schema = lxml.etree.ElementTree(file=v3_rng_file)

def get_element_tags():
    tags = set()
    elements = v3_schema.xpath("/x:grammar/x:define/x:element", namespaces=namespaces)
    for element in elements:
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_meta_tags():
    "Get tags that can have text content but don't occur in regular text"
    tags = set()
    for elem in ['author', 'address', 'front', 'postal', 'reference' ]:
        refs = v3_schema.xpath("/x:grammar/x:define[@name='%s']//x:ref"%elem, namespaces=namespaces)
        for r in refs:
            name = r.get('name')
            if not name in tags:
                tags.add(name)
    return tags

def get_text_tags():
    "Get tags that can have text content from the schema"
    tags = set()
    texts = v3_schema.xpath("/x:grammar/x:define/x:element//x:text", namespaces=namespaces)
    for t in texts:
        element = list(t.iterancestors('{*}element'))[0]
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_text_or_block_tags():
    "Get tags that can have either text or block contents from the schema"
    tags = set()
    texts = v3_schema.xpath("/x:grammar/x:define/x:element/x:choice/*/x:choice//x:text", namespaces=namespaces)
    for t in texts:
        element = list(t.iterancestors('{*}element'))[0]
        name = element.get('name')
        if not name in tags:
            tags.add(name)
    return tags

def get_inline_tags():
    "Get tags that can occur within text from the schema"
    tags = set()
    referenced = v3_schema.xpath("/x:grammar/x:define/x:element//x:ref", namespaces=namespaces)
    for ref in referenced:
        name = ref.get('name')
        if not name in tags:
            p = ref.getparent()
            text = p.find('x:text', namespaces=namespaces)
            if text != None:
                tags.add(name)
    return tags

def get_xref_tags():
    "Get tags that can occur as direct children of <xref>"
    tags = set()
    refs = v3_schema.xpath("/x:grammar/x:define/x:element[@name='xref']//x:ref", namespaces=namespaces)
    tags = set([ c.get('name') for c in refs ])
    return tags


meta_tags   = get_meta_tags() - deprecated_element_tags
text_tags   = get_text_tags() - deprecated_element_tags
inline_tags = get_inline_tags() - deprecated_element_tags
block_tags  = get_element_tags() - inline_tags - deprecated_element_tags
mixed_tags  = get_text_or_block_tags() - deprecated_element_tags
xref_tags   = get_xref_tags()

# --------------------------------------------------------------------------------------------------

class BaseV3Writer(object):

    def __init__(self, xmlrfc, quiet=None, options=default_options, date=datetime.date.today()):
        global v3_rng_file, v3_schema
        self.xmlrfc = xmlrfc
        self.tree = xmlrfc.tree if xmlrfc else None
        self.root = self.tree.getroot() if xmlrfc else None
        self.options = options
        self.date = date
        self.rng_file = v3_rng_file
        self.v3_rng = None
        self.schema = v3_schema
        self.index_items = []
        self.meta_tags = set(meta_tags)
        self.text_tags = set(text_tags)
        self.inline_tags = set(inline_tags)
        self.mixed_tags = set(mixed_tags)
        self.xref_tags = set(xref_tags)
        self.attribute_defaults = self.get_all_attribute_defaults()
        #
        #
        self.errors = []
#         if options.debug:
#             found_handlers = []
#             missing_handlers = []
#             for tag in (self.get_element_tags() - deprecated_element_tags):
#                 func_name = "render_%s" % (tag.lower(),)
#                 if getattr(self, func_name, False):
#                     found_handlers.append(func_name)
#                 else:
#                     missing_handlers.append(func_name)
#             debug.pprint('found_handlers')
#             debug.show('len(found_handlers)')
#             debug.pprint('missing_handlers')
#             debug.show('len(missing_handlers)')

    def log(self, msg):
        xml2rfc.log.write(msg)

    def msg(self, e, label, text):
        if e != None:
            lnum = getattr(e, 'sourceline', None)
            file = getattr(e, 'base', None)
            if lnum:
                msg = "%s(%s): %s %s" % (file or self.xmlrfc.source, lnum, label, text, )
            else:
                msg = "(No source line available): %s %s" % (label, text, )
        else:
            msg = "%s %s" % (label, text)
        return msg

    def get_relevant_pis(self, e):
        pis = []
        if e != None:
            # directly inside element
            for c in e.getchildren():
                if c.tag == lxml.etree.PI and c.target == xml2rfc.V3_PI_TARGET:
                    pis.append(c)
            # siblings before element
            for s in e.itersiblings(preceding=True):
                if s.tag == lxml.etree.PI and s.target == xml2rfc.V3_PI_TARGET:
                    pis.append(s)
            # ancestor's earlier siblings
            for a in e.iterancestors():
                for s in a.itersiblings(preceding=True):
                    if s.tag == lxml.etree.PI and s.target == xml2rfc.V3_PI_TARGET:
                        pis.append(s)
            # before root elements
            p = self.root.getprevious()
            while p != None:
                if p.tag == lxml.etree.PI and p.target == xml2rfc.V3_PI_TARGET:
                    pis.append(p)
                p = p.getprevious()
        return pis

    def silenced(self, e, text):
        text = text.strip()
        pis = self.get_relevant_pis(e)
        silenced = filter(None, self.options.silence[:] + [ pi.get('silence') for pi in pis ])
        for s in silenced:
            for label in ['Note: ', 'Warning: ']:
                if s.startswith(label):
                    s = s[len(label):]
            if text.startswith(s):
                return True
            try:
                if re.match(s, text):
                    return True
            except re.error:
                pass
        return False

    def note(self, e, text, label='Note:'):
        if self.options.verbose:
            if not self.silenced(e, text):
                self.log(self.msg(e, label, text))

    def warn(self, e, text, label='Warning:'):
        if self.options.verbose or not self.silenced(e, text):
            self.log(self.msg(e, label, text))

    def err(self, e, text, trace=False):
        msg = self.msg(e, 'Error:', text)
        self.errors.append(msg)
        if trace or self.options.debug:
            raise RuntimeError(msg)
        else:
            self.log(msg)

    def die(self, e, text, trace=False):
        msg = self.msg(e, 'Error:', text)
        self.errors.append(msg)
        raise RfcWriterError(msg)

    def get_refname_mapping(self):
        reflist = self.root.xpath('.//references/reference|.//references/referencegroup')
        if self.root.get('symRefs', 'true') == 'true':
            refname_mapping = dict( (e.get('anchor'), e.get('anchor')) for e in reflist )
        else:
            refname_mapping = (dict( (e.get('anchor'), str(i+1)) for i,e in enumerate(reflist) ))
        refname_mapping.update(dict( (e.get('target'), e.get('to')) for e in self.root.xpath('.//displayreference') ))
        return refname_mapping

    def get_all_attribute_defaults(self):
        defaults = {}
        elements = self.schema.xpath("/x:grammar/x:define/x:element", namespaces=namespaces)
        for tag in [ e.get('name') for e in elements ]:
            attr = self.schema.xpath("/x:grammar/x:define/x:element[@name='%s']//x:attribute" % tag, namespaces=namespaces)
            defaults[tag] = dict( (a.get('name'), a.get("{%s}defaultValue"%namespaces['a'], None)) for a in attr )
        return defaults

    def check_refs_numbered(self):
        # see if references should be numbered.  This is True unless the last
        # top-level section of <middle/> had numbered='false'.
        if not hasattr(self, '_refs_numbered'):
            last_middle_section = ([None, ] + list(self.root.find('middle').iterchildren('section')))[-1]
            self._refs_numbered = last_middle_section.get('numbered', 'true') == 'true'
        return self._refs_numbered

    # methods operating on the xml tree

    def get_element_from_id(self, id):
        for a in 'anchor', 'pn', 'slugifiedName':
            elem = self.root.find('.//*[@%s="%s"]'%(a, id, ))
            if elem != None:
                break
        return elem

    def get_element_page(self, e):
        page = getattr(e, 'page', None)
        if not page:
            for a in e.iterancestors():
                page = getattr(a, 'page', None)
                if page != None:
                    break
        return page

#     def get_valid_child_tags(self, tag):
#         refs = self.schema.xpath("/x:grammar/x:define/x:element[@name='%s']//x:ref" % tag, namespaces=utils.namespaces)
#         names = set([ r.get('name') for r in refs ])
#         return names

    def write(self, filename):
        raise NotImplementedError()

    # methods for use in setting up page headers and footers

    def page_top_left(self):
        if self.root.get('ipr') == 'none':
            return ''
        number = self.root.get('number')
        if number != None:
            return 'RFC %s' % number
        info = self.root.find('./front/seriesInfo[@name="RFC"]')
        if info != None:
            return 'RFC %s' % info.get('value')
        return 'Internet-Draft'

    def page_top_center(self):
        title = self.root.find('./front/title')
        text = title.get('abbrev') or ' '.join(title.itertext())
        return text

    def page_top_right(self):
        date = self.root.find('./front/date')
        year, month, day = extract_date(date, self.options.date)
        year, month, day = augment_date(year, month, day, self.options.date)
        text = format_date(year, month, None, legacy=True)
        return text

    def page_bottom_left(self):
        authors = self.root.findall('front/author')
        authors = [ a for a in authors if a.get('role') != 'contributor' ]
        surnames = [ n for n in [ short_author_ascii_name_parts(a)[1] for a in authors ] if n ]
        if len(surnames) == 1:
            text = surnames[0]
        elif len(surnames) == 2:
            text = '%s & %s' % (surnames[0], surnames[1])
        elif len(surnames) > 2:
            text = '%s, et al.' % surnames[0]
        return text

    def page_bottom_center(self):
        # Either expiry date or category
        if self.options.rfc or self.root.get('ipr') == 'none':
            cat = self.root.get('category')
            text = strings.category_name[cat]
        else:
            date = get_expiry_date(self.tree, self.options.date)
            parts = date.year, date.month, date.day
            text = 'Expires %s' % format_date(*parts, legacy=self.options.legacy_date_format)
        return text

    def downcode_punctuation(self):
        self.downcode(replacements=punctuation)
        self.downcode_reference_punctuation()

    def downcode(self, replacements=unicode_replacements):
        """

        Traverses an lxml.etree and replaces unicode characters with the proper
        equivalents specified in rfc2629-xhtml.ent, resulting in no non-ascii
        characters except in elements that explicitly permit non-ascii content.

        """
        for e in self.tree.iter():
            if e.text:
                if not e.tag in unicode_content_tags:
                    try:
                        e.text.encode('ascii')
                    except UnicodeEncodeError:
                        e.text = downcode(e.text, replacements=replacements)
                elif e.tag in bare_unicode_tags:
                    pass
                elif e.tag in bare_latin_tags and is_script(e.text, 'Latin'):
                    pass
                elif not e.get('ascii'):
                    try:
                        e.text.encode('ascii')
                    except UnicodeEncodeError:
                        e.text = downcode(e.text, replacements=replacements)
            if e.tail:
                if not e.getparent().tag in unicode_content_tags:
                    try:
                        e.tail.encode('ascii')
                    except UnicodeEncodeError:
                        e.tail = downcode(e.tail, replacements=replacements)

    def downcode_attributes(self, replacements=unicode_replacements):
        for e in self.tree.iter():
            for key in e.attrib.keys():
                if not (e.tag, key) in unicode_attributes:
                    try:
                        e.get(key).encode('ascii')
                    except UnicodeEncodeError:
                        e.set(key, downcode(e.get(key), replacements=replacements))

    def downcode_reference_punctuation(self):
        for r in self.tree.xpath('.//references'):
            for e in r.iter():
                for key in e.attrib.keys():
                    try:
                        e.get(key).encode('ascii')
                    except UnicodeEncodeError:
                        e.set(key, downcode_punctuation(e.get(key)))

    def pretty_print_prep(self, e, p):
        ind = self.options.indent
        ## The actual printing is done in self.write()
        def indent(e, i):
            if e.tag in (lxml.etree.CDATA, ):
                return
            if e.tag in (lxml.etree.Comment, lxml.etree.PI, ):
                if not e.tail:
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
                return
            #
            if e.tag not in text_tags:
                if len(e) and (e.text is None or e.text.strip()==''):
                    e.text = '\n'+' '*(i+ind)
            elif e.tag in ['blockquote', 'li', 'dd', 'td', 'th' ]: # mixed content
                pass
                if len(e) and e[0] not in inline_tags and (e.text is None or e.text.strip()==''):
                    e.text = '\n'+' '*(i+ind)
            elif e.tag in ['artwork', 'sourcecode', ]:
                pass
            else:
                # inline tag
                if e.tail == '':
                    e.tail = None
                if len(e):
                    z = e[-1]
                    if z.tail and re.search(r'\n[ \t]+$', z.tail):
                        z.tail = re.sub(r'\n[ \t]+$', '\n'+' '*(i), z.tail)
                else:
                    if e.text and re.search(r'\n[ \t]+$', e.text):
                        e.text = re.sub(r'\n[ \t]+$', '\n'+' '*(i), e.text)
                    
            #
            for c in e:
                indent(c, i+ind)
            #
            if e.tag not in inline_tags:# and e.tag not in ['artwork', 'sourcecode', ]:
                if e.tail is None or e.tail.strip()=='':
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
                else:
                    self.warn(e, 'Unexpected tail: <%s>%s' % (e.tag, e.tail))
            else:
                if e.tail != None and e.tail.strip()=='' and '\n' in e.tail:
                    if e.getnext() != None:
                        e.tail = '\n'+' '*i
                    else:
                        e.tail = '\n'+' '*(i-ind)
        indent(e, 0)
        e.tail = None

    def validate(self, when='', warn=False):
        # Note: Our schema doesn't permit xi:include elements, so the document
        # must have had XInclude processing done before calling validate()

        # The lxml Relax NG validator checks that xsd:ID values are unique,
        # but unfortunately the error messages are completely unhelpful (lxml
        # 4.1.1, libxml 2.9.1): "Element li has extra content: t" when 't' has
        # a duplicate xsd:ID attribute.  So we check all attributes with
        # content specified as xsd:ID first, and give better messages:

        # Get the attributes we need to check
        if when and not when.startswith(' '):
            when = ' '+when
        dups = find_duplicate_ids(self.schema, self.tree)
        for attr, id, e in dups:
            self.warn(e, 'Duplicate xsd:ID attribute %s="%s" found.  This will cause validation failure.' % (attr, id, ))

        try:
            if not self.v3_rng:
                self.v3_rng = lxml.etree.RelaxNG(file=self.rng_file)
                
            self.v3_rng.assertValid(self.tree)
            return True
        except Exception as e:
            lxmlver = lxml.etree.LXML_VERSION[:3]
            if lxmlver < (3, 8, 0):
                self.warn(None, "The available version of the lxml library (%s) does not provide xpath "
                          "information as part of validation errors.  Upgrade to version 3.8.0 or "
                          "higher for better error messages." % ('.'.join(str(v) for v in lxmlver), ))
            # These warnings are occasionally incorrect -- disable this
            # output for now:
            if hasattr(e, 'error_log'):
                for error in e.error_log:
                    path = getattr(error, 'path', '')
                    msg = "%s(%s): %s: %s, at %s" % (self.xmlrfc.source, error.line, error.level_name.title(), error.message, path)
                    self.log(msg)
                    if error.message.startswith("Did not expect text"):
                        items = self.tree.xpath(error.path + '/text()')
                        for item in items:
                            if item.strip():
                                self.log("  Unexpected text:\n    %s" % item.strip())

            else:
                log.warn('\nInvalid document: %s' % (e,))
            if warn:
                self.warn(self.root, 'Invalid document%s.' % (when, ))
                return False
            else:
                self.die(self.root, 'Invalid document%s.' % (when, ))

    def remove(self, p, e):
        # Element.remove(child) removes both the child and its tail, so in
        # order not to loose text when removing comments or other elements,
        # we need to handle that case:
        if e.tail:
            s = e.getprevious()
            if s != None:
                s.tail = (s.tail or '') + e.tail
            else:
                p.text = (p.text or '') + e.tail
        p.remove(e)
