lxml.html

Package lxml :: Package html |
[hide private]
[frames] | no frames]

# Source Code for Package lxml.html

       1  # Copyright (c) 2004 Ian Bicking. All rights reserved.
       2  #
       3  # Redistribution and use in source and binary forms, with or without
       4  # modification, are permitted provided that the following conditions are
       5  # met:
       6  #
       7  # 1. Redistributions of source code must retain the above copyright
       8  # notice, this list of conditions and the following disclaimer.
       9  #
      10  # 2. Redistributions in binary form must reproduce the above copyright
      11  # notice, this list of conditions and the following disclaimer in
      12  # the documentation and/or other materials provided with the
      13  # distribution.
      14  #
      15  # 3. Neither the name of Ian Bicking nor the names of its contributors may
      16  # be used to endorse or promote products derived from this software
      17  # without specific prior written permission.
      18  #
      19  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
      20  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
      21  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
      22  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IAN BICKING OR
      23  # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
      24  # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
      25  # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
      26  # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
      27  # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
      28  # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      29  # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30
      31  """The ``lxml.html`` tool set for HTML handling.
      32  """
      33
      34  from __future__ import absolute_import
      35
      36  __all__ = [
      37      'document_fromstring', 'fragment_fromstring', 'fragments_fromstring', 'fromstring',
      38      'tostring', 'Element', 'defs', 'open_in_browser', 'submit_form',
      39      'find_rel_links', 'find_class', 'make_links_absolute',
      40      'resolve_base_href', 'iterlinks', 'rewrite_links', 'open_in_browser', 'parse']
      41
      42
      43  import copy
      44  import sys
      45  import re
      46  from functools import partial
      47
      48  try:
      49      # while unnecessary, importing from 'collections.abc' is the right way to do it
      50      from collections.abc import MutableMapping, MutableSet
      51  except ImportError:
      52      from collections import MutableMapping, MutableSet
      53
      54  from .. import etree
      55  from . import defs
      56  from ._setmixin import SetMixin
      57
      58  try:
      59      from urlparse import urljoin
      60  except ImportError:
      61      # Python 3
      62      from urllib.parse import urljoin
      63
      64  try:
      65      unicode
      66  except NameError:
      67      # Python 3
      68      unicode = str
      69  try:
      70      basestring
      71  except NameError:
      72      # Python 3
      73      basestring = (str, bytes)
      74
      75
      76 -def __fix_docstring(s):
      77      if not s:
      78          return s
      79      if sys.version_info[0] >= 3:
      80          sub = re.compile(r"^(\s*)u'", re.M).sub
      81      else:
      82          sub = re.compile(r"^(\s*)b'", re.M).sub
      83      return sub(r"\1'", s)
      84
      85
      86  XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
      87
      88  _rel_links_xpath = etree.XPath("descendant-or-self::a[@rel]|descendant-or-self::x:a[@rel]",
      89                                 namespaces={'x':XHTML_NAMESPACE})
      90  _options_xpath = etree.XPath("descendant-or-self::option|descendant-or-self::x:option",
      91                               namespaces={'x':XHTML_NAMESPACE})
      92  _forms_xpath = etree.XPath("descendant-or-self::form|descendant-or-self::x:form",
      93                             namespaces={'x':XHTML_NAMESPACE})
      94  #_class_xpath = etree.XPath(r"descendant-or-self::*[regexp:match(@class, concat('\b', $class_name, '\b'))]", {'regexp': 'http://exslt.org/regular-expressions'})
      95  _class_xpath = etree.XPath("descendant-or-self::*[@class and contains(concat(' ', normalize-space(@class), ' '), concat(' ', $class_name, ' '))]")
      96  _id_xpath = etree.XPath("descendant-or-self::*[@id=$id]")
      97  _collect_string_content = etree.XPath("string()")
      98  _iter_css_urls = re.compile(r'url\(('+'["][^"]*["]|'+"['][^']*[']|"+r'[^)]*)\)', re.I).finditer
      99  _iter_css_imports = re.compile(r'@import "(.*?)"').finditer
     100  _label_xpath = etree.XPath("//label[@for=$id]|//x:label[@for=$id]",
     101                             namespaces={'x':XHTML_NAMESPACE})
     102  _archive_re = re.compile(r'[^ ]+')
     103  _parse_meta_refresh_url = re.compile(
     104      r'[^;=]*;\s*(?:url\s*=\s*)?(?P<url>.*)$', re.I).search
     105
     106
     107 -def _unquote_match(s, pos):
     108      if s[:1] == '"' and s[-1:] == '"' or s[:1] == "'" and s[-1:] == "'":
     109          return s[1:-1], pos+1
     110      else:
     111          return s,pos
     112
     113
     114 -def _transform_result(typ, result):
     115      """Convert the result back into the input type.
     116      """
     117      if issubclass(typ, bytes):
     118          return tostring(result, encoding='utf-8')
     119      elif issubclass(typ, unicode):
     120          return tostring(result, encoding='unicode')
     121      else:
     122          return result
     123
     124
     125 -def _nons(tag):
     126      if isinstance(tag, basestring):
     127          if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
     128              return tag.split('}')[-1]
     129      return tag
     130
     131
     132 -class Classes(MutableSet):
     133      """Provides access to an element's class attribute as a set-like collection.
     134      Usage::
     135
     136          >>> el = fromstring('<p class="hidden large">Text</p>')
     137          >>> classes = el.classes  # or: classes = Classes(el.attrib)
     138          >>> classes |= ['block', 'paragraph']
     139          >>> el.get('class')
     140          'hidden large block paragraph'
     141          >>> classes.toggle('hidden')
     142          False
     143          >>> el.get('class')
     144          'large block paragraph'
     145          >>> classes -= ('some', 'classes', 'block')
     146          >>> el.get('class')
     147          'large paragraph'
     148      """
     149 -    def __init__(self, attributes):
     150          self._attributes = attributes
     151          self._get_class_value = partial(attributes.get, 'class', '')
     152
     153 -    def add(self, value):
     154          """
     155          Add a class.
     156
     157          This has no effect if the class is already present.
     158          """
     159          if not value or re.search(r'\s', value):
     160              raise ValueError("Invalid class name: %r" % value)
     161          classes = self._get_class_value().split()
     162          if value in classes:
     163              return
     164          classes.append(value)
     165          self._attributes['class'] = ' '.join(classes)
     166
     167 -    def discard(self, value):
     168          """
     169          Remove a class if it is currently present.
     170
     171          If the class is not present, do nothing.
     172          """
     173          if not value or re.search(r'\s', value):
     174              raise ValueError("Invalid class name: %r" % value)
     175          classes = [name for name in self._get_class_value().split()
     176                     if name != value]
     177          if classes:
     178              self._attributes['class'] = ' '.join(classes)
     179          elif 'class' in self._attributes:
     180              del self._attributes['class']
     181
     182 -    def remove(self, value):
     183          """
     184          Remove a class; it must currently be present.
     185
     186          If the class is not present, raise a KeyError.
     187          """
     188          if not value or re.search(r'\s', value):
     189              raise ValueError("Invalid class name: %r" % value)
     190          super(Classes, self).remove(value)
     191
     192 -    def __contains__(self, name):
     193          classes = self._get_class_value()
     194          return name in classes and name in classes.split()
     195
     196 -    def __iter__(self):
     197          return iter(self._get_class_value().split())
     198
     199 -    def __len__(self):
     200          return len(self._get_class_value().split())
     201
     202      # non-standard methods
     203
     204 -    def update(self, values):
     205          """
     206          Add all names from 'values'.
     207          """
     208          classes = self._get_class_value().split()
     209          extended = False
     210          for value in values:
     211              if value not in classes:
     212                  classes.append(value)
     213                  extended = True
     214          if extended:
     215              self._attributes['class'] = ' '.join(classes)
     216
     217 -    def toggle(self, value):
     218          """
     219          Add a class name if it isn't there yet, or remove it if it exists.
     220
     221          Returns true if the class was added (and is now enabled) and
     222          false if it was removed (and is now disabled).
     223          """
     224          if not value or re.search(r'\s', value):
     225              raise ValueError("Invalid class name: %r" % value)
     226          classes = self._get_class_value().split()
     227          try:
     228              classes.remove(value)
     229              enabled = False
     230          except ValueError:
     231              classes.append(value)
     232              enabled = True
     233          if classes:
     234              self._attributes['class'] = ' '.join(classes)
     235          else:
     236              del self._attributes['class']
     237          return enabled
     238
     239
     240 -class HtmlMixin(object):
     241
     242 -    def set(self, key, value=None):
     243          """set(self, key, value=None)
     244
     245          Sets an element attribute.  If no value is provided, or if the value is None,
     246          creates a 'boolean' attribute without value, e.g. "<form novalidate></form>"
     247          for ``form.set('novalidate')``.
     248          """
     249          super(HtmlElement, self).set(key, value)
     250
     251      @property
     252 -    def classes(self):
     253          """
     254          A set-like wrapper around the 'class' attribute.
     255          """
     256          return Classes(self.attrib)
     257
     258      @classes.setter
     259 -    def classes(self, classes):
     260          assert isinstance(classes, Classes)  # only allow "el.classes |= ..." etc.
     261          value = classes._get_class_value()
     262          if value:
     263              self.set('class', value)
     264          elif self.get('class') is not None:
     265              del self.attrib['class']
     266
     267      @property
     268 -    def base_url(self):
     269          """
     270          Returns the base URL, given when the page was parsed.
     271
     272          Use with ``urlparse.urljoin(el.base_url, href)`` to get
     273          absolute URLs.
     274          """
     275          return self.getroottree().docinfo.URL
     276
     277      @property
     278 -    def forms(self):
     279          """
     280          Return a list of all the forms
     281          """
     282          return _forms_xpath(self)
     283
     284      @property
     285 -    def body(self):
     286          """
     287          Return the <body> element.  Can be called from a child element
     288          to get the document's head.
     289          """
     290          return self.xpath('//body|//x:body', namespaces={'x':XHTML_NAMESPACE})[0]
     291
     292      @property
     293 -    def head(self):
     294          """
     295          Returns the <head> element.  Can be called from a child
     296          element to get the document's head.
     297          """
     298          return self.xpath('//head|//x:head', namespaces={'x':XHTML_NAMESPACE})[0]
     299
     300      @property
     301 -    def label(self):
     302          """
     303          Get or set any <label> element associated with this element.
     304          """
     305          id = self.get('id')
     306          if not id:
     307              return None
     308          result = _label_xpath(self, id=id)
     309          if not result:
     310              return None
     311          else:
     312              return result[0]
     313
     314      @label.setter
     315 -    def label(self, label):
     316          id = self.get('id')
     317          if not id:
     318              raise TypeError(
     319                  "You cannot set a label for an element (%r) that has no id"
     320                  % self)
     321          if _nons(label.tag) != 'label':
     322              raise TypeError(
     323                  "You can only assign label to a label element (not %r)"
     324                  % label)
     325          label.set('for', id)
     326
     327      @label.deleter
     328 -    def label(self):
     329          label = self.label
     330          if label is not None:
     331              del label.attrib['for']
     332
     333 -    def drop_tree(self):
     334          """
     335          Removes this element from the tree, including its children and
     336          text.  The tail text is joined to the previous element or
     337          parent.
     338          """
     339          parent = self.getparent()
     340          assert parent is not None
     341          if self.tail:
     342              previous = self.getprevious()
     343              if previous is None:
     344                  parent.text = (parent.text or '') + self.tail
     345              else:
     346                  previous.tail = (previous.tail or '') + self.tail
     347          parent.remove(self)
     348
     349 -    def drop_tag(self):
     350          """
     351          Remove the tag, but not its children or text.  The children and text
     352          are merged into the parent.
     353
     354          Example::
     355
     356              >>> h = fragment_fromstring('<div>Hello <b>World!</b></div>')
     357              >>> h.find('.//b').drop_tag()
     358              >>> print(tostring(h, encoding='unicode'))
     359              <div>Hello World!</div>
     360          """
     361          parent = self.getparent()
     362          assert parent is not None
     363          previous = self.getprevious()
     364          if self.text and isinstance(self.tag, basestring):
     365              # not a Comment, etc.
     366              if previous is None:
     367                  parent.text = (parent.text or '') + self.text
     368              else:
     369                  previous.tail = (previous.tail or '') + self.text
     370          if self.tail:
     371              if len(self):
     372                  last = self[-1]
     373                  last.tail = (last.tail or '') + self.tail
     374              elif previous is None:
     375                  parent.text = (parent.text or '') + self.tail
     376              else:
     377                  previous.tail = (previous.tail or '') + self.tail
     378          index = parent.index(self)
     379          parent[index:index+1] = self[:]
     380
     381 -    def find_rel_links(self, rel):
     382          """
     383          Find any links like ``<a rel="{rel}">...</a>``; returns a list of elements.
     384          """
     385          rel = rel.lower()
     386          return [el for el in _rel_links_xpath(self)
     387                  if el.get('rel').lower() == rel]
     388
     389 -    def find_class(self, class_name):
     390          """
     391          Find any elements with the given class name.
     392          """
     393          return _class_xpath(self, class_name=class_name)
     394
     395 -    def get_element_by_id(self, id, *default):
     396          """
     397          Get the first element in a document with the given id.  If none is
     398          found, return the default argument if provided or raise KeyError
     399          otherwise.
     400
     401          Note that there can be more than one element with the same id,
     402          and this isn't uncommon in HTML documents found in the wild.
     403          Browsers return only the first match, and this function does
     404          the same.
     405          """
     406          try:
     407              # FIXME: should this check for multiple matches?
     408              # browsers just return the first one
     409              return _id_xpath(self, id=id)[0]
     410          except IndexError:
     411              if default:
     412                  return default[0]
     413              else:
     414                  raise KeyError(id)
     415
     416 -    def text_content(self):
     417          """
     418          Return the text content of the tag (and the text in any children).
     419          """
     420          return _collect_string_content(self)
     421
     422 -    def cssselect(self, expr, translator='html'):
     423          """
     424          Run the CSS expression on this element and its children,
     425          returning a list of the results.
     426
     427          Equivalent to lxml.cssselect.CSSSelect(expr, translator='html')(self)
     428          -- note that pre-compiling the expression can provide a substantial
     429          speedup.
     430          """
     431          # Do the import here to make the dependency optional.
     432          from lxml.cssselect import CSSSelector
     433          return CSSSelector(expr, translator=translator)(self)
     434
     435      ########################################
     436      ## Link functions
     437      ########################################
     438
     439 -    def make_links_absolute(self, base_url=None, resolve_base_href=True,
     440                              handle_failures=None):
     441          """
     442          Make all links in the document absolute, given the
     443          ``base_url`` for the document (the full URL where the document
     444          came from), or if no ``base_url`` is given, then the ``.base_url``
     445          of the document.
     446
     447          If ``resolve_base_href`` is true, then any ``<base href>``
     448          tags in the document are used *and* removed from the document.
     449          If it is false then any such tag is ignored.
     450
     451          If ``handle_failures`` is None (default), a failure to process
     452          a URL will abort the processing.  If set to 'ignore', errors
     453          are ignored.  If set to 'discard', failing URLs will be removed.
     454          """
     455          if base_url is None:
     456              base_url = self.base_url
     457              if base_url is None:
     458                  raise TypeError(
     459                      "No base_url given, and the document has no base_url")
     460          if resolve_base_href:
     461              self.resolve_base_href()
     462
     463          if handle_failures == 'ignore':
     464              def link_repl(href):
     465                  try:
     466                      return urljoin(base_url, href)
     467                  except ValueError:
     468                      return href
     469          elif handle_failures == 'discard':
     470              def link_repl(href):
     471                  try:
     472                      return urljoin(base_url, href)
     473                  except ValueError:
     474                      return None
     475          elif handle_failures is None:
     476              def link_repl(href):
     477                  return urljoin(base_url, href)
     478          else:
     479              raise ValueError(
     480                  "unexpected value for handle_failures: %r" % handle_failures)
     481
     482          self.rewrite_links(link_repl)
     483
     484 -    def resolve_base_href(self, handle_failures=None):
     485          """
     486          Find any ``<base href>`` tag in the document, and apply its
     487          values to all links found in the document.  Also remove the
     488          tag once it has been applied.
     489
     490          If ``handle_failures`` is None (default), a failure to process
     491          a URL will abort the processing.  If set to 'ignore', errors
     492          are ignored.  If set to 'discard', failing URLs will be removed.
     493          """
     494          base_href = None
     495          basetags = self.xpath('//base[@href]|//x:base[@href]',
     496                                namespaces={'x': XHTML_NAMESPACE})
     497          for b in basetags:
     498              base_href = b.get('href')
     499              b.drop_tree()
     500          if not base_href:
     501              return
     502          self.make_links_absolute(base_href, resolve_base_href=False,
     503                                   handle_failures=handle_failures)
     504
     505 -    def iterlinks(self):
     506          """
     507          Yield (element, attribute, link, pos), where attribute may be None
     508          (indicating the link is in the text).  ``pos`` is the position
     509          where the link occurs; often 0, but sometimes something else in
     510          the case of links in stylesheets or style tags.
     511
     512          Note: <base href> is *not* taken into account in any way.  The
     513          link you get is exactly the link in the document.
     514
     515          Note: multiple links inside of a single text string or
     516          attribute value are returned in reversed order.  This makes it
     517          possible to replace or delete them from the text string value
     518          based on their reported text positions.  Otherwise, a
     519          modification at one text position can change the positions of
     520          links reported later on.
     521          """
     522          link_attrs = defs.link_attrs
     523          for el in self.iter(etree.Element):
     524              attribs = el.attrib
     525              tag = _nons(el.tag)
     526              if tag == 'object':
     527                  codebase = None
     528                  ## <object> tags have attributes that are relative to
     529                  ## codebase
     530                  if 'codebase' in attribs:
     531                      codebase = el.get('codebase')
     532                      yield (el, 'codebase', codebase, 0)
     533                  for attrib in ('classid', 'data'):
     534                      if attrib in attribs:
     535                          value = el.get(attrib)
     536                          if codebase is not None:
     537                              value = urljoin(codebase, value)
     538                          yield (el, attrib, value, 0)
     539                  if 'archive' in attribs:
     540                      for match in _archive_re.finditer(el.get('archive')):
     541                          value = match.group(0)
     542                          if codebase is not None:
     543                              value = urljoin(codebase, value)
     544                          yield (el, 'archive', value, match.start())
     545              else:
     546                  for attrib in link_attrs:
     547                      if attrib in attribs:
     548                          yield (el, attrib, attribs[attrib], 0)
     549              if tag == 'meta':
     550                  http_equiv = attribs.get('http-equiv', '').lower()
     551                  if http_equiv == 'refresh':
     552                      content = attribs.get('content', '')
     553                      match = _parse_meta_refresh_url(content)
     554                      url = (match.group('url') if match else content).strip()
     555                      # unexpected content means the redirect won't work, but we might
     556                      # as well be permissive and return the entire string.
     557                      if url:
     558                          url, pos = _unquote_match(
     559                              url, match.start('url') if match else content.find(url))
     560                          yield (el, 'content', url, pos)
     561              elif tag == 'param':
     562                  valuetype = el.get('valuetype') or ''
     563                  if valuetype.lower() == 'ref':
     564                      ## FIXME: while it's fine we *find* this link,
     565                      ## according to the spec we aren't supposed to
     566                      ## actually change the value, including resolving
     567                      ## it.  It can also still be a link, even if it
     568                      ## doesn't have a valuetype="ref" (which seems to be the norm)
     569                      ## http://www.w3.org/TR/html401/struct/objects.html#adef-valuetype
     570                      yield (el, 'value', el.get('value'), 0)
     571              elif tag == 'style' and el.text:
     572                  urls = [
     573                      # (start_pos, url)
     574                      _unquote_match(match.group(1), match.start(1))[::-1]
     575                      for match in _iter_css_urls(el.text)
     576                      ] + [
     577                      (match.start(1), match.group(1))
     578                      for match in _iter_css_imports(el.text)
     579                      ]
     580                  if urls:
     581                      # sort by start pos to bring both match sets back into order
     582                      # and reverse the list to report correct positions despite
     583                      # modifications
     584                      urls.sort(reverse=True)
     585                      for start, url in urls:
     586                          yield (el, None, url, start)
     587              if 'style' in attribs:
     588                  urls = list(_iter_css_urls(attribs['style']))
     589                  if urls:
     590                      # return in reversed order to simplify in-place modifications
     591                      for match in urls[::-1]:
     592                          url, start = _unquote_match(match.group(1), match.start(1))
     593                          yield (el, 'style', url, start)
     594
     595 -    def rewrite_links(self, link_repl_func, resolve_base_href=True,
     596                        base_href=None):
     597          """
     598          Rewrite all the links in the document.  For each link
     599          ``link_repl_func(link)`` will be called, and the return value
     600          will replace the old link.
     601
     602          Note that links may not be absolute (unless you first called
     603          ``make_links_absolute()``), and may be internal (e.g.,
     604          ``'#anchor'``).  They can also be values like
     605          ``'mailto:email'`` or ``'javascript:expr'``.
     606
     607          If you give ``base_href`` then all links passed to
     608          ``link_repl_func()`` will take that into account.
     609
     610          If the ``link_repl_func`` returns None, the attribute or
     611          tag text will be removed completely.
     612          """
     613          if base_href is not None:
     614              # FIXME: this can be done in one pass with a wrapper
     615              # around link_repl_func
     616              self.make_links_absolute(
     617                  base_href, resolve_base_href=resolve_base_href)
     618          elif resolve_base_href:
     619              self.resolve_base_href()
     620
     621          for el, attrib, link, pos in self.iterlinks():
     622              new_link = link_repl_func(link.strip())
     623              if new_link == link:
     624                  continue
     625              if new_link is None:
     626                  # Remove the attribute or element content
     627                  if attrib is None:
     628                      el.text = ''
     629                  else:
     630                      del el.attrib[attrib]
     631                  continue
     632
     633              if attrib is None:
     634                  new = el.text[:pos] + new_link + el.text[pos+len(link):]
     635                  el.text = new
     636              else:
     637                  cur = el.get(attrib)
     638                  if not pos and len(cur) == len(link):
     639                      new = new_link  # most common case
     640                  else:
     641                      new = cur[:pos] + new_link + cur[pos+len(link):]
     642                  el.set(attrib, new)
     643
     644
     645 -class _MethodFunc(object):
     646      """
     647      An object that represents a method on an element as a function;
     648      the function takes either an element or an HTML string.  It
     649      returns whatever the function normally returns, or if the function
     650      works in-place (and so returns None) it returns a serialized form
     651      of the resulting document.
     652      """
     653 -    def __init__(self, name, copy=False, source_class=HtmlMixin):
     654          self.name = name
     655          self.copy = copy
     656          self.__doc__ = getattr(source_class, self.name).__doc__
     657 -    def __call__(self, doc, *args, **kw):
     658          result_type = type(doc)
     659          if isinstance(doc, basestring):
     660              if 'copy' in kw:
     661                  raise TypeError(
     662                      "The keyword 'copy' can only be used with element inputs to %s, not a string input" % self.name)
     663              doc = fromstring(doc, **kw)
     664          else:
     665              if 'copy' in kw:
     666                  make_a_copy = kw.pop('copy')
     667              else:
     668                  make_a_copy = self.copy
     669              if make_a_copy:
     670                  doc = copy.deepcopy(doc)
     671          meth = getattr(doc, self.name)
     672          result = meth(*args, **kw)
     673          # FIXME: this None test is a bit sloppy
     674          if result is None:
     675              # Then return what we got in
     676              return _transform_result(result_type, doc)
     677          else:
     678              return result
     679
     680
     681  find_rel_links = _MethodFunc('find_rel_links', copy=False)
     682  find_class = _MethodFunc('find_class', copy=False)
     683  make_links_absolute = _MethodFunc('make_links_absolute', copy=True)
     684  resolve_base_href = _MethodFunc('resolve_base_href', copy=True)
     685  iterlinks = _MethodFunc('iterlinks', copy=False)
     686  rewrite_links = _MethodFunc('rewrite_links', copy=True)
     687
     688
     689 -class HtmlComment(etree.CommentBase, HtmlMixin):
     690      pass
     691
     692
     693 -class HtmlElement(etree.ElementBase, HtmlMixin):
     694      # Override etree.ElementBase.cssselect() and set(), despite the MRO (FIXME: change base order?)
     695      cssselect = HtmlMixin.cssselect
     696      set = HtmlMixin.set
     697
     698
     699 -class HtmlProcessingInstruction(etree.PIBase, HtmlMixin):
     700      pass
     701
     702
     703 -class HtmlEntity(etree.EntityBase, HtmlMixin):
     704      pass
     705
     706
     707 -class HtmlElementClassLookup(etree.CustomElementClassLookup):
     708      """A lookup scheme for HTML Element classes.
     709
     710      To create a lookup instance with different Element classes, pass a tag
     711      name mapping of Element classes in the ``classes`` keyword argument and/or
     712      a tag name mapping of Mixin classes in the ``mixins`` keyword argument.
     713      The special key '*' denotes a Mixin class that should be mixed into all
     714      Element classes.
     715      """
     716      _default_element_classes = {}
     717
     718 -    def __init__(self, classes=None, mixins=None):
     719          etree.CustomElementClassLookup.__init__(self)
     720          if classes is None:
     721              classes = self._default_element_classes.copy()
     722          if mixins:
     723              mixers = {}
     724              for name, value in mixins:
     725                  if name == '*':
     726                      for n in classes.keys():
     727                          mixers.setdefault(n, []).append(value)
     728                  else:
     729                      mixers.setdefault(name, []).append(value)
     730              for name, mix_bases in mixers.items():
     731                  cur = classes.get(name, HtmlElement)
     732                  bases = tuple(mix_bases + [cur])
     733                  classes[name] = type(cur.__name__, bases, {})
     734          self._element_classes = classes
     735
     736 -    def lookup(self, node_type, document, namespace, name):
     737          if node_type == 'element':
     738              return self._element_classes.get(name.lower(), HtmlElement)
     739          elif node_type == 'comment':
     740              return HtmlComment
     741          elif node_type == 'PI':
     742              return HtmlProcessingInstruction
     743          elif node_type == 'entity':
     744              return HtmlEntity
     745          # Otherwise normal lookup
     746          return None
     747
     748
     749  ################################################################################
     750  # parsing
     751  ################################################################################
     752
     753  _looks_like_full_html_unicode = re.compile(
     754      unicode(r'^\s*<(?:html|!doctype)'), re.I).match
     755  _looks_like_full_html_bytes = re.compile(
     756      r'^\s*<(?:html|!doctype)'.encode('ascii'), re.I).match
     757
     758
     759 -def document_fromstring(html, parser=None, ensure_head_body=False, **kw):
     760      if parser is None:
     761          parser = html_parser
     762      value = etree.fromstring(html, parser, **kw)
     763      if value is None:
     764          raise etree.ParserError(
     765              "Document is empty")
     766      if ensure_head_body and value.find('head') is None:
     767          value.insert(0, Element('head'))
     768      if ensure_head_body and value.find('body') is None:
     769          value.append(Element('body'))
     770      return value
     771
     772
     773 -def fragments_fromstring(html, no_leading_text=False, base_url=None,
     774                           parser=None, **kw):
     775      """Parses several HTML elements, returning a list of elements.
     776
     777      The first item in the list may be a string.
     778      If no_leading_text is true, then it will be an error if there is
     779      leading text, and it will always be a list of only elements.
     780
     781      base_url will set the document's base_url attribute
     782      (and the tree's docinfo.URL).
     783      """
     784      if parser is None:
     785          parser = html_parser
     786      # FIXME: check what happens when you give html with a body, head, etc.
     787      if isinstance(html, bytes):
     788          if not _looks_like_full_html_bytes(html):
     789              # can't use %-formatting in early Py3 versions
     790              html = ('<html><body>'.encode('ascii') + html +
     791                      '</body></html>'.encode('ascii'))
     792      else:
     793          if not _looks_like_full_html_unicode(html):
     794              html = '<html><body>%s</body></html>' % html
     795      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     796      assert _nons(doc.tag) == 'html'
     797      bodies = [e for e in doc if _nons(e.tag) == 'body']
     798      assert len(bodies) == 1, ("too many bodies: %r in %r" % (bodies, html))
     799      body = bodies[0]
     800      elements = []
     801      if no_leading_text and body.text and body.text.strip():
     802          raise etree.ParserError(
     803              "There is leading text: %r" % body.text)
     804      if body.text and body.text.strip():
     805          elements.append(body.text)
     806      elements.extend(body)
     807      # FIXME: removing the reference to the parent artificial document
     808      # would be nice
     809      return elements
     810
     811
     812 -def fragment_fromstring(html, create_parent=False, base_url=None,
     813                          parser=None, **kw):
     814      """
     815      Parses a single HTML element; it is an error if there is more than
     816      one element, or if anything but whitespace precedes or follows the
     817      element.
     818
     819      If ``create_parent`` is true (or is a tag name) then a parent node
     820      will be created to encapsulate the HTML in a single element.  In this
     821      case, leading or trailing text is also allowed, as are multiple elements
     822      as result of the parsing.
     823
     824      Passing a ``base_url`` will set the document's ``base_url`` attribute
     825      (and the tree's docinfo.URL).
     826      """
     827      if parser is None:
     828          parser = html_parser
     829
     830      accept_leading_text = bool(create_parent)
     831
     832      elements = fragments_fromstring(
     833          html, parser=parser, no_leading_text=not accept_leading_text,
     834          base_url=base_url, **kw)
     835
     836      if create_parent:
     837          if not isinstance(create_parent, basestring):
     838              create_parent = 'div'
     839          new_root = Element(create_parent)
     840          if elements:
     841              if isinstance(elements[0], basestring):
     842                  new_root.text = elements[0]
     843                  del elements[0]
     844              new_root.extend(elements)
     845          return new_root
     846
     847      if not elements:
     848          raise etree.ParserError('No elements found')
     849      if len(elements) > 1:
     850          raise etree.ParserError(
     851              "Multiple elements found (%s)"
     852              % ', '.join([_element_name(e) for e in elements]))
     853      el = elements[0]
     854      if el.tail and el.tail.strip():
     855          raise etree.ParserError(
     856              "Element followed by text: %r" % el.tail)
     857      el.tail = None
     858      return el
     859
     860
     861 -def fromstring(html, base_url=None, parser=None, **kw):
     862      """
     863      Parse the html, returning a single element/document.
     864
     865      This tries to minimally parse the chunk of text, without knowing if it
     866      is a fragment or a document.
     867
     868      base_url will set the document's base_url attribute (and the tree's docinfo.URL)
     869      """
     870      if parser is None:
     871          parser = html_parser
     872      if isinstance(html, bytes):
     873          is_full_html = _looks_like_full_html_bytes(html)
     874      else:
     875          is_full_html = _looks_like_full_html_unicode(html)
     876      doc = document_fromstring(html, parser=parser, base_url=base_url, **kw)
     877      if is_full_html:
     878          return doc
     879      # otherwise, lets parse it out...
     880      bodies = doc.findall('body')
     881      if not bodies:
     882          bodies = doc.findall('{%s}body' % XHTML_NAMESPACE)
     883      if bodies:
     884          body = bodies[0]
     885          if len(bodies) > 1:
     886              # Somehow there are multiple bodies, which is bad, but just
     887              # smash them into one body
     888              for other_body in bodies[1:]:
     889                  if other_body.text:
     890                      if len(body):
     891                          body[-1].tail = (body[-1].tail or '') + other_body.text
     892                      else:
     893                          body.text = (body.text or '') + other_body.text
     894                  body.extend(other_body)
     895                  # We'll ignore tail
     896                  # I guess we are ignoring attributes too
     897                  other_body.drop_tree()
     898      else:
     899          body = None
     900      heads = doc.findall('head')
     901      if not heads:
     902          heads = doc.findall('{%s}head' % XHTML_NAMESPACE)
     903      if heads:
     904          # Well, we have some sort of structure, so lets keep it all
     905          head = heads[0]
     906          if len(heads) > 1:
     907              for other_head in heads[1:]:
     908                  head.extend(other_head)
     909                  # We don't care about text or tail in a head
     910                  other_head.drop_tree()
     911          return doc
     912      if body is None:
     913          return doc
     914      if (len(body) == 1 and (not body.text or not body.text.strip())
     915          and (not body[-1].tail or not body[-1].tail.strip())):
     916          # The body has just one element, so it was probably a single
     917          # element passed in
     918          return body[0]
     919      # Now we have a body which represents a bunch of tags which have the
     920      # content that was passed in.  We will create a fake container, which
     921      # is the body tag, except <body> implies too much structure.
     922      if _contains_block_level_tag(body):
     923          body.tag = 'div'
     924      else:
     925          body.tag = 'span'
     926      return body
     927
     928
     929 -def parse(filename_or_url, parser=None, base_url=None, **kw):
     930      """
     931      Parse a filename, URL, or file-like object into an HTML document
     932      tree.  Note: this returns a tree, not an element.  Use
     933      ``parse(...).getroot()`` to get the document root.
     934
     935      You can override the base URL with the ``base_url`` keyword.  This
     936      is most useful when parsing from a file-like object.
     937      """
     938      if parser is None:
     939          parser = html_parser
     940      return etree.parse(filename_or_url, parser, base_url=base_url, **kw)
     941
     942
     943 -def _contains_block_level_tag(el):
     944      # FIXME: I could do this with XPath, but would that just be
     945      # unnecessarily slow?
     946      for el in el.iter(etree.Element):
     947          if _nons(el.tag) in defs.block_tags:
     948              return True
     949      return False
     950
     951
     952 -def _element_name(el):
     953      if isinstance(el, etree.CommentBase):
     954          return 'comment'
     955      elif isinstance(el, basestring):
     956          return 'string'
     957      else:
     958          return _nons(el.tag)
     959
     960
     961  ################################################################################
     962  # form handling
     963  ################################################################################
     964
     965 -class FormElement(HtmlElement):
     966      """
     967      Represents a <form> element.
     968      """
     969
     970      @property
     971 -    def inputs(self):
     972          """
     973          Returns an accessor for all the input elements in the form.
     974
     975          See `InputGetter` for more information about the object.
     976          """
     977          return InputGetter(self)
     978
     979      @property
     980 -    def fields(self):
     981          """
     982          Dictionary-like object that represents all the fields in this
     983          form.  You can set values in this dictionary to effect the
     984          form.
     985          """
     986          return FieldsDict(self.inputs)
     987
     988      @fields.setter
     989 -    def fields(self, value):
     990          fields = self.fields
     991          prev_keys = fields.keys()
     992          for key, value in value.items():
     993              if key in prev_keys:
     994                  prev_keys.remove(key)
     995              fields[key] = value
     996          for key in prev_keys:
     997              if key is None:
     998                  # Case of an unnamed input; these aren't really
     999                  # expressed in form_values() anyway.
    1000                  continue
    1001              fields[key] = None
    1002
    1003 -    def _name(self):
    1004          if self.get('name'):
    1005              return self.get('name')
    1006          elif self.get('id'):
    1007              return '#' + self.get('id')
    1008          iter_tags = self.body.iter
    1009          forms = list(iter_tags('form'))
    1010          if not forms:
    1011              forms = list(iter_tags('{%s}form' % XHTML_NAMESPACE))
    1012          return str(forms.index(self))
    1013
    1014 -    def form_values(self):
    1015          """
    1016          Return a list of tuples of the field values for the form.
    1017          This is suitable to be passed to ``urllib.urlencode()``.
    1018          """
    1019          results = []
    1020          for el in self.inputs:
    1021              name = el.name
    1022              if not name or 'disabled' in el.attrib:
    1023                  continue
    1024              tag = _nons(el.tag)
    1025              if tag == 'textarea':
    1026                  results.append((name, el.value))
    1027              elif tag == 'select':
    1028                  value = el.value
    1029                  if el.multiple:
    1030                      for v in value:
    1031                          results.append((name, v))
    1032                  elif value is not None:
    1033                      results.append((name, el.value))
    1034              else:
    1035                  assert tag == 'input', (
    1036                      "Unexpected tag: %r" % el)
    1037                  if el.checkable and not el.checked:
    1038                      continue
    1039                  if el.type in ('submit', 'image', 'reset', 'file'):
    1040                      continue
    1041                  value = el.value
    1042                  if value is not None:
    1043                      results.append((name, el.value))
    1044          return results
    1045
    1046      @property
    1047 -    def action(self):
    1048          """
    1049          Get/set the form's ``action`` attribute.
    1050          """
    1051          base_url = self.base_url
    1052          action = self.get('action')
    1053          if base_url and action is not None:
    1054              return urljoin(base_url, action)
    1055          else:
    1056              return action
    1057
    1058      @action.setter
    1059 -    def action(self, value):
    1060          self.set('action', value)
    1061
    1062      @action.deleter
    1063 -    def action(self):
    1064          attrib = self.attrib
    1065          if 'action' in attrib:
    1066              del attrib['action']
    1067
    1068      @property
    1069 -    def method(self):
    1070          """
    1071          Get/set the form's method.  Always returns a capitalized
    1072          string, and defaults to ``'GET'``
    1073          """
    1074          return self.get('method', 'GET').upper()
    1075
    1076      @method.setter
    1077 -    def method(self, value):
    1078          self.set('method', value.upper())
    1079
    1080
    1081  HtmlElementClassLookup._default_element_classes['form'] = FormElement
    1082
    1083
    1084 -def submit_form(form, extra_values=None, open_http=None):
    1085      """
    1086      Helper function to submit a form.  Returns a file-like object, as from
    1087      ``urllib.urlopen()``.  This object also has a ``.geturl()`` function,
    1088      which shows the URL if there were any redirects.
    1089
    1090      You can use this like::
    1091
    1092          form = doc.forms[0]
    1093          form.inputs['foo'].value = 'bar' # etc
    1094          response = form.submit()
    1095          doc = parse(response)
    1096          doc.make_links_absolute(response.geturl())
    1097
    1098      To change the HTTP requester, pass a function as ``open_http`` keyword
    1099      argument that opens the URL for you.  The function must have the following
    1100      signature::
    1101
    1102          open_http(method, URL, values)
    1103
    1104      The action is one of 'GET' or 'POST', the URL is the target URL as a
    1105      string, and the values are a sequence of ``(name, value)`` tuples with the
    1106      form data.
    1107      """
    1108      values = form.form_values()
    1109      if extra_values:
    1110          if hasattr(extra_values, 'items'):
    1111              extra_values = extra_values.items()
    1112          values.extend(extra_values)
    1113      if open_http is None:
    1114          open_http = open_http_urllib
    1115      if form.action:
    1116          url = form.action
    1117      else:
    1118          url = form.base_url
    1119      return open_http(form.method, url, values)
    1120
    1121
    1122 -def open_http_urllib(method, url, values):
    1123      if not url:
    1124          raise ValueError("cannot submit, no URL provided")
    1125      ## FIXME: should test that it's not a relative URL or something
    1126      try:
    1127          from urllib import urlencode, urlopen
    1128      except ImportError: # Python 3
    1129          from urllib.request import urlopen
    1130          from urllib.parse import urlencode
    1131      if method == 'GET':
    1132          if '?' in url:
    1133              url += '&'
    1134          else:
    1135              url += '?'
    1136          url += urlencode(values)
    1137          data = None
    1138      else:
    1139          data = urlencode(values)
    1140          if not isinstance(data, bytes):
    1141              data = data.encode('ASCII')
    1142      return urlopen(url, data)
    1143
    1144
    1145 -class FieldsDict(MutableMapping):
    1146
    1147 -    def __init__(self, inputs):
    1148          self.inputs = inputs
    1149 -    def __getitem__(self, item):
    1150          return self.inputs[item].value
    1151 -    def __setitem__(self, item, value):
    1152          self.inputs[item].value = value
    1153 -    def __delitem__(self, item):
    1154          raise KeyError(
    1155              "You cannot remove keys from ElementDict")
    1156 -    def keys(self):
    1157          return self.inputs.keys()
    1158 -    def __contains__(self, item):
    1159          return item in self.inputs
    1160 -    def __iter__(self):
    1161          return iter(self.inputs.keys())
    1162 -    def __len__(self):
    1163          return len(self.inputs)
    1164
    1165 -    def __repr__(self):
    1166          return '<%s for form %s>' % (
    1167              self.__class__.__name__,
    1168              self.inputs.form._name())
    1169
    1170
    1171 -class InputGetter(object):
    1172
    1173      """
    1174      An accessor that represents all the input fields in a form.
    1175
    1176      You can get fields by name from this, with
    1177      ``form.inputs['field_name']``.  If there are a set of checkboxes
    1178      with the same name, they are returned as a list (a `CheckboxGroup`
    1179      which also allows value setting).  Radio inputs are handled
    1180      similarly.
    1181
    1182      You can also iterate over this to get all input elements.  This
    1183      won't return the same thing as if you get all the names, as
    1184      checkboxes and radio elements are returned individually.
    1185      """
    1186
    1187      _name_xpath = etree.XPath(".//*[@name = $name and (local-name(.) = 'select' or local-name(.) = 'input' or local-name(.) = 'textarea')]")
    1188      _all_xpath = etree.XPath(".//*[local-name() = 'select' or local-name() = 'input' or local-name() = 'textarea']")
    1189
    1190 -    def __init__(self, form):
    1191          self.form = form
    1192
    1193 -    def __repr__(self):
    1194          return '<%s for form %s>' % (
    1195              self.__class__.__name__,
    1196              self.form._name())
    1197
    1198      ## FIXME: there should be more methods, and it's unclear if this is
    1199      ## a dictionary-like object or list-like object
    1200
    1201 -    def __getitem__(self, name):
    1202          results = self._name_xpath(self.form, name=name)
    1203          if results:
    1204              type = results[0].get('type')
    1205              if type == 'radio' and len(results) > 1:
    1206                  group = RadioGroup(results)
    1207                  group.name = name
    1208                  return group
    1209              elif type == 'checkbox' and len(results) > 1:
    1210                  group = CheckboxGroup(results)
    1211                  group.name = name
    1212                  return group
    1213              else:
    1214                  # I don't like throwing away elements like this
    1215                  return results[0]
    1216          else:
    1217              raise KeyError(
    1218                  "No input element with the name %r" % name)
    1219
    1220 -    def __contains__(self, name):
    1221          results = self._name_xpath(self.form, name=name)
    1222          return bool(results)
    1223
    1224 -    def keys(self):
    1225          names = set()
    1226          for el in self:
    1227              names.add(el.name)
    1228          if None in names:
    1229              names.remove(None)
    1230          return list(names)
    1231
    1232 -    def __iter__(self):
    1233          ## FIXME: kind of dumb to turn a list into an iterator, only
    1234          ## to have it likely turned back into a list again :(
    1235          return iter(self._all_xpath(self.form))
    1236
    1237
    1238 -class InputMixin(object):
    1239      """
    1240      Mix-in for all input elements (input, select, and textarea)
    1241      """
    1242      @property
    1243 -    def name(self):
    1244          """
    1245          Get/set the name of the element
    1246          """
    1247          return self.get('name')
    1248
    1249      @name.setter
    1250 -    def name(self, value):
    1251          self.set('name', value)
    1252
    1253      @name.deleter
    1254 -    def name(self):
    1255          attrib = self.attrib
    1256          if 'name' in attrib:
    1257              del attrib['name']
    1258
    1259 -    def __repr__(self):
    1260          type_name = getattr(self, 'type', None)
    1261          if type_name:
    1262              type_name = ' type=%r' % type_name
    1263          else:
    1264              type_name = ''
    1265          return '<%s %x name=%r%s>' % (
    1266              self.__class__.__name__, id(self), self.name, type_name)
    1267
    1268
    1269 -class TextareaElement(InputMixin, HtmlElement):
    1270      """
    1271      ``<textarea>`` element.  You can get the name with ``.name`` and
    1272      get/set the value with ``.value``
    1273      """
    1274      @property
    1275 -    def value(self):
    1276          """
    1277          Get/set the value (which is the contents of this element)
    1278          """
    1279          content = self.text or ''
    1280          if self.tag.startswith("{%s}" % XHTML_NAMESPACE):
    1281              serialisation_method = 'xml'
    1282          else:
    1283              serialisation_method = 'html'
    1284          for el in self:
    1285              # it's rare that we actually get here, so let's not use ''.join()
    1286              content += etree.tostring(
    1287                  el, method=serialisation_method, encoding='unicode')
    1288          return content
    1289
    1290      @value.setter
    1291 -    def value(self, value):
    1292          del self[:]
    1293          self.text = value
    1294
    1295      @value.deleter
    1296 -    def value(self):
    1297          self.text = ''
    1298          del self[:]
    1299
    1300
    1301  HtmlElementClassLookup._default_element_classes['textarea'] = TextareaElement
    1302
    1303
    1304 -class SelectElement(InputMixin, HtmlElement):
    1305      """
    1306      ``<select>`` element.  You can get the name with ``.name``.
    1307
    1308      ``.value`` will be the value of the selected option, unless this
    1309      is a multi-select element (``<select multiple>``), in which case
    1310      it will be a set-like object.  In either case ``.value_options``
    1311      gives the possible values.
    1312
    1313      The boolean attribute ``.multiple`` shows if this is a
    1314      multi-select.
    1315      """
    1316      @property
    1317 -    def value(self):
    1318          """
    1319          Get/set the value of this select (the selected option).
    1320
    1321          If this is a multi-select, this is a set-like object that
    1322          represents all the selected options.
    1323          """
    1324          if self.multiple:
    1325              return MultipleSelectOptions(self)
    1326          options = _options_xpath(self)
    1327
    1328          try:
    1329              selected_option = next(el for el in reversed(options) if el.get('selected') is not None)
    1330          except StopIteration:
    1331              try:
    1332                  selected_option = next(el for el in options if el.get('disabled') is None)
    1333              except StopIteration:
    1334                  return None
    1335          value = selected_option.get('value')
    1336          if value is None:
    1337              value = (selected_option.text or '').strip()
    1338          return value
    1339
    1340      @value.setter
    1341 -    def value(self, value):
    1342          if self.multiple:
    1343              if isinstance(value, basestring):
    1344                  raise TypeError("You must pass in a sequence")
    1345              values = self.value
    1346              values.clear()
    1347              values.update(value)
    1348              return
    1349          checked_option = None
    1350          if value is not None:
    1351              for el in _options_xpath(self):
    1352                  opt_value = el.get('value')
    1353                  if opt_value is None:
    1354                      opt_value = (el.text or '').strip()
    1355                  if opt_value == value:
    1356                      checked_option = el
    1357                      break
    1358              else:
    1359                  raise ValueError(
    1360                      "There is no option with the value of %r" % value)
    1361          for el in _options_xpath(self):
    1362              if 'selected' in el.attrib:
    1363                  del el.attrib['selected']
    1364          if checked_option is not None:
    1365              checked_option.set('selected', '')
    1366
    1367      @value.deleter
    1368 -    def value(self):
    1369          # FIXME: should del be allowed at all?
    1370          if self.multiple:
    1371              self.value.clear()
    1372          else:
    1373              self.value = None
    1374
    1375      @property
    1376 -    def value_options(self):
    1377          """
    1378          All the possible values this select can have (the ``value``
    1379          attribute of all the ``<option>`` elements.
    1380          """
    1381          options = []
    1382          for el in _options_xpath(self):
    1383              value = el.get('value')
    1384              if value is None:
    1385                  value = (el.text or '').strip()
    1386              options.append(value)
    1387          return options
    1388
    1389      @property
    1390 -    def multiple(self):
    1391          """
    1392          Boolean attribute: is there a ``multiple`` attribute on this element.
    1393          """
    1394          return 'multiple' in self.attrib
    1395
    1396      @multiple.setter
    1397 -    def multiple(self, value):
    1398          if value:
    1399              self.set('multiple', '')
    1400          elif 'multiple' in self.attrib:
    1401              del self.attrib['multiple']
    1402
    1403
    1404  HtmlElementClassLookup._default_element_classes['select'] = SelectElement
    1405
    1406
    1407 -class MultipleSelectOptions(SetMixin):
    1408      """
    1409      Represents all the selected options in a ``<select multiple>`` element.
    1410
    1411      You can add to this set-like option to select an option, or remove
    1412      to unselect the option.
    1413      """
    1414
    1415 -    def __init__(self, select):
    1416          self.select = select
    1417
    1418      @property
    1419 -    def options(self):
    1420          """
    1421          Iterator of all the ``<option>`` elements.
    1422          """
    1423          return iter(_options_xpath(self.select))
    1424
    1425 -    def __iter__(self):
    1426          for option in self.options:
    1427              if 'selected' in option.attrib:
    1428                  opt_value = option.get('value')
    1429                  if opt_value is None:
    1430                      opt_value = (option.text or '').strip()
    1431                  yield opt_value
    1432
    1433 -    def add(self, item):
    1434          for option in self.options:
    1435              opt_value = option.get('value')
    1436              if opt_value is None:
    1437                  opt_value = (option.text or '').strip()
    1438              if opt_value == item:
    1439                  option.set('selected', '')
    1440                  break
    1441          else:
    1442              raise ValueError(
    1443                  "There is no option with the value %r" % item)
    1444
    1445 -    def remove(self, item):
    1446          for option in self.options:
    1447              opt_value = option.get('value')
    1448              if opt_value is None:
    1449                  opt_value = (option.text or '').strip()
    1450              if opt_value == item:
    1451                  if 'selected' in option.attrib:
    1452                      del option.attrib['selected']
    1453                  else:
    1454                      raise ValueError(
    1455                          "The option %r is not currently selected" % item)
    1456                  break
    1457          else:
    1458              raise ValueError(
    1459                  "There is not option with the value %r" % item)
    1460
    1461 -    def __repr__(self):
    1462          return '<%s {%s} for select name=%r>' % (
    1463              self.__class__.__name__,
    1464              ', '.join([repr(v) for v in self]),
    1465              self.select.name)
    1466
    1467
    1468 -class RadioGroup(list):
    1469      """
    1470      This object represents several ``<input type=radio>`` elements
    1471      that have the same name.
    1472
    1473      You can use this like a list, but also use the property
    1474      ``.value`` to check/uncheck inputs.  Also you can use
    1475      ``.value_options`` to get the possible values.
    1476      """
    1477      @property
    1478 -    def value(self):
    1479          """
    1480          Get/set the value, which checks the radio with that value (and
    1481          unchecks any other value).
    1482          """
    1483          for el in self:
    1484              if 'checked' in el.attrib:
    1485                  return el.get('value')
    1486          return None
    1487
    1488      @value.setter
    1489 -    def value(self, value):
    1490          checked_option = None
    1491          if value is not None:
    1492              for el in self:
    1493                  if el.get('value') == value:
    1494                      checked_option = el
    1495                      break
    1496              else:
    1497                  raise ValueError("There is no radio input with the value %r" % value)
    1498          for el in self:
    1499              if 'checked' in el.attrib:
    1500                  del el.attrib['checked']
    1501          if checked_option is not None:
    1502              checked_option.set('checked', '')
    1503
    1504      @value.deleter
    1505 -    def value(self):
    1506          self.value = None
    1507
    1508      @property
    1509 -    def value_options(self):
    1510          """
    1511          Returns a list of all the possible values.
    1512          """
    1513          return [el.get('value') for el in self]
    1514
    1515 -    def __repr__(self):
    1516          return '%s(%s)' % (
    1517              self.__class__.__name__,
    1518              list.__repr__(self))
    1519
    1520
    1521 -class CheckboxGroup(list):
    1522      """
    1523      Represents a group of checkboxes (``<input type=checkbox>``) that
    1524      have the same name.
    1525
    1526      In addition to using this like a list, the ``.value`` attribute
    1527      returns a set-like object that you can add to or remove from to
    1528      check and uncheck checkboxes.  You can also use ``.value_options``
    1529      to get the possible values.
    1530      """
    1531      @property
    1532 -    def value(self):
    1533          """
    1534          Return a set-like object that can be modified to check or
    1535          uncheck individual checkboxes according to their value.
    1536          """
    1537          return CheckboxValues(self)
    1538
    1539      @value.setter
    1540 -    def value(self, value):
    1541          values = self.value
    1542          values.clear()
    1543          if not hasattr(value, '__iter__'):
    1544              raise ValueError(
    1545                  "A CheckboxGroup (name=%r) must be set to a sequence (not %r)"
    1546                  % (self[0].name, value))
    1547          values.update(value)
    1548
    1549      @value.deleter
    1550 -    def value(self):
    1551          self.value.clear()
    1552
    1553      @property
    1554 -    def value_options(self):
    1555          """
    1556          Returns a list of all the possible values.
    1557          """
    1558          return [el.get('value') for el in self]
    1559
    1560 -    def __repr__(self):
    1561          return '%s(%s)' % (
    1562              self.__class__.__name__, list.__repr__(self))
    1563
    1564
    1565 -class CheckboxValues(SetMixin):
    1566      """
    1567      Represents the values of the checked checkboxes in a group of
    1568      checkboxes with the same name.
    1569      """
    1570
    1571 -    def __init__(self, group):
    1572          self.group = group
    1573
    1574 -    def __iter__(self):
    1575          return iter([
    1576              el.get('value')
    1577              for el in self.group
    1578              if 'checked' in el.attrib])
    1579
    1580 -    def add(self, value):
    1581          for el in self.group:
    1582              if el.get('value') == value:
    1583                  el.set('checked', '')
    1584                  break
    1585          else:
    1586              raise KeyError("No checkbox with value %r" % value)
    1587
    1588 -    def remove(self, value):
    1589          for el in self.group:
    1590              if el.get('value') == value:
    1591                  if 'checked' in el.attrib:
    1592                      del el.attrib['checked']
    1593                  else:
    1594                      raise KeyError(
    1595                          "The checkbox with value %r was already unchecked" % value)
    1596                  break
    1597          else:
    1598              raise KeyError(
    1599                  "No checkbox with value %r" % value)
    1600
    1601 -    def __repr__(self):
    1602          return '<%s {%s} for checkboxes name=%r>' % (
    1603              self.__class__.__name__,
    1604              ', '.join([repr(v) for v in self]),
    1605              self.group.name)
    1606
    1607
    1608 -class InputElement(InputMixin, HtmlElement):
    1609      """
    1610      Represents an ``<input>`` element.
    1611
    1612      You can get the type with ``.type`` (which is lower-cased and
    1613      defaults to ``'text'``).
    1614
    1615      Also you can get and set the value with ``.value``
    1616
    1617      Checkboxes and radios have the attribute ``input.checkable ==
    1618      True`` (for all others it is false) and a boolean attribute
    1619      ``.checked``.
    1620
    1621      """
    1622
    1623      ## FIXME: I'm a little uncomfortable with the use of .checked
    1624      @property
    1625 -    def value(self):
    1626          """
    1627          Get/set the value of this element, using the ``value`` attribute.
    1628
    1629          Also, if this is a checkbox and it has no value, this defaults
    1630          to ``'on'``.  If it is a checkbox or radio that is not
    1631          checked, this returns None.
    1632          """
    1633          if self.checkable:
    1634              if self.checked:
    1635                  return self.get('value') or 'on'
    1636              else:
    1637                  return None
    1638          return self.get('value')
    1639
    1640      @value.setter
    1641 -    def value(self, value):
    1642          if self.checkable:
    1643              if not value:
    1644                  self.checked = False
    1645              else:
    1646                  self.checked = True
    1647                  if isinstance(value, basestring):
    1648                      self.set('value', value)
    1649          else:
    1650              self.set('value', value)
    1651
    1652      @value.deleter
    1653 -    def value(self):
    1654          if self.checkable:
    1655              self.checked = False
    1656          else:
    1657              if 'value' in self.attrib:
    1658                  del self.attrib['value']
    1659
    1660      @property
    1661 -    def type(self):
    1662          """
    1663          Return the type of this element (using the type attribute).
    1664          """
    1665          return self.get('type', 'text').lower()
    1666
    1667      @type.setter
    1668 -    def type(self, value):
    1669          self.set('type', value)
    1670
    1671      @property
    1672 -    def checkable(self):
    1673          """
    1674          Boolean: can this element be checked?
    1675          """
    1676          return self.type in ('checkbox', 'radio')
    1677
    1678      @property
    1679 -    def checked(self):
    1680          """
    1681          Boolean attribute to get/set the presence of the ``checked``
    1682          attribute.
    1683
    1684          You can only use this on checkable input types.
    1685          """
    1686          if not self.checkable:
    1687              raise AttributeError('Not a checkable input type')
    1688          return 'checked' in self.attrib
    1689
    1690      @checked.setter
    1691 -    def checked(self, value):
    1692          if not self.checkable:
    1693              raise AttributeError('Not a checkable input type')
    1694          if value:
    1695              self.set('checked', '')
    1696          else:
    1697              attrib = self.attrib
    1698              if 'checked' in attrib:
    1699                  del attrib['checked']
    1700
    1701
    1702  HtmlElementClassLookup._default_element_classes['input'] = InputElement
    1703
    1704
    1705 -class LabelElement(HtmlElement):
    1706      """
    1707      Represents a ``<label>`` element.
    1708
    1709      Label elements are linked to other elements with their ``for``
    1710      attribute.  You can access this element with ``label.for_element``.
    1711      """
    1712      @property
    1713 -    def for_element(self):
    1714          """
    1715          Get/set the element this label points to.  Return None if it
    1716          can't be found.
    1717          """
    1718          id = self.get('for')
    1719          if not id:
    1720              return None
    1721          return self.body.get_element_by_id(id)
    1722
    1723      @for_element.setter
    1724 -    def for_element(self, other):
    1725          id = other.get('id')
    1726          if not id:
    1727              raise TypeError(
    1728                  "Element %r has no id attribute" % other)
    1729          self.set('for', id)
    1730
    1731      @for_element.deleter
    1732 -    def for_element(self):
    1733          attrib = self.attrib
    1734          if 'id' in attrib:
    1735              del attrib['id']
    1736
    1737
    1738  HtmlElementClassLookup._default_element_classes['label'] = LabelElement
    1739
    1740
    1741  ############################################################
    1742  ## Serialization
    1743  ############################################################
    1744
    1745 -def html_to_xhtml(html):
    1746      """Convert all tags in an HTML tree to XHTML by moving them to the
    1747      XHTML namespace.
    1748      """
    1749      try:
    1750          html = html.getroot()
    1751      except AttributeError:
    1752          pass
    1753      prefix = "{%s}" % XHTML_NAMESPACE
    1754      for el in html.iter(etree.Element):
    1755          tag = el.tag
    1756          if tag[0] != '{':
    1757              el.tag = prefix + tag
    1758
    1759
    1760 -def xhtml_to_html(xhtml):
    1761      """Convert all tags in an XHTML tree to HTML by removing their
    1762      XHTML namespace.
    1763      """
    1764      try:
    1765          xhtml = xhtml.getroot()
    1766      except AttributeError:
    1767          pass
    1768      prefix = "{%s}" % XHTML_NAMESPACE
    1769      prefix_len = len(prefix)
    1770      for el in xhtml.iter(prefix + "*"):
    1771          el.tag = el.tag[prefix_len:]
    1772
    1773
    1774  # This isn't a general match, but it's a match for what libxml2
    1775  # specifically serialises:
    1776  __str_replace_meta_content_type = re.compile(
    1777      r'<meta http-equiv="Content-Type"[^>]*>').sub
    1778  __bytes_replace_meta_content_type = re.compile(
    1779      r'<meta http-equiv="Content-Type"[^>]*>'.encode('ASCII')).sub
    1780
    1781
    1782 -def tostring(doc, pretty_print=False, include_meta_content_type=False,
    1783               encoding=None, method="html", with_tail=True, doctype=None):
    1784      """Return an HTML string representation of the document.
    1785
    1786      Note: if include_meta_content_type is true this will create a
    1787      ``<meta http-equiv="Content-Type" ...>`` tag in the head;
    1788      regardless of the value of include_meta_content_type any existing
    1789      ``<meta http-equiv="Content-Type" ...>`` tag will be removed
    1790
    1791      The ``encoding`` argument controls the output encoding (defauts to
    1792      ASCII, with &#...; character references for any characters outside
    1793      of ASCII).  Note that you can pass the name ``'unicode'`` as
    1794      ``encoding`` argument to serialise to a Unicode string.
    1795
    1796      The ``method`` argument defines the output method.  It defaults to
    1797      'html', but can also be 'xml' for xhtml output, or 'text' to
    1798      serialise to plain text without markup.
    1799
    1800      To leave out the tail text of the top-level element that is being
    1801      serialised, pass ``with_tail=False``.
    1802
    1803      The ``doctype`` option allows passing in a plain string that will
    1804      be serialised before the XML tree.  Note that passing in non
    1805      well-formed content here will make the XML output non well-formed.
    1806      Also, an existing doctype in the document tree will not be removed
    1807      when serialising an ElementTree instance.
    1808
    1809      Example::
    1810
    1811          >>> from lxml import html
    1812          >>> root = html.fragment_fromstring('<p>Hello<br>world!</p>')
    1813
    1814          >>> html.tostring(root)
    1815          b'<p>Hello<br>world!</p>'
    1816          >>> html.tostring(root, method='html')
    1817          b'<p>Hello<br>world!</p>'
    1818
    1819          >>> html.tostring(root, method='xml')
    1820          b'<p>Hello<br/>world!</p>'
    1821
    1822          >>> html.tostring(root, method='text')
    1823          b'Helloworld!'
    1824
    1825          >>> html.tostring(root, method='text', encoding='unicode')
    1826          u'Helloworld!'
    1827
    1828          >>> root = html.fragment_fromstring('<div><p>Hello<br>world!</p>TAIL</div>')
    1829          >>> html.tostring(root[0], method='text', encoding='unicode')
    1830          u'Helloworld!TAIL'
    1831
    1832          >>> html.tostring(root[0], method='text', encoding='unicode', with_tail=False)
    1833          u'Helloworld!'
    1834
    1835          >>> doc = html.document_fromstring('<p>Hello<br>world!</p>')
    1836          >>> html.tostring(doc, method='html', encoding='unicode')
    1837          u'<html><body><p>Hello<br>world!</p></body></html>'
    1838
    1839          >>> print(html.tostring(doc, method='html', encoding='unicode',
    1840          ...          doctype='<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"'
    1841          ...                  ' "http://www.w3.org/TR/html4/strict.dtd">'))
    1842          <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
    1843          <html><body><p>Hello<br>world!</p></body></html>
    1844      """
    1845      html = etree.tostring(doc, method=method, pretty_print=pretty_print,
    1846                            encoding=encoding, with_tail=with_tail,
    1847                            doctype=doctype)
    1848      if method == 'html' and not include_meta_content_type:
    1849          if isinstance(html, str):
    1850              html = __str_replace_meta_content_type('', html)
    1851          else:
    1852              html = __bytes_replace_meta_content_type(bytes(), html)
    1853      return html
    1854
    1855
    1856  tostring.__doc__ = __fix_docstring(tostring.__doc__)
    1857
    1858
    1859 -def open_in_browser(doc, encoding=None):
    1860      """
    1861      Open the HTML document in a web browser, saving it to a temporary
    1862      file to open it.  Note that this does not delete the file after
    1863      use.  This is mainly meant for debugging.
    1864      """
    1865      import os
    1866      import webbrowser
    1867      import tempfile
    1868      if not isinstance(doc, etree._ElementTree):
    1869          doc = etree.ElementTree(doc)
    1870      handle, fn = tempfile.mkstemp(suffix='.html')
    1871      f = os.fdopen(handle, 'wb')
    1872      try:
    1873          doc.write(f, method="html", encoding=encoding or doc.docinfo.encoding or "UTF-8")
    1874      finally:
    1875          # we leak the file itself here, but we should at least close it
    1876          f.close()
    1877      url = 'file://' + fn.replace(os.path.sep, '/')
    1878      print(url)
    1879      webbrowser.open(url)
    1880
    1881
    1882  ################################################################################
    1883  # configure Element class lookup
    1884  ################################################################################
    1885
    1886 -class HTMLParser(etree.HTMLParser):
    1887      """An HTML parser that is configured to return lxml.html Element
    1888      objects.
    1889      """
    1890 -    def __init__(self, **kwargs):
    1891          super(HTMLParser, self).__init__(**kwargs)
    1892          self.set_element_class_lookup(HtmlElementClassLookup())
    1893
    1894
    1895 -class XHTMLParser(etree.XMLParser):
    1896      """An XML parser that is configured to return lxml.html Element
    1897      objects.
    1898
    1899      Note that this parser is not really XHTML aware unless you let it
    1900      load a DTD that declares the HTML entities.  To do this, make sure
    1901      you have the XHTML DTDs installed in your catalogs, and create the
    1902      parser like this::
    1903
    1904          >>> parser = XHTMLParser(load_dtd=True)
    1905
    1906      If you additionally want to validate the document, use this::
    1907
    1908          >>> parser = XHTMLParser(dtd_validation=True)
    1909
    1910      For catalog support, see http://www.xmlsoft.org/catalog.html.
    1911      """
    1912 -    def __init__(self, **kwargs):
    1913          super(XHTMLParser, self).__init__(**kwargs)
    1914          self.set_element_class_lookup(HtmlElementClassLookup())
    1915
    1916
    1917 -def Element(*args, **kw):
    1918      """Create a new HTML Element.
    1919
    1920      This can also be used for XHTML documents.
    1921      """
    1922      v = html_parser.makeelement(*args, **kw)
    1923      return v
    1924
    1925
    1926  html_parser = HTMLParser()
    1927  xhtml_parser = XHTMLParser()
    1928