(* Note:

     This file is taken 'as-is' from the pxp distribution, whose
     copyright notice is reproduced below.

     PXP can be found at:
     http://www.ocaml-programming.de/programming/pxp.html

   - Jerome

Copyright 1999 by Gerd Stolpmann

The package PXP is copyright by Gerd Stolpmann. 

Permission is hereby granted, free of charge, to any person obtaining
a copy of this document and the PXP software (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.

The Software is provided ``as is'', without warranty of any kind, express
or implied, including but not limited to the warranties of
merchantability, fitness for a particular purpose and noninfringement.
In no event shall Gerd Stolpmann be liable for any claim, damages or
other liability, whether in an action of contract, tort or otherwise,
arising from, out of or in connection with the Software or the use or
other dealings in the software.

*)

(* $Id: pxp_lex_defs_drv_utf8.def,v 1.1 2002/10/27 20:02:52 simeon Exp $
 * ----------------------------------------------------------------------
 *
 *)

let ws = [ ' ' '\t' '\r' '\n' ]

let ascii_digit = ['0'-'9']

let ascii_hexdigit = ['0'-'9' 'a'-'h' 'A'-'H']

let namechar = letter | extender | digit | '.' | ':' | '_' | '-' | combiningChar

let name = ( letter | '_' | ':' ) namechar*

let nmtoken = namechar+

(**** Added for Galax - Jerome *)

let whitespace = ws*
let whitespaceplus = ws+

let nmstart  = letter | '_'
let nmchar   = letter | extender | digit | '.' | '-' | '_' | combiningChar
 
let ncname   = nmstart nmchar*

let qname    = ncname (':' ncname)?

(**** Until here - Jerome *)

(* Valid characters are:
 * #9, #10, #13, #32-#xD7FF, #xE000-#xFFFD, #x10000-#x10FFFF
 *
 * #xD7FF as UTF-8 sequence:
 * 1110xxxx 10xxxxxx 10xxxxxx
 * 1110...D 10...7.. 10.F...F  = ED 9F BF
 *
 * #xE000 as UTF-8 sequence:
 * 1110xxxx 10xxxxxx 10xxxxxx
 * 1110...E 10...0.. 10.0...0  = EE 80 80
 *
 * UTF-8 sequence CF BE BF as character:
 * 1110xxxx 10xxxxxx 10xxxxxx
 * 1110...F 10111110 10111111  = #FFBF
 *
 * #xFFFD as UTF-8 sequence:
 * 1110xxxx 10xxxxxx 10xxxxxx
 * 1110...F 10...F.. 10.F...D  = EF BF BD
 *
 * #x010000 as UTF-8 sequence:
 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
 * 111100.. 10.1...0 10...0.. 10.0...0 = F0 90 80 80
 *
 * #x10FFFF as UTF-8 sequence:
 * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
 * 111101.. 10.0...F 10...F.. 10.F...F = F4 8F BF BF
 *)


let non_ascii_character = 
  ['\192'-'\223'] ['\128'-'\191']                     (* #x80-#x7FF *)
| ['\224'-'\236'] ['\128'-'\191'] ['\128'-'\191']     (* #x800-#xCFFF *)
| '\237'          ['\128'-'\159'] ['\128'-'\191']     (* #xD000-#xD7FF *)
| '\238'          ['\128'-'\191'] ['\128'-'\191']     (* #xE000-#xEFFF *)
| '\239'          ['\128'-'\190'] ['\128'-'\191']     (* #xF000-#xFFBF *)
| '\239'          '\191'          ['\128'-'\189']     (* #xFFC0-#xFFFD *)
| '\240'          ['\144'-'\191'] ['\128'-'\191'] ['\128'-'\191']     
                                                      (* #x010000-#x03FFFF *)
| ['\241'-'\243'] ['\128'-'\191'] ['\128'-'\191'] ['\128'-'\191'] 
                                                      (* #x040000-#x0FFFFF *)
| '\244'          ['\128'-'\143'] ['\128'-'\191'] ['\128'-'\191'] 
                                                      (* #x100000-#10FFFFF *)

let character =
  [ '\009' '\010' '\013' '\032'-'\127' ]
| non_ascii_character


let character_except_question_mark =                    (* '?' = '\063' *)
  [ '\009' '\010' '\013' '\032'-'\062' '\064'-'\127' ]
| non_ascii_character


let character_except_right_angle_bracket =              (* '>' = '\062' *)
  [ '\009' '\010' '\013' '\032'-'\061' '\063'-'\127' ]
| non_ascii_character


let character_except_minus =                            (* '-' = '\045' *)
  [ '\009' '\010' '\013' '\032'-'\044' '\046'-'\127' ]
| non_ascii_character


let character_except_quot =                             (* '"' = '\034' *)
  [ '\009' '\010' '\013' '\032'-'\033' '\035'-'\255' ]
| non_ascii_character


let character_except_apos =                             (* '\'' = '\039' *)
  [ '\009' '\010' '\013' '\032'-'\038' '\040'-'\255' ]
| non_ascii_character


let pi_string = character_except_question_mark* 
                ( '?' character_except_right_angle_bracket 
                      character_except_question_mark* )* 
                '?'?


let comment_string = character_except_minus* 
                     ('-' character_except_minus+ )*


let normal_character = 
  (* Character except '&' = '\038', '<' = '\060', ']' = '\093', and CR LF *)
  [ '\009' '\032'-'\037' '\039'-'\059' '\061'-'\092' '\094'-'\127' ]
| non_ascii_character


let character_except_rbracket =                               (* ']' = '\093' *)
  [ '\009' '\010' '\013' '\032'-'\092' '\094'-'\127' ]
| non_ascii_character


let character_except_rbracket_rangle =          (* ']' = '\093', '>' = '\062' *)
  [ '\009' '\010' '\013' '\032'-'\061' '\063'-'\092' '\094'-'\127' ]
| non_ascii_character


let cdata_string = 
  character_except_rbracket*
  ( "]" character_except_rbracket+ |
    "]]" ']'* character_except_rbracket_rangle character_except_rbracket*
  )*
  ']'*


let printable_character_except_amp_lt =
  (* '&' = '\038', '<' = '\060' *)
  [ '\032'-'\037' '\039'-'\059' '\061'-'\127']
| non_ascii_character


let printable_character_except_amp_percent =
  (* '%' = '\037', '&' = '\038' *)
  [ '\032'-'\036' '\039'-'\127']
| non_ascii_character


let character_except_special =
  (* '<'=060, ']'=093, '"'=034, '\''=039 *)
  [ '\009' '\010' '\013' '\032'-'\033' '\035'-'\038' '\040'-'\059' 
                         '\061'-'\092' '\094'-'\127' ]
| non_ascii_character

  

(* ======================================================================
 * History:
 * 
 * $Log: pxp_lex_defs_drv_utf8.def,v $
 * Revision 1.1  2002/10/27 20:02:52  simeon
 * October 27, 2002 - Jerome
 *
 *   ** Welcome to the big fall cleanup! **
 *
 *   ** This is a large reorganization which will require people to
 *   ** migrate. Recommended is to do a full checkout again, except for
 *   ** people who made actual changes. More changes in the terminology
 *   ** inside the Datamodel and Datatypes modules are still expected.
 *
 *   - Fairly deep cleanup of the directory structure, file names, and
 *     module organization. Here is the new organization, in compilation
 *     order:
 *
 *         o ./base	 --  Basic modules, not really XML specific
 *
 *             XML related modules have been moved later in the
 *             compilation chain. Remains:
 *
 *               Gmisc -- Additions to Caml's stdlib
 *               Id -- Identifiers generation
 *               Pool -- String pools
 *               Finfo -- Handles locations in file
 *               Error -- Galax error handling
 *               Pervasive -- Wrapping of the pervasive.xq file
 *               Conf -- Global variables and configuration parameters
 * 	      Args -- Dealing with function arguments
 * 	      Whitespace -- Operations on whitespace in strings
 * 	      Occurrence -- Operations on occurrence indicators
 *
 *         o ./fsa		 -- Generic package for finite state automatas
 *
 *             This package has been made generic. Most modules only
 *             contains functors, parameterized by an alphabet of
 *             letters, which will make the transitions for the
 *             automatas.
 *
 *               Fsautil -- Functions for the rest of the FSA package
 * 	      Nfa -- Non deterministic FSAs
 * 	      Dfa -- Deterministic FSAs
 * 	      Regexp -- Regular expressions and Glushkov construction
 * 	        from regular expressions to NFA and DFA. This module
 * 	        relies on the fact that states in the automata should
 * 	        be positions in the regular expression, as specified
 * 	        in the Glushkov construction. This also contains
 * 	        intersection and reachability (emptiness check) for
 * 	        the automata(s)
 *
 *         o ./namespace	--  Operations on XML names, and namespaces
 *
 *             All basic support for XML names is located here.
 *
 * 	      Qname_lexer_{encoding} -- Lexer for qualified names for
 *   	        a given character encoding
 *               Qname_lexer -- toplevel access to the QName lexer(s)
 * 	      Namespace -- Support for namespace names, local names,
 *   	        namespace prefixes and URIs. It also implements
 *   	        namespace environments
 *               Sym -- Support for 'compiled' names, i.e., names stored
 *   	        somewhere in a string pool
 *               Resolve -- Support for namespace resolution. Given a
 *   	        qname and a namespace environment, it can return you a
 *   	        resolved name
 *
 *         o ./datatypes	--  Support for XML Schema datatypes
 *
 *             Support for XML Schema datatypes should now all be located
 *             in this directory.
 *
 *               Decimal -- xsd:int, xsd:integer, xsd:decimal
 * 	      DateTime -- xsd:dateTime, xsd:date, xsd:time
 * 	      AnyURI -- xsd:anyURI
 * 	      Datatypes_lexer -- Lexer for the various datatypes
 * 	      Datatypes -- Top-level support for datatypes. Contains
 * 	        implementation for atomic values in the XML Query data
 * 	        model
 *
 *         o ./parsing	--  Abstract Syntax Trees, XML&XQuery parsers
 *
 *             The parsing
 *
 *               Xml_ast -- XML Abstract Syntax Tree (if materialized)
 * 	      Xquery_ast -- XQuery Abstract Syntax Tree
 * 	      Print -- Printing functions for the AST's
 *  	      Xml_astutil -- Construction and misc. operations on the
 * 	        XML AST
 *  	      Xquery_astutil -- Construction and misc. operations on
 * 	        the XQuery AST
 *               Lexutil -- Lexing utilities, including support for
 *   	        multiple lexical states
 *               Xquery_parser -- XQuery parser
 *               Document_lexer_{encoding} -- XML Document prolog lexer
 * 	        for a given character encoding
 *               Opening_tag_lexer_{encoding} -- Opening tag lexer for a
 * 	        given character encoding
 *               Closing_tag_lexer_{encoding} -- Closing tag lexer for a
 * 	        given character encoding
 *               Text_lexer_{encoding} -- Text content lexer for a given
 * 	        character encoding
 *               Lexers_{encoding} -- Toplevel access to the lexer(s) for
 *                 a given encoding
 *               Xmldecl_lexer -- XML declaration lexer
 * 	      Type_lexer -- XQuery type system lexer
 * 	      Xquery_lexer -- Main XQuery lexer
 *
 *         o ./monitor	--  Monitoring of Galax activity
 *
 *               Monitor -- Used to monitor CPU and memory consumption
 *
 *         o ./schema      --  XML Schema and XQuery type system
 *
 *             One step further in typing, now support for XML Schema and
 *             the XQuery type system. A major rewrite of this package is
 *             expected [Support for named typing].
 *
 *               Glushkov -- Wrapper on top of the FSA
 *                 package. Implements low-level inclusion and
 *                 intersection over content models
 *               Schemamapping -- Mapping from XML Schema to the XQuery
 *                 type system
 *               Schemautil -- Various operations on types. Pretty messy
 *                 in there...
 *               Theta -- Subsumption mappings between type names, used
 *                 during the subtyping algorithm
 *               Subsumption -- Main subtyping module
 *               Simplification -- simplification over computed types
 *               Typeutil -- More operations on types... used in the rest
 *                 of the type checking
 *
 *         o ./normalization -- Query normalization
 *
 *             This module corresponds to normalization as specified in
 *             the XQuery 1.0 and XPath 2.0 Formal Semantics.
 *
 *   - Clean up of terminology in several modules, notably datatypes, and
 *     data model. Here are the detailed changes:
 *       --> Bound module		becomes Occurrence module
 *           Bound.bound type	becomes Occurrence.occurs type
 *       --> Simple module		becomes Datatypes module
 *       --> Data in Simple	becomes Atomic in Datatype
 *       --> _text in Simple	becomes _untyped in Datatype
 *       --> oforest 		becomes Sequence
 *       --> InstanceNode		becomes Node
 *       --> Instance		becomes Item
 *       --> InstanceForest	becomes Value
 *
 *   - Node names are aligned with names in the XQuery data model
 *     document.
 *       --> ElemNode		becomes ElementNode
 *       --> AttrNode		becomes AttributeNode
 *       --> DocNode		becomes DocumentNode
 *       --> NSNode		becomes NamespaceNode
 *       --> PINode		becomes ProcessingInstructionNode
 *
 *   - Fixed bug in toplevel Galax processing files in reverse input
 *     order when given on the command line! :^)
 *
 * October 26, 2002 - Jerome
 *
 *   - Made a number of changes to try to understand the reason the data
 *     model is so big in memory. Problems are in part due to the way
 *     memory measure is done, which makes difficult to measure the
 *     actual size of a memory object.
 *
 *       o Added some printing of the size of the string
 *         pools.
 *       o Simplified the way some of the loading with projection is done
 *         to avoid registering names which are discarded by projection.
 *       o Changed the SAX events to use QNames instead of symbols from
 *         the pools. This is better since this way events do not get
 *         recorded in the string pools. SAX event are discarded anyway.
 *
 * October 25, 2002 - Jerome
 *
 *   - Fixed bug in xf:index-of which was not using node identity has it
 *     should have. This bug was triggered by projection, since it was
 *     removing many other nodes and making it more likely that other
 *     nodes would have the same structure among siblings.
 *   - Fixed monitor to print the phase name for each line in the
 *     summary.
 *   - Fixed the document function to make sure it calls the monitor.
 *
 * Revision 1.1  2002/09/03 19:52:09  simeon
 * Added support for multiple character encodings.
 *
 * Revision 1.2  2000/08/26 19:58:08  gerd
 * 	Bugfix in character_except_apos. The bug caused that attribute
 * values delimited by &apos; could not be scanned at all.
 *
 * Revision 1.1  2000/05/20 20:33:25  gerd
 * 	Initial revision.
 *
 * 
 *)
